From ea2dc59aaa687f3a4eb3d58b03fef6c99f759cd0 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Wed, 17 Dec 2025 17:06:23 -0800 Subject: [PATCH 001/137] wip --- Cargo.lock | 1175 ++--------------- Cargo.toml | 58 +- artifact/Cargo.toml | 5 +- artifact/src/artifact.rs | 217 ++- artifact/src/hash.rs | 102 ++ artifact/src/hubris.rs | 71 + artifact/src/installinator.rs | 50 +- artifact/src/kind.rs | 361 ----- artifact/src/lib.rs | 7 +- artifact/src/tags.rs | 204 +++ artifact/src/version.rs | 7 +- bin/Cargo.toml | 25 +- bin/README.adoc | 36 - .../duplicate-artifact-2.toml | 24 - bin/invalid-manifests/duplicate-artifact.toml | 14 - .../duplicate-zone-file-name.toml | 14 - bin/manifests/fake-non-semver.toml | 94 -- bin/manifests/fake.toml | 93 -- bin/src/assemble.rs | 105 ++ bin/src/date.rs | 21 - bin/src/dispatch.rs | 200 --- bin/src/hint.rs | 29 - bin/src/lib.rs | 9 - bin/src/main.rs | 23 +- bin/src/sign.rs | 87 ++ bin/tests/integration-tests/command_tests.rs | 199 --- bin/tests/integration-tests/main.rs | 5 - bin/tests/manifest-tests.rs | 18 - brand-metadata/Cargo.toml | 1 - brand-metadata/src/lib.rs | 11 +- lib/Cargo.toml | 26 +- lib/src/archive.rs | 297 ----- lib/src/artifact.rs | 638 --------- lib/src/artifact/composite.rs | 220 --- lib/src/assemble/build.rs | 167 --- lib/src/assemble/deployment_unit.rs | 262 ---- lib/src/assemble/manifest.rs | 1019 -------------- lib/src/edit/editor.rs | 498 +++++++ lib/src/edit/key.rs | 62 + lib/src/{assemble => edit}/mod.rs | 15 +- lib/src/{ => edit}/root.rs | 43 +- lib/src/edit/sign.rs | 293 ++++ lib/src/edit/source.rs | 257 ++++ lib/src/error.rs | 215 +++ lib/src/key.rs | 84 -- lib/src/lib.rs | 23 +- lib/src/loader.rs | 216 +++ lib/src/repo.rs | 204 +++ lib/src/repo/v1.rs | 506 +++++++ lib/src/repository.rs | 670 ---------- lib/src/schema.rs | 27 + lib/src/target.rs | 106 -- lib/src/util.rs | 93 ++ lib/src/utils.rs | 60 - lib/src/zip_transport.rs | 457 +++++++ lib/src/zip_writer.rs | 199 +++ lib/tests/tests.rs | 71 + rust-toolchain.toml | 2 +- 58 files changed, 3988 insertions(+), 6007 deletions(-) create mode 100644 artifact/src/hash.rs create mode 100644 artifact/src/hubris.rs delete mode 100644 artifact/src/kind.rs create mode 100644 artifact/src/tags.rs delete mode 100644 bin/README.adoc delete mode 100644 bin/invalid-manifests/duplicate-artifact-2.toml delete mode 100644 bin/invalid-manifests/duplicate-artifact.toml delete mode 100644 bin/invalid-manifests/duplicate-zone-file-name.toml delete mode 100644 bin/manifests/fake-non-semver.toml delete mode 100644 bin/manifests/fake.toml create mode 100644 bin/src/assemble.rs delete mode 100644 bin/src/date.rs delete mode 100644 bin/src/dispatch.rs delete mode 100644 bin/src/hint.rs delete mode 100644 bin/src/lib.rs create mode 100644 bin/src/sign.rs delete mode 100644 bin/tests/integration-tests/command_tests.rs delete mode 100644 bin/tests/integration-tests/main.rs delete mode 100644 bin/tests/manifest-tests.rs delete mode 100644 lib/src/archive.rs delete mode 100644 lib/src/artifact.rs delete mode 100644 lib/src/artifact/composite.rs delete mode 100644 lib/src/assemble/build.rs delete mode 100644 lib/src/assemble/deployment_unit.rs delete mode 100644 lib/src/assemble/manifest.rs create mode 100644 lib/src/edit/editor.rs create mode 100644 lib/src/edit/key.rs rename lib/src/{assemble => edit}/mod.rs (62%) rename lib/src/{ => edit}/root.rs (54%) create mode 100644 lib/src/edit/sign.rs create mode 100644 lib/src/edit/source.rs create mode 100644 lib/src/error.rs delete mode 100644 lib/src/key.rs create mode 100644 lib/src/loader.rs create mode 100644 lib/src/repo.rs create mode 100644 lib/src/repo/v1.rs delete mode 100644 lib/src/repository.rs create mode 100644 lib/src/schema.rs delete mode 100644 lib/src/target.rs create mode 100644 lib/src/util.rs delete mode 100644 lib/src/utils.rs create mode 100644 lib/src/zip_transport.rs create mode 100644 lib/src/zip_writer.rs create mode 100644 lib/tests/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 20dcd74..2f87fa2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -107,18 +107,6 @@ name = "anyhow" version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b964d184e89d9b6b67dd2715bc8e74cf3107fb2b529990c90cf517326150bf4" -dependencies = [ - "backtrace", -] - -[[package]] -name = "arbitrary" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" -dependencies = [ - "derive_arbitrary", -] [[package]] name = "arc-swap" @@ -135,22 +123,6 @@ dependencies = [ "nodrop", ] -[[package]] -name = "assert_cmd" -version = "2.0.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1835b7f27878de8525dc71410b5a31cdcc5f230aed5ba5df968e09c201b23d" -dependencies = [ - "anstyle", - "bstr", - "doc-comment", - "libc", - "predicates", - "predicates-core", - "predicates-tree", - "wait-timeout", -] - [[package]] name = "async-recursion" version = "1.1.1" @@ -162,28 +134,6 @@ dependencies = [ "syn 2.0.96", ] -[[package]] -name = "async-stream" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" -dependencies = [ - "async-stream-impl", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-stream-impl" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.96", -] - [[package]] name = "async-trait" version = "0.1.86" @@ -196,10 +146,15 @@ dependencies = [ ] [[package]] -name = "atomic-waker" -version = "1.1.2" +name = "atomicwrites" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +checksum = "3ef1bb8d1b645fe38d51dfc331d720fb5fc2c94b440c76cc79c80ff265ca33e3" +dependencies = [ + "rustix 0.38.37", + "tempfile", + "windows-sys 0.52.0", +] [[package]] name = "autocfg" @@ -275,7 +230,7 @@ dependencies = [ "bitflags", "cexpr", "clang-sys", - "itertools 0.12.1", + "itertools", "lazy_static", "lazycell", "log", @@ -347,19 +302,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c" dependencies = [ "memchr", - "regex-automata", "serde", ] -[[package]] -name = "buf-list" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f56bd1685d994a3e2a3ed802eb1ecee8cb500b0ad4df48cb4d5d1a2f04749c3a" -dependencies = [ - "bytes", -] - [[package]] name = "bumpalo" version = "3.16.0" @@ -404,9 +349,6 @@ name = "camino" version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3" -dependencies = [ - "serde", -] [[package]] name = "camino-tempfile" @@ -452,8 +394,10 @@ checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" dependencies = [ "android-tzdata", "iana-time-zone", + "js-sys", "num-traits", "serde", + "wasm-bindgen", "windows-link", ] @@ -590,9 +534,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.13" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" dependencies = [ "crossbeam-utils", ] @@ -634,24 +578,6 @@ dependencies = [ "syn 2.0.96", ] -[[package]] -name = "datatest-stable" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a560b3fd20463b56397bd457aa71243ccfdcffe696050b66e3b1e0ec0457e7f1" -dependencies = [ - "camino", - "fancy-regex", - "libtest-mimic", - "walkdir", -] - -[[package]] -name = "debug-ignore" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffe7ed1d93f4553003e20b629abe9085e1e81b1429520f897f8f8860bc6dfc21" - [[package]] name = "der" version = "0.7.9" @@ -685,23 +611,6 @@ dependencies = [ "powerfmt", ] -[[package]] -name = "derive_arbitrary" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.96", -] - -[[package]] -name = "difflib" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" - [[package]] name = "digest" version = "0.10.7" @@ -713,27 +622,6 @@ dependencies = [ "crypto-common", ] -[[package]] -name = "dirs-next" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" -dependencies = [ - "cfg-if", - "dirs-sys-next", -] - -[[package]] -name = "dirs-sys-next" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" -dependencies = [ - "libc", - "redox_users", - "winapi", -] - [[package]] name = "displaydoc" version = "0.2.5" @@ -745,77 +633,6 @@ dependencies = [ "syn 2.0.96", ] -[[package]] -name = "doc-comment" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" - -[[package]] -name = "dropshot" -version = "0.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84e9c34a06ac21fefe60cf9e5cc321eac9f3d3e2d693e030da3709cf4275479" -dependencies = [ - "async-stream", - "async-trait", - "base64 0.22.1", - "bytes", - "camino", - "chrono", - "debug-ignore", - "dropshot_endpoint", - "form_urlencoded", - "futures", - "hostname 0.4.0", - "http", - "http-body-util", - "hyper", - "hyper-util", - "indexmap 2.7.1", - "multer", - "openapiv3", - "paste", - "percent-encoding", - "rustls 0.22.4", - "rustls-pemfile", - "schemars", - "scopeguard", - "semver", - "serde", - "serde_json", - "serde_path_to_error", - "serde_urlencoded", - "sha1", - "slog", - "slog-async", - "slog-bunyan", - "slog-json", - "slog-term", - "thiserror 2.0.12", - "tokio", - "tokio-rustls", - "toml 0.8.20", - "uuid", - "version_check", - "waitgroup", -] - -[[package]] -name = "dropshot_endpoint" -version = "0.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e4c7e4e96bfedd670ecbaffc1848ab28dd5892b214003517d9667e7a5b465ce" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "semver", - "serde", - "serde_tokenstream", - "syn 2.0.96", -] - [[package]] name = "dunce" version = "1.0.5" @@ -840,15 +657,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" -[[package]] -name = "encoding_rs" -version = "0.8.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" -dependencies = [ - "cfg-if", -] - [[package]] name = "env_logger" version = "0.10.2" @@ -876,12 +684,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "escape8259" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5692dd7b5a1978a5aeb0ce83b7655c58ca8efdcb79d21036ea249da95afec2c6" - [[package]] name = "expectorate" version = "1.1.0" @@ -893,17 +695,6 @@ dependencies = [ "similar", ] -[[package]] -name = "fancy-regex" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2" -dependencies = [ - "bit-set", - "regex-automata", - "regex-syntax", -] - [[package]] name = "fastrand" version = "2.1.1" @@ -938,15 +729,6 @@ dependencies = [ "miniz_oxide", ] -[[package]] -name = "float-cmp" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" -dependencies = [ - "num-traits", -] - [[package]] name = "fnv" version = "1.0.7" @@ -955,20 +737,21 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "form_urlencoded" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" dependencies = [ "percent-encoding", ] [[package]] name = "fs-err" -version = "2.11.0" +version = "3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88a41f105fe1d5b6b34b2055e3dc59bb79b46b48b2040b9e6c7b4b5de097aa41" +checksum = "824f08d01d0f496b3eca4f001a13cf17690a6ee930043d20817f547455fd98f8" dependencies = [ "autocfg", + "tokio", ] [[package]] @@ -1130,25 +913,6 @@ dependencies = [ "regex-syntax", ] -[[package]] -name = "h2" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http", - "indexmap 2.7.1", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "hashbrown" version = "0.12.3" @@ -1206,74 +970,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "hostname" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" -dependencies = [ - "libc", - "match_cfg", - "winapi", -] - -[[package]] -name = "hostname" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9c7c7c8ac16c798734b8a24560c1362120597c40d5e1459f09498f8f6c8f2ba" -dependencies = [ - "cfg-if", - "libc", - "windows", -] - -[[package]] -name = "http" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - -[[package]] -name = "http-body" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" -dependencies = [ - "bytes", - "http", -] - -[[package]] -name = "http-body-util" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" -dependencies = [ - "bytes", - "futures-util", - "http", - "http-body", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" - -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - [[package]] name = "hubtools" version = "0.4.6" @@ -1288,56 +984,10 @@ dependencies = [ "thiserror 1.0.69", "tlvc", "tlvc-text", - "toml 0.7.8", + "toml", "x509-cert", "zerocopy 0.6.6", - "zip 0.6.6", -] - -[[package]] -name = "humantime" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" - -[[package]] -name = "hyper" -version = "1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "256fb8d4bd6413123cc9d91832d78325c48ff41677595be797d90f42969beae0" -dependencies = [ - "bytes", - "futures-channel", - "futures-util", - "h2", - "http", - "http-body", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "smallvec 1.14.0", - "tokio", - "want", -] - -[[package]] -name = "hyper-util" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" -dependencies = [ - "bytes", - "futures-channel", - "futures-util", - "http", - "http-body", - "hyper", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", + "zip", ] [[package]] @@ -1483,9 +1133,9 @@ dependencies = [ [[package]] name = "idna" -version = "1.0.3" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" dependencies = [ "idna_adapter", "smallvec 1.14.0", @@ -1502,12 +1152,6 @@ dependencies = [ "icu_properties", ] -[[package]] -name = "indent_write" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cfe9645a18782869361d9c8732246be7b410ad4e919d3609ebabdac00ba12c3" - [[package]] name = "indexmap" version = "1.9.3" @@ -1526,14 +1170,18 @@ checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" dependencies = [ "equivalent", "hashbrown 0.15.1", - "serde", ] [[package]] -name = "ipnet" -version = "2.10.1" +name = "io-uring" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" +checksum = "fdd7bddefd0a8833b88a4b68f90dae22c7450d11b354198baee3874fd811b344" +dependencies = [ + "bitflags", + "cfg-if", + "libc", +] [[package]] name = "is-terminal" @@ -1561,15 +1209,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.11" @@ -1642,18 +1281,6 @@ dependencies = [ "redox_syscall", ] -[[package]] -name = "libtest-mimic" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc0bda45ed5b3a2904262c1bb91e526127aa70e7ef3758aba2ef93cf896b9b58" -dependencies = [ - "clap", - "escape8259", - "termcolor", - "threadpool", -] - [[package]] name = "linux-raw-sys" version = "0.4.14" @@ -1672,16 +1299,6 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" -[[package]] -name = "lock_api" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" -dependencies = [ - "autocfg", - "scopeguard", -] - [[package]] name = "log" version = "0.4.22" @@ -1724,12 +1341,6 @@ dependencies = [ "zerocopy 0.6.6", ] -[[package]] -name = "match_cfg" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" - [[package]] name = "maybe-uninit" version = "2.0.0" @@ -1742,12 +1353,6 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1775,23 +1380,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "multer" -version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" -dependencies = [ - "bytes", - "encoding_rs", - "futures-util", - "http", - "httparse", - "memchr", - "mime", - "spin", - "version_check", -] - [[package]] name = "newline-converter" version = "0.3.0" @@ -1817,19 +1405,12 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "normalize-line-endings" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" - [[package]] name = "num-bigint-dig" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" dependencies = [ - "byteorder", "lazy_static", "libm", "num-integer", @@ -1877,25 +1458,6 @@ dependencies = [ "libm", ] -[[package]] -name = "num_cpus" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" -dependencies = [ - "hermit-abi 0.3.9", - "libc", -] - -[[package]] -name = "num_threads" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" -dependencies = [ - "libc", -] - [[package]] name = "object" version = "0.30.4" @@ -1934,17 +1496,6 @@ version = "1.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" -[[package]] -name = "openapiv3" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc02deea53ffe807708244e5914f6b099ad7015a207ee24317c22112e17d9c5c" -dependencies = [ - "indexmap 2.7.1", - "serde", - "serde_json", -] - [[package]] name = "packed_struct" version = "0.10.1" @@ -1967,35 +1518,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "parking_lot" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec 1.14.0", - "windows-targets", -] - -[[package]] -name = "parse-size" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "487f2ccd1e17ce8c1bfab3a65c89525af41cfad4c8659021a1e9a2aacd73b89b" - [[package]] name = "paste" version = "1.0.15" @@ -2029,9 +1551,9 @@ dependencies = [ [[package]] name = "percent-encoding" -version = "2.3.1" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pin-project" @@ -2107,36 +1629,6 @@ dependencies = [ "zerocopy 0.7.35", ] -[[package]] -name = "predicates" -version = "3.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5d19ee57562043d37e82899fade9a22ebab7be9cef5026b07fda9cdd4293573" -dependencies = [ - "anstyle", - "difflib", - "float-cmp", - "normalize-line-endings", - "predicates-core", - "regex", -] - -[[package]] -name = "predicates-core" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae8177bee8e75d6846599c6b9ff679ed51e882816914eec639944d7c9aa11931" - -[[package]] -name = "predicates-tree" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41b740d195ed3166cd147c8047ec98db0e22ec019eb8eeb76d343b795304fb13" -dependencies = [ - "predicates-core", - "termtree", -] - [[package]] name = "prettyplease" version = "0.2.29" @@ -2242,6 +1734,12 @@ dependencies = [ "rand_core", ] +[[package]] +name = "rawzip" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27cc19f82b641448e861623f52a6a6413bbc0595b62a9d45bf31ccdf18aab72c" + [[package]] name = "redox_syscall" version = "0.5.7" @@ -2251,17 +1749,6 @@ dependencies = [ "bitflags", ] -[[package]] -name = "redox_users" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" -dependencies = [ - "getrandom 0.2.15", - "libredox", - "thiserror 1.0.69", -] - [[package]] name = "regex" version = "1.11.1" @@ -2291,44 +1778,6 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" -[[package]] -name = "reqwest" -version = "0.12.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43e734407157c3c2034e0258f5e4473ddb361b1e85f95a66690d67264d7cd1da" -dependencies = [ - "base64 0.22.1", - "bytes", - "futures-core", - "futures-util", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-util", - "ipnet", - "js-sys", - "log", - "mime", - "once_cell", - "percent-encoding", - "pin-project-lite", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tokio-util", - "tower", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-streams", - "web-sys", - "windows-registry", -] - [[package]] name = "ring" version = "0.17.13" @@ -2415,20 +1864,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "rustls" -version = "0.22.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" -dependencies = [ - "log", - "ring", - "rustls-pki-types", - "rustls-webpki", - "subtle", - "zeroize", -] - [[package]] name = "rustls" version = "0.23.19" @@ -2444,15 +1879,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "rustls-pemfile" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" -dependencies = [ - "rustls-pki-types", -] - [[package]] name = "rustls-pki-types" version = "1.10.0" @@ -2471,12 +1897,6 @@ dependencies = [ "untrusted 0.9.0", ] -[[package]] -name = "rustversion" -version = "1.0.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" - [[package]] name = "rusty-fork" version = "0.3.0" @@ -2512,10 +1932,8 @@ checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615" dependencies = [ "dyn-clone", "schemars_derive", - "semver", "serde", "serde_json", - "uuid", ] [[package]] @@ -2530,12 +1948,6 @@ dependencies = [ "syn 2.0.96", ] -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - [[package]] name = "semver" version = "1.0.25" @@ -2608,16 +2020,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_path_to_error" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6" -dependencies = [ - "itoa", - "serde", -] - [[package]] name = "serde_plain" version = "1.0.2" @@ -2636,41 +2038,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_tokenstream" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64060d864397305347a78851c51588fd283767e7e7589829e8121d65512340f1" -dependencies = [ - "proc-macro2", - "quote", - "serde", - "syn 2.0.96", -] - -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "sha1" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - [[package]] name = "sha2" version = "0.10.8" @@ -2688,15 +2055,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" -[[package]] -name = "signal-hook-registry" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" -dependencies = [ - "libc", -] - [[package]] name = "signature" version = "2.2.0" @@ -2740,18 +2098,6 @@ dependencies = [ "thread_local", ] -[[package]] -name = "slog-bunyan" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcaaf6e68789d3f0411f1e72bc443214ef252a1038b6e344836e50442541f190" -dependencies = [ - "hostname 0.3.1", - "slog", - "slog-json", - "time", -] - [[package]] name = "slog-envlogger" version = "2.2.0" @@ -2767,18 +2113,6 @@ dependencies = [ "slog-term", ] -[[package]] -name = "slog-json" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e1e53f61af1e3c8b852eef0a9dee29008f55d6dd63794f3f12cef786cf0f219" -dependencies = [ - "serde", - "serde_json", - "slog", - "time", -] - [[package]] name = "slog-scope" version = "4.4.0" @@ -2803,10 +2137,11 @@ dependencies = [ [[package]] name = "slog-term" -version = "2.9.1" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6e022d0b998abfe5c3782c1f03551a596269450ccd677ea51c56f8b214610e8" +checksum = "5cb1fc680b38eed6fad4c02b3871c09d2c81db8c96aa4e9c0a34904c830f09b5" dependencies = [ + "chrono", "is-terminal", "slog", "term", @@ -2852,16 +2187,6 @@ dependencies = [ "syn 2.0.96", ] -[[package]] -name = "socket2" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - [[package]] name = "spin" version = "0.9.8" @@ -2913,28 +2238,6 @@ dependencies = [ "syn 2.0.96", ] -[[package]] -name = "strum" -version = "0.26.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" -dependencies = [ - "strum_macros", -] - -[[package]] -name = "strum_macros" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.96", -] - [[package]] name = "subtle" version = "2.6.1" @@ -2963,15 +2266,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "sync_wrapper" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" -dependencies = [ - "futures-core", -] - [[package]] name = "synstructure" version = "0.13.1" @@ -3021,13 +2315,11 @@ dependencies = [ [[package]] name = "term" -version = "0.7.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +checksum = "d8c27177b12a6399ffc08b98f76f7c9a1f4fe9fc967c784c5a071fa8d93cf7e1" dependencies = [ - "dirs-next", - "rustversion", - "winapi", + "windows-sys 0.59.0", ] [[package]] @@ -3049,12 +2341,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "termtree" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" - [[package]] name = "test-strategy" version = "0.4.0" @@ -3117,15 +2403,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "threadpool" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" -dependencies = [ - "num_cpus", -] - [[package]] name = "time" version = "0.3.36" @@ -3134,9 +2411,7 @@ checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ "deranged", "itoa", - "libc", "num-conv", - "num_threads", "powerfmt", "serde", "time-core", @@ -3228,20 +2503,18 @@ dependencies = [ [[package]] name = "tokio" -version = "1.43.0" +version = "1.47.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" +checksum = "36cde6a64bcbb101731e7db34c087674206357a5316e6f695f5fef730bd711de" dependencies = [ "backtrace", "bytes", + "io-uring", "libc", "mio", - "parking_lot", "pin-project-lite", - "signal-hook-registry", - "socket2", + "slab", "tokio-macros", - "windows-sys 0.52.0", ] [[package]] @@ -3255,22 +2528,11 @@ dependencies = [ "syn 2.0.96", ] -[[package]] -name = "tokio-rustls" -version = "0.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" -dependencies = [ - "rustls 0.22.4", - "rustls-pki-types", - "tokio", -] - [[package]] name = "tokio-util" -version = "0.7.13" +version = "0.7.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" +checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594" dependencies = [ "bytes", "futures-core", @@ -3288,19 +2550,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.19.15", -] - -[[package]] -name = "toml" -version = "0.8.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit 0.22.24", + "toml_edit", ] [[package]] @@ -3322,27 +2572,14 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "winnow 0.5.40", -] - -[[package]] -name = "toml_edit" -version = "0.22.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" -dependencies = [ - "indexmap 2.7.1", - "serde", - "serde_spanned", - "toml_datetime", - "winnow 0.7.3", + "winnow", ] [[package]] name = "tough" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fe73519c5c485dc0b585088523ad861cda19836b2eb94896fac278db68bd5ab" +checksum = "e88d0ee9525696569cc2af5d46f8a739028c0268895071e0386957195b0c9161" dependencies = [ "async-recursion", "async-trait", @@ -3358,8 +2595,7 @@ dependencies = [ "olpc-cjson", "pem", "percent-encoding", - "reqwest", - "rustls 0.23.19", + "rustls", "serde", "serde_json", "serde_plain", @@ -3373,154 +2609,82 @@ dependencies = [ "walkdir", ] -[[package]] -name = "tower" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" -dependencies = [ - "futures-core", - "futures-util", - "pin-project-lite", - "sync_wrapper", - "tokio", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - -[[package]] -name = "tower-service" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" - -[[package]] -name = "tracing" -version = "0.1.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" -dependencies = [ - "pin-project-lite", - "tracing-core", -] - -[[package]] -name = "tracing-core" -version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" -dependencies = [ - "once_cell", -] - -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - [[package]] name = "tufaceous" -version = "0.1.0" +version = "0.2.0" dependencies = [ - "anyhow", - "assert_cmd", + "atomicwrites", + "aws-lc-rs", + "bytes", "camino", + "camino-tempfile", "chrono", - "clap", - "console", - "datatest-stable", - "dropshot", - "fs-err", - "humantime", - "predicates", + "flate2", + "futures-util", + "hex", + "hubtools", + "rawzip", "semver", + "serde", + "serde_json", + "sha2", "slog", - "slog-async", - "slog-envlogger", - "slog-term", - "tempfile", + "tar", + "thiserror 2.0.12", "tokio", + "tokio-util", + "tough", "tufaceous-artifact", - "tufaceous-lib", + "tufaceous-brand-metadata", + "url", ] [[package]] name = "tufaceous-artifact" -version = "0.1.0" +version = "0.2.0" dependencies = [ "daft", "expectorate", "hex", + "hubtools", "proptest", "regex", "schemars", - "semver", "serde", "serde_human_bytes", "serde_json", - "strum", "test-strategy", "thiserror 2.0.12", ] [[package]] -name = "tufaceous-brand-metadata" -version = "0.1.0" +name = "tufaceous-bin" +version = "0.2.0" dependencies = [ + "anyhow", + "camino", + "chrono", + "clap", + "fs-err", "semver", - "serde", - "serde_json", - "tar", + "slog", + "slog-async", + "slog-envlogger", + "slog-term", + "tokio", + "tough", + "tufaceous", "tufaceous-artifact", ] [[package]] -name = "tufaceous-lib" -version = "0.1.0" +name = "tufaceous-brand-metadata" +version = "0.2.0" dependencies = [ - "anyhow", - "async-trait", - "aws-lc-rs", - "base64 0.22.1", - "buf-list", - "bytes", - "camino", - "camino-tempfile", - "chrono", - "daft", - "debug-ignore", - "dropshot", - "flate2", - "fs-err", - "futures", - "hex", - "hubtools", - "indent_write", - "itertools 0.13.0", - "parse-size", - "rand", - "semver", "serde", "serde_json", - "serde_path_to_error", - "sha2", - "slog", "tar", - "thiserror 2.0.12", - "tokio", - "toml 0.8.20", - "tough", "tufaceous-artifact", - "tufaceous-brand-metadata", - "url", - "zip 4.2.0", ] [[package]] @@ -3582,13 +2746,14 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.3" +version = "2.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" +checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] @@ -3609,16 +2774,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" -[[package]] -name = "uuid" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "744018581f9a3454a9e15beb8a33b017183f1e7c0cd170232a2d1453b23a51c4" -dependencies = [ - "getrandom 0.2.15", - "serde", -] - [[package]] name = "version_check" version = "0.9.5" @@ -3634,15 +2789,6 @@ dependencies = [ "libc", ] -[[package]] -name = "waitgroup" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1f50000a783467e6c0200f9d10642f4bc424e39efc1b770203e88b488f79292" -dependencies = [ - "atomic-waker", -] - [[package]] name = "walkdir" version = "2.5.0" @@ -3653,15 +2799,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -3703,18 +2840,6 @@ dependencies = [ "wasm-bindgen-shared", ] -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b" -dependencies = [ - "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", -] - [[package]] name = "wasm-bindgen-macro" version = "0.2.95" @@ -3744,29 +2869,6 @@ version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" -[[package]] -name = "wasm-streams" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e072d4e72f700fb3443d8fe94a39315df013eef1104903cdb0a2abd322bbecd" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - -[[package]] -name = "web-sys" -version = "0.3.72" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - [[package]] name = "which" version = "4.4.2" @@ -3779,22 +2881,6 @@ dependencies = [ "rustix 0.38.37", ] -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - [[package]] name = "winapi-util" version = "0.1.9" @@ -3804,22 +2890,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" -dependencies = [ - "windows-core", - "windows-targets", -] - [[package]] name = "windows-core" version = "0.52.0" @@ -3835,36 +2905,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dccfd733ce2b1753b03b6d3c65edf020262ea35e20ccdf3e288043e6dd620e3" -[[package]] -name = "windows-registry" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" -dependencies = [ - "windows-result", - "windows-strings", - "windows-targets", -] - -[[package]] -name = "windows-result" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-strings" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" -dependencies = [ - "windows-result", - "windows-targets", -] - [[package]] name = "windows-sys" version = "0.52.0" @@ -3956,15 +2996,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "winnow" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7f4ea97f6f78012141bcdb6a216b2609f0979ada50b20ca5b52dde2eac2bb1" -dependencies = [ - "memchr", -] - [[package]] name = "wit-bindgen-rt" version = "0.39.0" @@ -4158,15 +3189,3 @@ dependencies = [ "crc32fast", "crossbeam-utils", ] - -[[package]] -name = "zip" -version = "4.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ab361742de920c5535880f89bbd611ee62002bf11341d16a5f057bb8ba6899" -dependencies = [ - "arbitrary", - "crc32fast", - "indexmap 2.7.1", - "memchr", -] diff --git a/Cargo.toml b/Cargo.toml index 8d55e22..62f04e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,65 +8,59 @@ members = [ ] [workspace.package] -version = "0.1.0" -edition = "2021" +version = "0.2.0" +edition = "2024" license = "MPL-2.0" publish = false [workspace.dependencies] anyhow = "1.0.96" assert_cmd = "2.0.16" -async-trait = "0.1.86" +atomicwrites = "0.4.4" aws-lc-rs = "1.12.4" -base64 = "0.22.1" -buf-list = "1.0.3" bytes = "1.10.0" -camino = { version = "1.1.9", features = ["serde1"] } +camino = "1.1.9" camino-tempfile = "1.1.1" chrono = { version = "0.4.40", default-features = false, features = ["std"] } -clap = { version = "4.5.31", features = ["cargo", "derive", "env", "wrap_help"] } -console = { version = "0.15.10", default-features = false } +clap = { version = "4.5.31", features = ["cargo", "derive", "wrap_help"] } daft = { version = "0.1.1", features = ["derive"] } -datatest-stable = "0.2.9" -debug-ignore = "1.0.5" -dropshot = "0.15.1" expectorate = "1.1.0" flate2 = "1.0.35" -fs-err = "2.11.0" -futures = "0.3.31" +fs-err = "3.2.0" +futures-util = "0.3.31" hex = "0.4.3" hubtools = { git = "https://github.com/oxidecomputer/hubtools.git", branch = "main" } -humantime = "2.1.0" -indent_write = "2.2.0" -itertools = "0.13.0" -parse-size = "1.1.0" -predicates = "3.1.3" proptest = "1.5.0" -rand = "0.8.5" +rawzip = "0.4.2" regex = "1.11.1" -schemars = { version = "0.8.21", features = ["semver"] } +schemars = "0.8.21" semver = { version = "1.0.25", features = ["serde"] } serde = { version = "1.0.218", features = ["derive"] } serde_human_bytes = { git = "https://github.com/oxidecomputer/serde_human_bytes", branch = "main" } serde_json = "1.0.139" -serde_path_to_error = "0.1.16" sha2 = "0.10.8" slog = "2.7.0" slog-async = "2.8.0" slog-envlogger = "2.2.0" -slog-term = "2.9.1" -strum = { version = "0.26.3", features = ["derive"] } +slog-term = "2.9.2" tar = "0.4.44" -tempfile = "3.13.0" test-strategy = "0.4.0" thiserror = "2.0.12" -tokio = "1.43.0" -toml = "0.8.20" -tough = { version = "0.20.0", features = [ "http" ] } -tufaceous-artifact = { path = "artifact", default-features = false } -tufaceous-brand-metadata = { path = "brand-metadata" } -tufaceous-lib = { path = "lib" } -url = "2.5.3" -zip = { version = "4.2.0", default-features = false } +tokio = "1.47.0" +tokio-util = "0.7.17" +tough = "0.21.0" +tufaceous.path = "lib" +tufaceous-artifact.path = "artifact" +tufaceous-brand-metadata.path = "brand-metadata" +url = "2.5.4" [workspace.lints.clippy] + +[profile.dev.package.flate2] +opt-level = 3 +[profile.dev.package.miniz_oxide] +opt-level = 3 +[profile.dev.package.rawzip] +opt-level = 3 +[profile.dev.package.sha2] +opt-level = 3 diff --git a/artifact/Cargo.toml b/artifact/Cargo.toml index c0e8f99..d047b8d 100644 --- a/artifact/Cargo.toml +++ b/artifact/Cargo.toml @@ -6,18 +6,19 @@ license.workspace = true publish.workspace = true [features] +hubtools = ["dep:hubtools"] proptest = ["dep:proptest", "dep:test-strategy"] schemars = ["dep:schemars"] [dependencies] daft.workspace = true hex.workspace = true +hubtools = { workspace = true, optional = true } proptest = { workspace = true, optional = true } schemars = { workspace = true, optional = true } -semver.workspace = true serde.workspace = true serde_human_bytes.workspace = true -strum.workspace = true +serde_json.workspace = true test-strategy = { workspace = true, optional = true } thiserror.workspace = true diff --git a/artifact/src/artifact.rs b/artifact/src/artifact.rs index 5f66f14..27f11aa 100644 --- a/artifact/src/artifact.rs +++ b/artifact/src/artifact.rs @@ -2,164 +2,123 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use std::{fmt, str::FromStr}; +use std::collections::BTreeMap; +use std::collections::HashMap; +use std::collections::hash_map::IntoValues; +use std::collections::hash_map::Values; +use std::iter::Flatten; -use daft::Diffable; -use hex::FromHexError; -use semver::Version; -use serde::{Deserialize, Serialize}; +use crate::ArtifactHash; +use crate::ArtifactVersion; +use crate::KnownArtifactTags; -use crate::{ArtifactKind, ArtifactVersion}; +#[derive(Debug, Clone)] +pub struct Artifact { + pub target_name: String, + pub version: ArtifactVersion, + pub tags: BTreeMap, + pub sha256: ArtifactHash, + pub length: u64, +} + +impl Artifact { + pub fn known_tags(&self) -> Option { + KnownArtifactTags::from_tags(&self.tags).ok() + } +} -/// Description of the `artifacts.json` target found in rack update -/// repositories. -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ArtifactsDocument { - pub system_version: Version, - pub artifacts: Vec, +#[derive(Debug, Clone, Default)] +pub struct Artifacts { + inner: HashMap, Vec>, } -impl ArtifactsDocument { - /// The name of the artifacts document: `artifacts.json`. - pub const FILE_NAME: &'static str = "artifacts.json"; +impl Artifacts { + pub fn new(iter: impl IntoIterator) -> Self { + Self::from_iter(iter) + } - /// Creates an artifacts document with the provided system version and an - /// empty list of artifacts. - pub fn empty(system_version: Version) -> Self { - Self { system_version, artifacts: Vec::new() } + pub fn insert(&mut self, artifact: Artifact) { + self.inner.entry(artifact.known_tags()).or_default().push(artifact); } -} -/// Describes an artifact available in the repository. -#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] -pub struct Artifact { - /// Used to differentiate between different series of artifacts of the same - /// kind. This is used by the control plane to select the correct artifact. - /// - /// For SP and ROT images ([`GimletSp`](crate::KnownArtifactKind::GimletSp), - /// [`GimletRot`](crate::KnownArtifactKind::GimletRot), - /// [`PscSp`](crate::KnownArtifactKind::PscSp), - /// [`PscRot`](crate::KnownArtifactKind::PscRot), - /// [`SwitchSp`](crate::KnownArtifactKind::SwitchSp), - /// [`SwitchRot`](crate::KnownArtifactKind::SwitchRot)), `name` is the value - /// of the board (`BORD`) tag in the image caboose. - /// - /// In the future when - /// [`ControlPlane`](crate::KnownArtifactKind::ControlPlane) is split up - /// into separate zones, `name` will be the zone name. - pub name: String, - pub version: ArtifactVersion, - pub kind: ArtifactKind, - pub target: String, + pub fn get(&self, tags: KnownArtifactTags) -> Result<&Artifact, GetError> { + let vec = self.inner.get(&Some(tags)).ok_or(GetError::NotFound)?; + if vec.len() == 1 { Ok(&vec[0]) } else { Err(GetError::TooMany) } + } + + pub fn get_all(&self, tags: KnownArtifactTags) -> &[Artifact] { + self.inner.get(&Some(tags)).map(Vec::as_slice).unwrap_or_default() + } + + pub fn iter(&self) -> Iter<'_> { + Iter { inner: self.inner.values().flatten() } + } } -/// The hash of an artifact. -#[derive( - Copy, - Clone, - Diffable, - Eq, - PartialEq, - Ord, - PartialOrd, - Hash, - Serialize, - Deserialize, -)] -#[daft(leaf)] -#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] -#[cfg_attr(feature = "proptest", derive(test_strategy::Arbitrary))] -#[serde(transparent)] -pub struct ArtifactHash( - #[serde(with = "serde_human_bytes::hex_array")] - #[cfg_attr( - feature = "schemars", - schemars(schema_with = "hex_schema::<32>") - )] - pub [u8; 32], -); - -impl AsRef<[u8]> for ArtifactHash { - fn as_ref(&self) -> &[u8] { - &self.0 +impl Extend for Artifacts { + fn extend>(&mut self, iter: T) { + for artifact in iter { + self.insert(artifact); + } } } -impl fmt::Debug for ArtifactHash { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_tuple("ArtifactHash").field(&hex::encode(self.0)).finish() +impl FromIterator for Artifacts { + fn from_iter>(iter: T) -> Self { + let mut artifacts = Self::default(); + artifacts.extend(iter); + artifacts } } -impl fmt::Display for ArtifactHash { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - hex::encode(self.0).fmt(f) +impl IntoIterator for Artifacts { + type Item = Artifact; + type IntoIter = IntoIter; + + fn into_iter(self) -> Self::IntoIter { + IntoIter { inner: self.inner.into_values().flatten() } } } -impl FromStr for ArtifactHash { - type Err = FromHexError; +impl<'a> IntoIterator for &'a Artifacts { + type Item = &'a Artifact; + type IntoIter = Iter<'a>; - fn from_str(s: &str) -> std::result::Result { - let mut out = [0u8; 32]; - hex::decode_to_slice(s, &mut out)?; - Ok(Self(out)) + fn into_iter(self) -> Self::IntoIter { + self.iter() } } -/// A hash-based identifier for an artifact or deployment unit: the kind and -/// hash. -#[derive( - Debug, - Diffable, - Clone, - PartialEq, - Eq, - Hash, - Ord, - PartialOrd, - Deserialize, - Serialize, -)] -#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] -pub struct ArtifactHashId { - /// The kind of artifact this is. - pub kind: ArtifactKind, - - /// The hash of the artifact. - pub hash: ArtifactHash, +#[derive(Debug)] +pub struct IntoIter { + inner: Flatten, Vec>>, } -impl fmt::Display for ArtifactHashId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}/{}", self.kind, self.hash) +impl Iterator for IntoIter { + type Item = Artifact; + + fn next(&mut self) -> Option { + self.inner.next() } } -/// Produce an OpenAPI schema describing a hex array of a specific length (e.g., -/// a hash digest). -#[cfg(feature = "schemars")] -fn hex_schema( - gen: &mut schemars::SchemaGenerator, -) -> schemars::schema::Schema { - use schemars::JsonSchema; - - let mut schema: schemars::schema::SchemaObject = - ::json_schema(gen).into(); - schema.format = Some(format!("hex string ({N} bytes)")); - schema.into() +#[derive(Debug, Clone)] +pub struct Iter<'a> { + inner: Flatten, Vec>>, } -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn display_respects_padding() { - let h = ArtifactHash([0; 32]); - assert_eq!( - format!("{h:x>100}"), - "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx0000000000000000000000000000000000000000000000000000000000000000" - ); +impl<'a> Iterator for Iter<'a> { + type Item = &'a Artifact; + + fn next(&mut self) -> Option { + self.inner.next() } } + +#[derive(Debug, thiserror::Error)] +pub enum GetError { + #[error("artifact not found")] + NotFound, + #[error("more than one artifact found")] + TooMany, +} diff --git a/artifact/src/hash.rs b/artifact/src/hash.rs new file mode 100644 index 0000000..19ef43d --- /dev/null +++ b/artifact/src/hash.rs @@ -0,0 +1,102 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::array::TryFromSliceError; +use std::fmt; +use std::str::FromStr; + +use daft::Diffable; +use hex::FromHexError; +use serde::Deserialize; +use serde::Serialize; + +/// The hash of an artifact. +#[derive( + Copy, + Clone, + Diffable, + Eq, + PartialEq, + Ord, + PartialOrd, + Hash, + Serialize, + Deserialize, +)] +#[daft(leaf)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +#[cfg_attr(feature = "proptest", derive(test_strategy::Arbitrary))] +#[serde(transparent)] +pub struct ArtifactHash( + #[serde(with = "serde_human_bytes::hex_array")] + #[cfg_attr( + feature = "schemars", + schemars(schema_with = "hex_schema::<32>") + )] + pub [u8; 32], +); + +impl AsRef<[u8]> for ArtifactHash { + fn as_ref(&self) -> &[u8] { + &self.0 + } +} + +impl fmt::Debug for ArtifactHash { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("ArtifactHash").field(&hex::encode(self.0)).finish() + } +} + +impl fmt::Display for ArtifactHash { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + hex::encode(self.0).fmt(f) + } +} + +impl FromStr for ArtifactHash { + type Err = FromHexError; + + fn from_str(s: &str) -> std::result::Result { + let mut out = [0u8; 32]; + hex::decode_to_slice(s, &mut out)?; + Ok(Self(out)) + } +} + +impl TryFrom<&[u8]> for ArtifactHash { + type Error = TryFromSliceError; + + fn try_from(value: &[u8]) -> Result { + value.try_into().map(Self) + } +} + +/// Produce an OpenAPI schema describing a hex array of a specific length (e.g., +/// a hash digest). +#[cfg(feature = "schemars")] +fn hex_schema( + generator: &mut schemars::SchemaGenerator, +) -> schemars::schema::Schema { + use schemars::JsonSchema; + + let mut schema: schemars::schema::SchemaObject = + ::json_schema(generator).into(); + schema.format = Some(format!("hex string ({N} bytes)")); + schema.into() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn display_respects_padding() { + let h = ArtifactHash([0; 32]); + assert_eq!( + format!("{h:x>100}"), + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx0000000000000000000000000000000000000000000000000000000000000000" + ); + } +} diff --git a/artifact/src/hubris.rs b/artifact/src/hubris.rs new file mode 100644 index 0000000..255f373 --- /dev/null +++ b/artifact/src/hubris.rs @@ -0,0 +1,71 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +#![cfg(feature = "hubtools")] + +use hubtools::Caboose; +use hubtools::CabooseError; + +use crate::KnownArtifactTags; +use crate::RotSlot; + +impl KnownArtifactTags { + pub fn from_rot_caboose( + caboose: &Caboose, + slot: RotSlot, + ) -> Result { + Ok(KnownArtifactTags::Rot { + board: read_board(caboose)?, + sign: read_sign(caboose)?, + slot, + }) + } + + pub fn from_rot_bootloader_caboose( + caboose: &Caboose, + ) -> Result { + Ok(KnownArtifactTags::RotBootloader { + board: read_board(caboose)?, + sign: read_sign(caboose)?, + }) + } + + pub fn from_sp_caboose( + caboose: &Caboose, + ) -> Result { + Ok(KnownArtifactTags::Sp { board: read_board(caboose)? }) + } +} + +fn utf8<'a>( + s: &'a [u8], + tag: &'static str, +) -> Result<&'a str, ReadCabooseError> { + std::str::from_utf8(s) + .map_err(|source| ReadCabooseError::Utf8 { tag, source }) +} + +fn read_board(caboose: &Caboose) -> Result { + Ok(utf8(caboose.board()?, "BORD")?.to_owned()) +} + +fn read_sign(caboose: &Caboose) -> Result, ReadCabooseError> { + match caboose.sign() { + Ok(sign) => Ok(Some(utf8(sign, "SIGN")?.to_owned())), + Err(CabooseError::MissingTag { .. }) => Ok(None), + Err(error) => Err(error.into()), + } +} + +pub fn read_version(caboose: &Caboose) -> Result<&str, ReadCabooseError> { + utf8(caboose.version()?, "VERS") +} + +#[derive(Debug, thiserror::Error)] +pub enum ReadCabooseError { + #[error(transparent)] + Caboose(#[from] hubtools::CabooseError), + #[error("{tag} is not valid UTF-8")] + Utf8 { tag: &'static str, source: std::str::Utf8Error }, +} diff --git a/artifact/src/installinator.rs b/artifact/src/installinator.rs index 0c35cfd..19ad7f0 100644 --- a/artifact/src/installinator.rs +++ b/artifact/src/installinator.rs @@ -2,66 +2,44 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use semver::Version; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; +use serde::Serialize; use crate::ArtifactHash; /// Artifact-specific information used by installinator. /// -/// This document contains information used by installinator to learn about -/// which artifacts to fetch. Unlike -/// [`ArtifactsDocument`](crate::ArtifactsDocument): +/// This document is treated as an opaque blob by Wicketd and Nexus, since +/// we'd like previous versions of those services to be able to process newer +/// versions of this document. /// -/// * This document is treated as an opaque blob by Wicketd and Nexus, since -/// we'd like previous versions of those services to be able to process newer -/// versions of this document. -/// * There are no backwards compatibility constraints for this document. The -/// version of installinator that processes this document is the same as the -/// version of tufaceous that creates it. +/// There are no backwards compatibility constraints for this document. The +/// version of installinator that processes this document is the same as the +/// version of tufaceous that creates it. #[derive(Debug, Clone, Deserialize, Serialize)] pub struct InstallinatorDocument { - pub system_version: Version, pub artifacts: Vec, } -impl InstallinatorDocument { - /// Creates an installinator document with the provided system version and - /// an empty list of artifacts. - pub fn empty(system_version: Version) -> Self { - Self { system_version, artifacts: Vec::new() } - } - - pub fn file_name(&self) -> String { - format!("installinator_document-{}.json", self.system_version) - } -} - /// Describes an artifact available to installinator. /// /// The fields here match [`Artifact`](crate::Artifact). #[derive(Debug, Clone, Deserialize, Serialize)] pub struct InstallinatorArtifact { + /// A file name; not necessarily the target name. pub name: String, - /// The kind of artifact. - /// - /// This is an [`InstallinatorArtifactKind`] rather than an - /// [`ArtifactKind`](crate::ArtifactKind) because there aren't any backwards - /// compatibility constraints with `InstallinatorArtifact`. pub kind: InstallinatorArtifactKind, - pub hash: ArtifactHash, + pub sha256: ArtifactHash, } /// The artifact kind for an installinator artifact. #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(rename_all = "snake_case")] pub enum InstallinatorArtifactKind { + /// A measurement corpus. + MeasurementCorpus, /// The host phase 2 artifact. - /// - /// This is extracted from the composite host artifact. HostPhase2, - /// The composite control plane artifact. - ControlPlane, - /// Measurement Corpus - MeasurementCorpus, + /// A control plane zone artifact. + Zone { name: String }, } diff --git a/artifact/src/kind.rs b/artifact/src/kind.rs deleted file mode 100644 index d9f9d28..0000000 --- a/artifact/src/kind.rs +++ /dev/null @@ -1,361 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use std::{borrow::Cow, convert::Infallible, fmt, str::FromStr}; - -use daft::Diffable; -use serde::{Deserialize, Serialize}; -use strum::{Display, EnumIter, EnumString, IntoEnumIterator}; -use thiserror::Error; - -/// The kind of artifact we are dealing with. -/// -/// To ensure older versions of Nexus can work with update repositories that -/// describe artifact kinds it is not yet aware of, this is a newtype wrapper -/// around a string. The set of known artifact kinds is described in -/// [`KnownArtifactKind`], and this type has conversions to and from it. -#[derive( - Debug, - Diffable, - Clone, - PartialEq, - Eq, - Hash, - Ord, - PartialOrd, - Deserialize, - Serialize, -)] -#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] -#[serde(transparent)] -pub struct ArtifactKind(Cow<'static, str>); - -impl ArtifactKind { - /// Creates a new `ArtifactKind` from a string. - pub fn new(kind: String) -> Self { - Self(kind.into()) - } - - /// Creates a new `ArtifactKind` from a static string. - pub const fn from_static(kind: &'static str) -> Self { - Self(Cow::Borrowed(kind)) - } - - /// Creates a new `ArtifactKind` from a known kind. - pub fn from_known(kind: KnownArtifactKind) -> Self { - Self::new(kind.to_string()) - } - - /// Returns the kind as a string. - pub fn as_str(&self) -> &str { - &self.0 - } - - /// Converts self to a `KnownArtifactKind`, if it is known. - pub fn to_known(&self) -> Option { - self.0.parse().ok() - } -} - -/// These artifact kinds are not stored anywhere, but are derived from stored -/// kinds and used as internal identifiers. -impl ArtifactKind { - /// Gimlet root of trust bootloader slot image identifier. - /// - /// Derived from [`KnownArtifactKind::GimletRotBootloader`]. - pub const GIMLET_ROT_STAGE0: Self = - Self::from_static("gimlet_rot_bootloader"); - - /// Gimlet root of trust A slot image identifier. - /// - /// Derived from [`KnownArtifactKind::GimletRot`]. - pub const GIMLET_ROT_IMAGE_A: Self = - Self::from_static("gimlet_rot_image_a"); - - /// Gimlet root of trust B slot image identifier. - /// - /// Derived from [`KnownArtifactKind::GimletRot`]. - pub const GIMLET_ROT_IMAGE_B: Self = - Self::from_static("gimlet_rot_image_b"); - - /// PSC root of trust stage0 image identifier. - /// - /// Derived from [`KnownArtifactKind::PscRotBootloader`]. - pub const PSC_ROT_STAGE0: Self = Self::from_static("psc_rot_bootloader"); - - /// PSC root of trust A slot image identifier. - /// - /// Derived from [`KnownArtifactKind::PscRot`]. - pub const PSC_ROT_IMAGE_A: Self = Self::from_static("psc_rot_image_a"); - - /// PSC root of trust B slot image identifier. - /// - /// Derived from [`KnownArtifactKind::PscRot`]. - pub const PSC_ROT_IMAGE_B: Self = Self::from_static("psc_rot_image_b"); - - /// Switch root of trust A slot image identifier. - /// - /// Derived from [`KnownArtifactKind::SwitchRotBootloader`]. - pub const SWITCH_ROT_STAGE0: Self = - Self::from_static("switch_rot_bootloader"); - - /// Switch root of trust A slot image identifier. - /// - /// Derived from [`KnownArtifactKind::SwitchRot`]. - pub const SWITCH_ROT_IMAGE_A: Self = - Self::from_static("switch_rot_image_a"); - - /// Switch root of trust B slot image identifier. - /// - /// Derived from [`KnownArtifactKind::SwitchRot`]. - pub const SWITCH_ROT_IMAGE_B: Self = - Self::from_static("switch_rot_image_b"); - - /// Gimlet Host phase 1 identifier. - /// - /// Derived from [`KnownArtifactKind::Host`]. - pub const GIMLET_HOST_PHASE_1: Self = - Self::from_static("gimlet_host_phase_1"); - - /// Cosmo Host phase 1 identifier. - /// - /// Derived from [`KnownArtifactKind::Host`]. - pub const COSMO_HOST_PHASE_1: Self = - Self::from_static("cosmo_host_phase_1"); - - /// Host phase 2 identifier. - /// - /// Derived from [`KnownArtifactKind::Host`]. - pub const HOST_PHASE_2: Self = Self::from_static("host_phase_2"); - - /// Gimlet Trampoline phase 1 identifier. - /// - /// Derived from [`KnownArtifactKind::Trampoline`]. - pub const GIMLET_TRAMPOLINE_PHASE_1: Self = - Self::from_static("gimlet_trampoline_phase_1"); - - /// Cosmo Trampoline phase 1 identifier. - /// - /// Derived from [`KnownArtifactKind::Trampoline`]. - pub const COSMO_TRAMPOLINE_PHASE_1: Self = - Self::from_static("cosmo_trampoline_phase_1"); - - /// Trampoline phase 2 identifier. - /// - /// Derived from [`KnownArtifactKind::Trampoline`]. - pub const TRAMPOLINE_PHASE_2: Self = - Self::from_static("trampoline_phase_2"); - - /// Measurement Corpus identifier. - /// - /// Derived from [`KnownArtifactKind::MeasurementCorpus`]. - pub const MEASUREMENT_CORPUS: Self = - Self::from_static("measurement_corpus"); -} - -impl From for ArtifactKind { - fn from(kind: KnownArtifactKind) -> Self { - Self::from_known(kind) - } -} - -impl fmt::Display for ArtifactKind { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.fmt(f) - } -} - -impl FromStr for ArtifactKind { - type Err = Infallible; - - fn from_str(s: &str) -> Result { - Ok(Self::new(s.to_owned())) - } -} - -/// Kinds of update artifacts, as used by Nexus to determine what updates are available and by -/// sled-agent to determine how to apply an update when asked. -#[derive( - Clone, - Copy, - Debug, - PartialEq, - Eq, - Hash, - Ord, - PartialOrd, - Display, - EnumString, - Deserialize, - Serialize, - EnumIter, -)] -#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] -#[strum(serialize_all = "snake_case")] -#[serde(rename_all = "snake_case")] -pub enum KnownArtifactKind { - // Sled Artifacts - GimletSp, - GimletRot, - GimletRotBootloader, - Host, - Trampoline, - /// Installinator document identifier. - /// - /// While the installinator document is a metadata file similar to - /// [`ArtifactsDocument`](crate::ArtifactsDocument), Wicketd and Nexus treat - /// it as an opaque single-unit artifact to avoid backwards compatibility - /// issues. - InstallinatorDocument, - /// Composite artifact of all control plane zones - ControlPlane, - /// Individual control plane zone - Zone, - /// MeasurementCorpus - MeasurementCorpus, - - // PSC Artifacts - PscSp, - PscRot, - PscRotBootloader, - - // Switch Artifacts - SwitchSp, - SwitchRot, - SwitchRotBootloader, -} - -impl KnownArtifactKind { - /// For an RoT variant, returns A and B deployment unit kinds. - pub fn rot_a_and_b_kinds( - self, - ) -> Result<(ArtifactKind, ArtifactKind), NotRotVariantError> { - match self { - KnownArtifactKind::GimletRot => Ok(( - ArtifactKind::GIMLET_ROT_IMAGE_A, - ArtifactKind::GIMLET_ROT_IMAGE_B, - )), - KnownArtifactKind::PscRot => Ok(( - ArtifactKind::PSC_ROT_IMAGE_A, - ArtifactKind::PSC_ROT_IMAGE_B, - )), - KnownArtifactKind::SwitchRot => Ok(( - ArtifactKind::SWITCH_ROT_IMAGE_A, - ArtifactKind::SWITCH_ROT_IMAGE_B, - )), - KnownArtifactKind::GimletSp - | KnownArtifactKind::GimletRotBootloader - | KnownArtifactKind::Host - | KnownArtifactKind::Trampoline - | KnownArtifactKind::InstallinatorDocument - | KnownArtifactKind::MeasurementCorpus - | KnownArtifactKind::ControlPlane - | KnownArtifactKind::Zone - | KnownArtifactKind::PscSp - | KnownArtifactKind::PscRotBootloader - | KnownArtifactKind::SwitchSp - | KnownArtifactKind::SwitchRotBootloader => { - Err(NotRotVariantError(self)) - } - } - } - - /// Returns an iterator over all the variants in this struct. - /// - /// This is provided as a helper so dependent packages don't have to pull in - /// strum explicitly. - pub fn iter() -> KnownArtifactKindIter { - ::iter() - } - - /// For fake artifacts we generate for tests, what `SIGN` value do we insert - /// in the Hubris caboose for this artifact kind? - pub fn fake_artifact_hubris_sign(&self) -> Option { - match self { - // Only RoT and RoT bootloader artifacts are signed. We want to use - // a distinct sign value for kind of system, just like real systems - // have. - KnownArtifactKind::GimletRot - | KnownArtifactKind::GimletRotBootloader => { - Some("sign-gimlet".to_string()) - } - KnownArtifactKind::SwitchRot - | KnownArtifactKind::SwitchRotBootloader => { - Some("sign-switch".to_string()) - } - KnownArtifactKind::PscRot | KnownArtifactKind::PscRotBootloader => { - Some("sign-psc".to_string()) - } - - KnownArtifactKind::GimletSp - | KnownArtifactKind::Host - | KnownArtifactKind::Trampoline - | KnownArtifactKind::InstallinatorDocument - | KnownArtifactKind::MeasurementCorpus - | KnownArtifactKind::ControlPlane - | KnownArtifactKind::Zone - | KnownArtifactKind::PscSp - | KnownArtifactKind::SwitchSp => None, - } - } -} - -#[derive(Debug, Error)] -#[error("expected an RoT variant, found {0:?}")] -pub struct NotRotVariantError(KnownArtifactKind); - -#[cfg(test)] -mod tests { - use super::{ArtifactKind, KnownArtifactKind}; - - #[test] - fn serde_artifact_kind() { - assert_eq!( - serde_json::from_str::("\"gimlet_sp\"") - .unwrap() - .to_known(), - Some(KnownArtifactKind::GimletSp) - ); - assert_eq!( - serde_json::from_str::("\"fhqwhgads\"") - .unwrap() - .to_known(), - None, - ); - assert!(serde_json::from_str::("null").is_err()); - - assert_eq!( - serde_json::to_string(&ArtifactKind::from_known( - KnownArtifactKind::GimletSp - )) - .unwrap(), - "\"gimlet_sp\"" - ); - assert_eq!( - serde_json::to_string(&ArtifactKind::new("fhqwhgads".to_string())) - .unwrap(), - "\"fhqwhgads\"" - ); - } - - #[test] - fn known_artifact_kind_roundtrip() { - for kind in KnownArtifactKind::iter() { - let as_string = kind.to_string(); - let kind2 = as_string.parse::().unwrap_or_else( - |error| panic!("error parsing kind {as_string}: {error}"), - ); - assert_eq!(kind, kind2); - } - } - - #[test] - fn display_respects_padding() { - let kind = ArtifactKind::from_static("foo"); - assert_eq!(format!("{kind:x>10}"), "xxxxxxxfoo"); - - let kind = KnownArtifactKind::Host; - assert_eq!(format!("{kind:x>10}"), "xxxxxxhost"); - } -} diff --git a/artifact/src/lib.rs b/artifact/src/lib.rs index 78e7351..fe2f4d1 100644 --- a/artifact/src/lib.rs +++ b/artifact/src/lib.rs @@ -3,11 +3,14 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. mod artifact; +mod hash; +pub mod hubris; mod installinator; -mod kind; +mod tags; mod version; pub use artifact::*; +pub use hash::*; pub use installinator::*; -pub use kind::*; +pub use tags::*; pub use version::*; diff --git a/artifact/src/tags.rs b/artifact/src/tags.rs new file mode 100644 index 0000000..06f6d51 --- /dev/null +++ b/artifact/src/tags.rs @@ -0,0 +1,204 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::BTreeMap; + +use serde::Deserialize; +use serde::Serialize; +use serde::de::value::MapDeserializer; + +use crate::InstallinatorArtifactKind; + +/// Sets of artifact tags known to the control plane. +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, +)] +#[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] +#[serde(tag = "kind", rename_all = "kebab-case")] +pub enum KnownArtifactTags { + /// JSON document describing the artifacts installinator is responsible for + /// writing during mupdate and sled recovery. + InstallinatorDocument {}, + + /// CORIM manifest for remote attestation. + MeasurementCorpus {}, + + /// Phase 1 OS image, written to flash. Differs based on the target board. + OsPhase1 { variant: OsVariant, board: OsBoard }, + + /// Phase 2 OS image, a ZFS pool with an Oxide-specific header written to + /// M.2 storage. Common across all target boards. + OsPhase2 { variant: OsVariant }, + + /// Hubris archive for a Root of Trust image. + Rot { + /// The `BORD` field in the caboose (such as `oxide-rot-1`). + board: String, + /// The `SIGN` field in the caboose. This is the Root Key Table Hash + /// (RKTH). + /// + /// For unsigned images this will not be present; this will generally + /// never occur in release repos but can be useful on hardware that has + /// not fully made it through manufacturing yet. + #[serde(skip_serializing_if = "Option::is_none")] + sign: Option, + /// ROT images are compiled for two different locations in flash; this + /// identifies which slot this image belongs to. + slot: RotSlot, + }, + + /// Hubris archive for a Root of Trust bootloader. + RotBootloader { + /// The `BORD` field in the caboose (such as `oxide-rot-1`). + board: String, + /// The `SIGN` field in the caboose. This is the Root Key Table Hash + /// (RKTH). + /// + /// For unsigned images this will not be present; this will generally + /// never occur in release repos but can be useful on hardware that has + /// not fully made it through manufacturing yet. + #[serde(skip_serializing_if = "Option::is_none")] + sign: Option, + }, + + /// Hubris archive for a Service Processor image. + Sp { + /// The `BORD` field in the caboose (such as `gimlet-d` or `cosmo-b`). + board: String, + }, + + /// Tarball of a Helios zone image. + Zone { + /// The zone name, as self-identified in the tarball's `oxide.json` + /// file. This may differ from the filename on disk. + name: String, + }, +} + +impl KnownArtifactTags { + pub fn from_tags( + tags: &BTreeMap, + ) -> Result { + Self::deserialize(MapDeserializer::new( + tags.iter().map(|(k, v)| (k.as_str(), v.as_str())), + )) + } + + pub fn to_tags(&self) -> BTreeMap { + self.to_tags_impl().unwrap_or_default() + } + + fn to_tags_impl(&self) -> Option> { + let value = serde_json::to_value(self).ok()?; + let serde_json::Value::Object(map) = value else { return None }; + map.into_iter() + .map(|(k, v)| match v { + serde_json::Value::String(v) => Some((k, v)), + _ => None, + }) + .collect() + } + + pub fn to_installinator(&self) -> Option { + match self { + KnownArtifactTags::MeasurementCorpus {} => { + Some(InstallinatorArtifactKind::MeasurementCorpus) + } + KnownArtifactTags::OsPhase2 { variant: OsVariant::Host } => { + Some(InstallinatorArtifactKind::HostPhase2) + } + KnownArtifactTags::Zone { name } => { + Some(InstallinatorArtifactKind::Zone { name: name.clone() }) + } + _ => None, + } + } +} + +macro_rules! display_serialize { + ($ty:ty) => { + impl std::fmt::Display for $ty { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.serialize(f) + } + } + }; +} + +/// OS variant artifact tag (host or recovery). +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Deserialize, + Serialize, +)] +#[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] +#[serde(rename_all = "kebab-case")] +pub enum OsVariant { + Host, + Recovery, +} +display_serialize!(OsVariant); + +/// OS board artifact tag (gimlet or cosmo). +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Deserialize, + Serialize, +)] +#[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] +#[serde(rename_all = "kebab-case")] +pub enum OsBoard { + Gimlet, + Cosmo, +} +display_serialize!(OsBoard); + +/// ROT slot artifact tag (A or B). +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Deserialize, + Serialize, +)] +#[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] +pub enum RotSlot { + A, + B, +} +display_serialize!(RotSlot); + +#[cfg(test)] +mod tests { + use test_strategy::proptest; + + use crate::KnownArtifactTags; + + #[proptest] + fn tags_roundtrip(tags: KnownArtifactTags) { + let tag_map = tags + .to_tags_impl() + .expect("serialized value trivially converts to tag map"); + assert_eq!(KnownArtifactTags::from_tags(&tag_map).unwrap(), tags); + } +} diff --git a/artifact/src/version.rs b/artifact/src/version.rs index 5478c7c..fc88499 100644 --- a/artifact/src/version.rs +++ b/artifact/src/version.rs @@ -2,10 +2,13 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use std::{borrow::Cow, fmt, str::FromStr}; +use std::borrow::Cow; +use std::fmt; +use std::str::FromStr; use daft::Diffable; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; +use serde::Serialize; use thiserror::Error; /// An artifact version. diff --git a/bin/Cargo.toml b/bin/Cargo.toml index 69c6c03..6dba6f8 100644 --- a/bin/Cargo.toml +++ b/bin/Cargo.toml @@ -1,38 +1,29 @@ [package] -name = "tufaceous" +name = "tufaceous-bin" version.workspace = true edition.workspace = true license.workspace = true publish.workspace = true -[[test]] -name = "manifest-tests" -harness = false +[[bin]] +name = "tufaceous" +path = "src/main.rs" [dependencies] -anyhow = { workspace = true, features = ["backtrace"] } +anyhow.workspace = true camino.workspace = true chrono.workspace = true clap.workspace = true -console.workspace = true -humantime.workspace = true +fs-err = { workspace = true, features = ["tokio"] } semver.workspace = true slog.workspace = true slog-async.workspace = true slog-envlogger.workspace = true slog-term.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +tough.workspace = true +tufaceous.workspace = true tufaceous-artifact.workspace = true -tufaceous-lib.workspace = true - -[dev-dependencies] -assert_cmd.workspace = true -datatest-stable.workspace = true -dropshot.workspace = true -fs-err.workspace = true -predicates.workspace = true -tempfile.workspace = true -tokio = { workspace = true, features = ["test-util"] } [lints] workspace = true diff --git a/bin/README.adoc b/bin/README.adoc deleted file mode 100644 index 86e4cfc..0000000 --- a/bin/README.adoc +++ /dev/null @@ -1,36 +0,0 @@ -# tufaceous - -Rack update repository generation tool. - -## TUF, keys and lifetime - -Rack update repositories use TUF. Consider reading https://theupdateframework.io/overview/[the TUF overview] and https://theupdateframework.io/metadata/[a high level summary of the metadata mandated by the specification]. - -The only keys currently supported by tufaceous are Ed25519 keys. Support for hardware-backed keys is planned. - -Each role has an expiration date. The default is one week, suitable for development testing. This can be modified with the `--expiry` option. - -## init - -Create a new repository in the current directory with `tufaceous init`. - -To change the target directory, use `-r/--repo`. This is accepted on all subcommands, and needs to come before the subcommand because Clap is picky. - -This will generate a new Ed25519 private key and display it on stderr if no keys are provided. - -Currently if keys are provided, they are allowed to sign all roles. For the time being if you need more advanced editing of the root role, use https://crates.io/crates/tuftool[tuftool]'s `root` subcommands. - -## add zones - -Usage: - ----- -tuftool [-r PATH/TO/REPO] add-zone [--name NAME] ZONE_TAR_GZ VERSION ----- - -Example: - ----- -$ tuftool add-zone out/nexus.tar.gz 0.0.0 -added zone nexus, version 0.0.0 ----- diff --git a/bin/invalid-manifests/duplicate-artifact-2.toml b/bin/invalid-manifests/duplicate-artifact-2.toml deleted file mode 100644 index d8f5a59..0000000 --- a/bin/invalid-manifests/duplicate-artifact-2.toml +++ /dev/null @@ -1,24 +0,0 @@ -# This manifest contains artifacts with the same kind and hash, but different -# artifact versions. - -system_version = "1.0.0" - -[[artifact.gimlet_sp]] -name = "fake-gimlet-sp" -version = "1.0.0" -source = { kind = "fake", size = "1MiB", data_version = "0.0.0" } - -[[artifact.switch_sp]] -name = "fake-switch-sp" -version = "1.0.0" -source = { kind = "fake", size = "1MiB", data_version = "0.0.0" } - -[[artifact.gimlet_sp]] -name = "fake-gimlet-sp" -version = "2.0.0" -source = { kind = "fake", size = "1MiB", data_version = "0.0.0" } - -[[artifact.switch_sp]] -name = "fake-switch-sp" -version = "2.0.0" -source = { kind = "fake", size = "1MiB", data_version = "0.0.0" } diff --git a/bin/invalid-manifests/duplicate-artifact.toml b/bin/invalid-manifests/duplicate-artifact.toml deleted file mode 100644 index 298ee87..0000000 --- a/bin/invalid-manifests/duplicate-artifact.toml +++ /dev/null @@ -1,14 +0,0 @@ -# This manifest contains two artifacts with the same name and version, which -# will hash to the same value. - -system_version = "1.0.0" - -[[artifact.gimlet_sp]] -name = "fake-gimlet-sp" -version = "1.0.0" -source = { kind = "fake", size = "1MiB" } - -[[artifact.gimlet_sp]] -name = "fake-gimlet-sp" -version = "1.0.0" -source = { kind = "fake", size = "1MiB" } diff --git a/bin/invalid-manifests/duplicate-zone-file-name.toml b/bin/invalid-manifests/duplicate-zone-file-name.toml deleted file mode 100644 index 0ccf489..0000000 --- a/bin/invalid-manifests/duplicate-zone-file-name.toml +++ /dev/null @@ -1,14 +0,0 @@ -# This manifest has two control plane zones that hash to the same contents. This -# is not allowed. - -system_version = "1.0.0" - -[[artifact.control_plane]] -name = "fake-control-plane" -version = "1.0.0" -[artifact.control_plane.source] -kind = "composite-control-plane" -zones = [ - { kind = "fake", artifact_name = "zone-1", file_name = "zone1.tar.gz", size = "1MiB" }, - { kind = "fake", artifact_name = "zone-1", file_name = "zone1-dup.tar.gz", size = "1MiB" }, -] diff --git a/bin/manifests/fake-non-semver.toml b/bin/manifests/fake-non-semver.toml deleted file mode 100644 index 1ef2508..0000000 --- a/bin/manifests/fake-non-semver.toml +++ /dev/null @@ -1,94 +0,0 @@ -# This is an artifact manifest that generates fake entries for all components. -# Some of the components have non-semver artifact versions. -# -# This is completely non-functional and is only useful for testing archive -# extraction in other parts of the repository. - -system_version = "2.0.0" - -[[artifact.gimlet_sp]] -name = "fake-gimlet-sp" -version = "2.0.0" -source = { kind = "fake", size = "1MiB" } - -[[artifact.gimlet_rot]] -name = "fake-gimlet-rot" -version = "2.0.0" -[artifact.gimlet_rot.source] -kind = "composite-rot" -archive_a = { kind = "fake", size = "512KiB" } -archive_b = { kind = "fake", size = "512KiB" } - -[[artifact.host]] -name = "fake-host" -version = "2.0.0" -[artifact.host.source] -kind = "composite-host" -gimlet_phase_1 = { kind = "fake", size = "512KiB" } -cosmo_phase_1 = { kind = "fake", size = "512KiB" } -phase_2 = { kind = "fake", size = "1MiB" } - -[[artifact.trampoline]] -name = "fake-trampoline" -version = "non-semver" -[artifact.trampoline.source] -kind = "composite-host" -gimlet_phase_1 = { kind = "fake", size = "512KiB" } -cosmo_phase_1 = { kind = "fake", size = "512KiB" } -phase_2 = { kind = "fake", size = "1MiB" } - -[[artifact.control_plane]] -name = "fake-control-plane" -version = "2.0.0" -[artifact.control_plane.source] -kind = "composite-control-plane" -zones = [ - { kind = "fake", artifact_name = "zone-1", file_name = "zone1.tar.gz", size = "1MiB" }, - { kind = "fake", artifact_name = "zone-2", file_name = "zone2.tar.gz", size = "1MiB" }, -] - -[[artifact.psc_sp]] -name = "fake-psc-sp" -version = "2.0.0" -source = { kind = "fake", size = "1MiB" } - -[[artifact.psc_rot]] -name = "fake-psc-rot" -version = "2.0.0" -[artifact.psc_rot.source] -kind = "composite-rot" -archive_a = { kind = "fake", size = "512KiB" } -archive_b = { kind = "fake", size = "512KiB" } - -[[artifact.switch_sp]] -name = "fake-switch-sp" -version = "2.0.0" -source = { kind = "fake", size = "1MiB" } - -[[artifact.switch_rot]] -name = "fake-switch-rot" -version = "2.0.0" -[artifact.switch_rot.source] -kind = "composite-rot" -archive_a = { kind = "fake", size = "512KiB" } -archive_b = { kind = "fake", size = "512KiB" } - -[[artifact.gimlet_rot_bootloader]] -name = "fake-gimlet-rot-bootloader" -version = "2.0.0" -source = { kind = "fake", size = "1MiB" } - -[[artifact.psc_rot_bootloader]] -name = "fake-psc-rot-bootloader" -version = "2.0.0" -source = { kind = "fake", size = "1MiB" } - -[[artifact.switch_rot_bootloader]] -name = "fake-switch-rot-bootloader" -version = "non-semver-2" -source = { kind = "fake", size = "1MiB" } - -[[artifact.measurement_corpus]] -name = "fake-corpus" -version = "1.0.0" -source = { kind = "fake", size ="1MiB" } diff --git a/bin/manifests/fake.toml b/bin/manifests/fake.toml deleted file mode 100644 index ae5e3f3..0000000 --- a/bin/manifests/fake.toml +++ /dev/null @@ -1,93 +0,0 @@ -# This is an artifact manifest that generates fake entries for all components. -# This is completely non-functional and is only useful for testing archive -# extraction in other parts of the repository. - -system_version = "1.0.0" - -[[artifact.gimlet_sp]] -name = "fake-gimlet-sp" -version = "1.0.0" -source = { kind = "fake", size = "1MiB" } - -[[artifact.gimlet_rot]] -name = "fake-gimlet-rot" -version = "1.0.0" -[artifact.gimlet_rot.source] -kind = "composite-rot" -archive_a = { kind = "fake", size = "512KiB" } -archive_b = { kind = "fake", size = "512KiB" } - -[[artifact.host]] -name = "fake-host" -version = "1.0.0" -[artifact.host.source] -kind = "composite-host" -gimlet_phase_1 = { kind = "fake", size = "512KiB" } -cosmo_phase_1 = { kind = "fake", size = "512KiB" } -phase_2 = { kind = "fake", size = "1MiB" } - -[[artifact.trampoline]] -name = "fake-trampoline" -version = "1.0.0" -[artifact.trampoline.source] -kind = "composite-host" -gimlet_phase_1 = { kind = "fake", size = "512KiB" } -cosmo_phase_1 = { kind = "fake", size = "512KiB" } -phase_2 = { kind = "fake", size = "1MiB" } - -[[artifact.control_plane]] -name = "fake-control-plane" -version = "1.0.0" -[artifact.control_plane.source] -kind = "composite-control-plane" -zones = [ - { kind = "fake", artifact_name = "zone-1", file_name = "zone1.tar.gz", size = "1MiB" }, - { kind = "fake", artifact_name = "zone-2", file_name = "zone2.tar.gz", size = "1MiB" }, -] - -[[artifact.psc_sp]] -name = "fake-psc-sp" -version = "1.0.0" -source = { kind = "fake", size = "1MiB" } - -[[artifact.psc_rot]] -name = "fake-psc-rot" -version = "1.0.0" -[artifact.psc_rot.source] -kind = "composite-rot" -archive_a = { kind = "fake", size = "512KiB" } -archive_b = { kind = "fake", size = "512KiB" } - -[[artifact.switch_sp]] -name = "fake-switch-sp" -version = "1.0.0" -source = { kind = "fake", size = "1MiB" } - -[[artifact.switch_rot]] -name = "fake-switch-rot" -version = "1.0.0" -[artifact.switch_rot.source] -kind = "composite-rot" -archive_a = { kind = "fake", size = "512KiB" } -archive_b = { kind = "fake", size = "512KiB" } - -[[artifact.gimlet_rot_bootloader]] -name = "fake-gimlet-rot-bootloader" -version = "1.0.0" -source = { kind = "fake", size = "1MiB" } - -[[artifact.psc_rot_bootloader]] -name = "fake-psc-rot-bootloader" -version = "1.0.0" -source = { kind = "fake", size = "1MiB" } - -[[artifact.switch_rot_bootloader]] -name = "fake-switch-rot-bootloader" -version = "1.0.0" -source = { kind = "fake", size = "1MiB" } - -[[artifact.measurement_corpus]] -name = "fake-corpus" -version = "1.0.0" -source = { kind = "fake", size ="1MiB" } - diff --git a/bin/src/assemble.rs b/bin/src/assemble.rs new file mode 100644 index 0000000..ae96889 --- /dev/null +++ b/bin/src/assemble.rs @@ -0,0 +1,105 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use anyhow::Context; +use anyhow::Result; +use camino::Utf8Path; +use camino::Utf8PathBuf; +use chrono::Utc; +use clap::Parser; +use semver::Version; +use tufaceous::edit::RepositoryEditor; +use tufaceous_artifact::ArtifactVersion; +use tufaceous_artifact::KnownArtifactTags; +use tufaceous_artifact::OsVariant; +use tufaceous_artifact::RotSlot; + +use crate::sign::SignOptions; + +#[derive(Debug, Parser)] +pub struct Args { + #[clap(long)] + version: Version, + #[clap(long)] + no_installinator_document: bool, + #[clap(long)] + measurement_corpus: Vec, + #[clap(long)] + host_os_dir: Option, + #[clap(long)] + recovery_os_dir: Option, + #[clap(long)] + rot_slot_a: Vec, + #[clap(long)] + rot_slot_b: Vec, + #[clap(long)] + rot_bootloader: Vec, + #[clap(long)] + sp: Vec, + #[clap(long)] + zone: Vec, + + #[clap(flatten)] + sign_options: SignOptions, + output: Utf8PathBuf, +} + +impl Args { + pub async fn run(self) -> Result<()> { + let version = ArtifactVersion::new(self.version.to_string())?; + let mut editor = RepositoryEditor::new(self.version); + if !self.no_installinator_document { + editor = editor.generate_installinator_document(); + } + for path in self.measurement_corpus { + editor = editor + .add_artifact( + file_name(&path)?, + version.clone(), + KnownArtifactTags::MeasurementCorpus {}, + path, + ) + .await?; + } + if let Some(path) = self.host_os_dir { + editor = editor.add_os_artifacts(OsVariant::Host, path).await?; + } + if let Some(path) = self.recovery_os_dir { + editor = editor.add_os_artifacts(OsVariant::Recovery, path).await?; + } + for (slot, paths) in + [(RotSlot::A, self.rot_slot_a), (RotSlot::B, self.rot_slot_b)] + { + for path in paths { + editor = + editor.add_rot_image(file_name(&path)?, slot, path).await?; + } + } + for path in self.rot_bootloader { + editor = editor + .add_rot_bootloader_image(file_name(&path)?, path) + .await?; + } + for path in self.sp { + editor = editor.add_sp_image(file_name(&path)?, path).await?; + } + for path in self.zone { + editor = editor.add_zone_image(&file_name(&path)?, path).await?; + } + + let unsigned = editor.finish().await?; + self.sign_options + .sign(unsigned) + .await? + .write_zip_file(self.output, Utc::now()) + .await?; + Ok(()) + } +} + +fn file_name(path: &Utf8Path) -> Result { + path.file_name() + .map(str::to_owned) + .with_context(|| format!("{path} has no file name")) +} diff --git a/bin/src/date.rs b/bin/src/date.rs deleted file mode 100644 index 52d6f4c..0000000 --- a/bin/src/date.rs +++ /dev/null @@ -1,21 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use anyhow::Result; -use chrono::{DateTime, Duration, Timelike, Utc}; - -/// Parser for datelike command line arguments. Can accept a duration (e.g. -/// "1w") or an ISO8601 timestamp. -pub(crate) fn parse_duration_or_datetime(s: &str) -> Result> { - match humantime::parse_duration(s) { - Ok(duration) => { - // Remove nanoseconds from the timestamp to keep it less - // overwhelming. `Timelike::with_nanosecond` returns None only when - // passed a value over 2 billion - let now = Utc::now().with_nanosecond(0).unwrap(); - Ok(now + Duration::from_std(duration)?) - } - Err(_) => Ok(s.parse()?), - } -} diff --git a/bin/src/dispatch.rs b/bin/src/dispatch.rs deleted file mode 100644 index a9b6274..0000000 --- a/bin/src/dispatch.rs +++ /dev/null @@ -1,200 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use anyhow::{Context, Result, bail}; -use camino::Utf8PathBuf; -use chrono::{DateTime, Utc}; -use clap::Parser; -use tufaceous_artifact::{ArtifactsDocument, KnownArtifactKind}; -use tufaceous_lib::assemble::{ArtifactManifest, OmicronRepoAssembler}; -use tufaceous_lib::{ArchiveExtractor, Key, OmicronRepo}; - -#[derive(Debug, Parser)] -pub struct Args { - #[clap(subcommand)] - command: Command, - - #[clap( - short = 'k', - long = "key", - env = "TUFACEOUS_KEY", - required = false, - global = true - )] - keys: Vec, - - #[clap(long, value_parser = crate::date::parse_duration_or_datetime, default_value = "7d", global = true)] - expiry: DateTime, - - /// TUF repository path (default: current working directory) - #[clap(short = 'r', long, global = true)] - repo: Option, -} - -impl Args { - /// Executes these arguments. - pub async fn exec(self, log: &slog::Logger) -> Result<()> { - match self.command { - Command::Assemble { - manifest_path, - output_path, - build_dir, - no_generate_key, - skip_all_present, - allow_non_semver, - no_installinator_document, - } => { - // The filename must end with "zip". - if output_path.extension() != Some("zip") { - bail!("output path `{output_path}` must end with .zip"); - } - - let manifest = ArtifactManifest::from_path(&manifest_path) - .context("error reading manifest")?; - if !allow_non_semver { - manifest.verify_all_semver()?; - } - if !skip_all_present { - manifest.verify_all_present()?; - } - - let keys = maybe_generate_keys(self.keys, no_generate_key)?; - let mut assembler = OmicronRepoAssembler::new( - log, - manifest, - keys, - self.expiry, - !no_installinator_document, - output_path, - ); - if let Some(dir) = build_dir { - assembler.set_build_dir(dir); - } - - assembler.build().await?; - - Ok(()) - } - Command::Extract { - archive_file, - dest, - no_installinator_document, - } => { - let mut extractor = ArchiveExtractor::from_path(&archive_file)?; - extractor.extract(&dest)?; - - // Now load the repository and ensure it's valid. - let repo = OmicronRepo::load_untrusted(log, &dest) - .await - .with_context(|| { - format!( - "error loading extracted repository at `{dest}` \ - (extracted files are still available)" - ) - })?; - let artifacts = - repo.read_artifacts().await.with_context(|| { - format!( - "error loading {} from extracted archive \ - at `{dest}`", - ArtifactsDocument::FILE_NAME - ) - })?; - if !no_installinator_document { - // There should be a reference to an installinator document - // within artifacts_document. - let installinator_doc_artifact = artifacts - .artifacts - .iter() - .find(|artifact| { - artifact.kind.to_known() - == Some( - KnownArtifactKind::InstallinatorDocument, - ) - }) - .context( - "could not find artifact with kind \ - `installinator_document` within artifacts.json", - )?; - - repo.read_installinator_document( - &installinator_doc_artifact.target, - ) - .await - .with_context(|| { - format!( - "error loading {} from extracted archive \ - at `{dest}`", - installinator_doc_artifact.target, - ) - })?; - } - - Ok(()) - } - } - } -} - -#[derive(Debug, Parser)] -enum Command { - /// Assembles a repository from a provided manifest. - Assemble { - /// Path to artifact manifest. - manifest_path: Utf8PathBuf, - - /// The path to write the archive to (must end with .zip). - output_path: Utf8PathBuf, - - /// Directory to use for building artifacts [default: temporary directory] - #[clap(long)] - build_dir: Option, - - /// Disable random key generation and exit if no keys are provided - #[clap(long)] - no_generate_key: bool, - - /// Skip checking to ensure all expected artifacts are present. - #[clap(long)] - skip_all_present: bool, - - /// Allow versions to be non-semver. - /// - /// Transitional option for v13 -> v14. After v14, versions will be - /// allowed to be non-semver by default. - #[clap(long)] - allow_non_semver: bool, - - /// Do not include the installinator document. - /// - /// Transitional option for v15 -> v16, meant to be used for testing. - #[clap(long)] - no_installinator_document: bool, - }, - /// Validates and extracts a repository created by the `assemble` command. - Extract { - /// The file to extract. - archive_file: Utf8PathBuf, - - /// The destination to extract the file to. - dest: Utf8PathBuf, - - /// Indicate that the file does not contain an installinator document. - #[clap(long)] - no_installinator_document: bool, - }, -} - -fn maybe_generate_keys( - keys: Vec, - no_generate_key: bool, -) -> Result> { - Ok(if !no_generate_key && keys.is_empty() { - let key = Key::generate_ed25519()?; - crate::hint::generated_key(&key); - vec![key] - } else { - keys - }) -} diff --git a/bin/src/hint.rs b/bin/src/hint.rs deleted file mode 100644 index 74385e3..0000000 --- a/bin/src/hint.rs +++ /dev/null @@ -1,29 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use tufaceous_lib::Key; - -fn print_hint(hint: &str) { - for line in hint.trim().lines() { - eprintln!("{}", console::style(format!("hint: {}", line)).yellow()); - } -} - -pub(crate) fn generated_key(key: &Key) { - print_hint(&format!( - r#" -Generated a random key: - - {key} - -To modify this repository, you will need this key. Use the -k/--key -command line flag or the TUFACEOUS_KEY environment variable: - - export TUFACEOUS_KEY={key} - -To prevent this default behavior, use --no-generate-key. - "#, - key = console::style(key).italic() - )) -} diff --git a/bin/src/lib.rs b/bin/src/lib.rs deleted file mode 100644 index 65ff581..0000000 --- a/bin/src/lib.rs +++ /dev/null @@ -1,9 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -mod date; -mod dispatch; -mod hint; - -pub use dispatch::*; diff --git a/bin/src/main.rs b/bin/src/main.rs index 014817e..78a81d1 100644 --- a/bin/src/main.rs +++ b/bin/src/main.rs @@ -2,22 +2,29 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +mod assemble; +mod sign; + use anyhow::Result; use clap::Parser; use slog::Drain; -use tufaceous::Args; +use slog::Logger; -#[tokio::main] -async fn main() -> Result<()> { - let log = setup_log(); - let args = Args::parse(); - args.exec(&log).await +#[derive(Debug, Parser)] +enum Command { + Assemble(assemble::Args), } -fn setup_log() -> slog::Logger { +#[tokio::main] +async fn main() -> Result<()> { let stderr_drain = stderr_env_drain("RUST_LOG"); let drain = slog_async::Async::new(stderr_drain).build().fuse(); - slog::Logger::root(drain, slog::o!()) + let _log = Logger::root(drain, slog::o!()); + + let args = Command::parse(); + match args { + Command::Assemble(args) => args.run().await, + } } fn stderr_env_drain(env_var: &str) -> impl Drain { diff --git a/bin/src/sign.rs b/bin/src/sign.rs new file mode 100644 index 0000000..b8739ef --- /dev/null +++ b/bin/src/sign.rs @@ -0,0 +1,87 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use anyhow::Result; +use camino::Utf8PathBuf; +use clap::Parser; +use fs_err::tokio as fs; +use tough::key_source::LocalKeySource; +use tufaceous::edit::SignedRepository; +use tufaceous::edit::UnsignedRepository; + +#[derive(Debug, Parser)] +#[cfg_attr(test, derive(PartialEq))] +pub struct SignOptions { + #[clap(long)] + no_generate_root: bool, + #[clap(long, required_if_eq("no_generate_root", "true"))] + root: Option, + #[clap(long)] + key: Vec, +} + +impl SignOptions { + pub async fn sign( + self, + mut unsigned: UnsignedRepository<'_>, + ) -> Result> { + if !self.no_generate_root { + unsigned = unsigned.generate_root(); + } + if let Some(path) = self.root { + let root = fs::read(path).await?; + unsigned = unsigned.root(root); + } + for path in self.key { + let source = LocalKeySource { path: path.into() }; + unsigned = unsigned.key(source); + } + Ok(unsigned.sign().await?) + } +} + +#[cfg(test)] +mod tests { + use clap::Parser; + + use crate::sign::SignOptions; + + #[test] + fn test_flags() { + assert_eq!( + SignOptions::try_parse_from([""]).unwrap(), + SignOptions { no_generate_root: false, root: None, key: vec![] } + ); + assert!( + SignOptions::try_parse_from(["", "--no-generate-root"]).is_err() + ); + let mut args = vec!["", "--root", "root.json"]; + assert_eq!( + SignOptions::try_parse_from(&args).unwrap(), + SignOptions { + no_generate_root: false, + root: Some("root.json".into()), + key: vec![], + } + ); + args.extend(["--key", "key.pem"]); + assert_eq!( + SignOptions::try_parse_from(&args).unwrap(), + SignOptions { + no_generate_root: false, + root: Some("root.json".into()), + key: vec!["key.pem".into()], + } + ); + args.extend(["--key", "key2.pem"]); + assert_eq!( + SignOptions::try_parse_from(&args).unwrap(), + SignOptions { + no_generate_root: false, + root: Some("root.json".into()), + key: vec!["key.pem".into(), "key2.pem".into()], + } + ); + } +} diff --git a/bin/tests/integration-tests/command_tests.rs b/bin/tests/integration-tests/command_tests.rs deleted file mode 100644 index 5fac6e3..0000000 --- a/bin/tests/integration-tests/command_tests.rs +++ /dev/null @@ -1,199 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use anyhow::Result; -use assert_cmd::Command; -use dropshot::test_util::LogContext; -use dropshot::{ConfigLogging, ConfigLoggingIfExists, ConfigLoggingLevel}; -use predicates::prelude::*; -use tufaceous_lib::Key; - -#[test] -fn test_assemble_fake() -> Result<()> { - let log_config = ConfigLogging::File { - level: ConfigLoggingLevel::Trace, - path: "UNUSED".into(), - if_exists: ConfigLoggingIfExists::Fail, - }; - let logctx = LogContext::new("test_assemble_fake", &log_config); - let tempdir = tempfile::tempdir().unwrap(); - let key = Key::generate_ed25519()?; - - let archive_path = tempdir.path().join("archive.zip"); - - let mut cmd = make_cmd(&key); - cmd.args(["assemble", "manifests/fake.toml"]); - cmd.arg(&archive_path); - cmd.assert().success(); - - // Extract the archive to a new directory. - let dest_path = tempdir.path().join("dest"); - let mut cmd = make_cmd(&key); - cmd.arg("extract"); - cmd.arg(&archive_path); - cmd.arg(&dest_path); - - cmd.assert().success(); - - logctx.cleanup_successful(); - Ok(()) -} - -#[test] -fn test_assemble_fake_non_semver() -> Result<()> { - let log_config = ConfigLogging::File { - level: ConfigLoggingLevel::Trace, - path: "UNUSED".into(), - if_exists: ConfigLoggingIfExists::Fail, - }; - let logctx = LogContext::new("test_assemble_fake_non_semver", &log_config); - let tempdir = tempfile::tempdir().unwrap(); - let key = Key::generate_ed25519()?; - - let archive_path = tempdir.path().join("archive.zip"); - - let mut cmd = make_cmd(&key); - cmd.args(["assemble", "manifests/fake-non-semver.toml"]); - cmd.arg(&archive_path); - cmd.assert().failure().stderr(predicate::str::contains( - "non-semver versions found: fake-trampoline (non-semver), \ - fake-switch-rot-bootloader (non-semver-2)", - )); - - let mut cmd = make_cmd(&key); - cmd.args([ - "assemble", - "manifests/fake-non-semver.toml", - "--allow-non-semver", - ]); - cmd.arg(&archive_path); - cmd.assert().success(); - - // Extract the archive to a new directory. - let dest_path = tempdir.path().join("dest"); - let mut cmd = make_cmd(&key); - cmd.arg("extract"); - cmd.arg(&archive_path); - cmd.arg(&dest_path); - - cmd.assert().success(); - - logctx.cleanup_successful(); - Ok(()) -} - -#[test] -fn test_assemble_duplicate_zone() -> Result<()> { - let log_config = ConfigLogging::File { - level: ConfigLoggingLevel::Trace, - path: "UNUSED".into(), - if_exists: ConfigLoggingIfExists::Fail, - }; - let logctx = LogContext::new("test_assemble_duplicate_zone", &log_config); - let tempdir = tempfile::tempdir().unwrap(); - let key = Key::generate_ed25519()?; - - let archive_path = tempdir.path().join("archive.zip"); - - let mut cmd = make_cmd(&key); - cmd.args([ - "assemble", - "--skip-all-present", - // TODO: should we also check duplicate zone artifact names? - "invalid-manifests/duplicate-zone-file-name.toml", - ]); - cmd.arg(&archive_path); - cmd.assert() - .failure() - .stderr(predicate::str::contains( - r#"a deployment unit with the same kind and hash already exists in this control_plane artifact:"#, - )) - .stderr(predicate::str::contains("zone/")) - .stderr(predicate::str::contains( - r#"(existing name: zone1.tar.gz, version: 1.0.0; new name: zone1-dup.tar.gz, version: 1.0.0)"#, - )); - - logctx.cleanup_successful(); - Ok(()) -} - -#[test] -fn test_assemble_duplicate_artifact() -> Result<()> { - let log_config = ConfigLogging::File { - level: ConfigLoggingLevel::Trace, - path: "UNUSED".into(), - if_exists: ConfigLoggingIfExists::Fail, - }; - let logctx = - LogContext::new("test_assemble_duplicate_artifact", &log_config); - let tempdir = tempfile::tempdir().unwrap(); - let key = Key::generate_ed25519()?; - - let archive_path = tempdir.path().join("archive.zip"); - - let mut cmd = make_cmd(&key); - cmd.args([ - "assemble", - "--skip-all-present", - "invalid-manifests/duplicate-artifact.toml", - ]); - cmd.arg(&archive_path); - cmd.assert().failure().stderr(predicate::str::contains( - "a target named gimlet_sp-fake-gimlet-sp-1.0.0.tar.gz \ - already exists in the repository", - )); - - logctx.cleanup_successful(); - Ok(()) -} - -#[test] -fn test_assemble_duplicate_artifact_2() -> Result<()> { - let log_config = ConfigLogging::File { - level: ConfigLoggingLevel::Trace, - path: "UNUSED".into(), - if_exists: ConfigLoggingIfExists::Fail, - }; - let logctx = - LogContext::new("test_assemble_duplicate_artifact_2", &log_config); - let tempdir = tempfile::tempdir().unwrap(); - let key = Key::generate_ed25519()?; - - let archive_path = tempdir.path().join("archive.zip"); - - let mut cmd = make_cmd(&key); - cmd.args([ - "assemble", - "--skip-all-present", - "invalid-manifests/duplicate-artifact-2.toml", - ]); - cmd.arg(&archive_path); - cmd.assert() - .failure() - .stderr(predicate::str::contains("2 errors encountered")) - .stderr(predicate::str::contains( - "a deployment unit with the same kind and hash already exists \ - in this repository:", - )) - .stderr(predicate::str::contains("gimlet_sp/")) - .stderr(predicate::str::contains( - "(existing name: fake-gimlet-sp, version: 1.0.0; \ - new name: fake-gimlet-sp, version: 2.0.0)", - )) - .stderr(predicate::str::contains("switch_sp/")) - .stderr(predicate::str::contains( - "(existing name: fake-switch-sp, version: 1.0.0; \ - new name: fake-switch-sp, version: 2.0.0)", - )); - - logctx.cleanup_successful(); - Ok(()) -} - -fn make_cmd(key: &Key) -> Command { - let mut cmd = Command::cargo_bin("tufaceous").unwrap(); - cmd.env("TUFACEOUS_KEY", key.to_string()); - - cmd -} diff --git a/bin/tests/integration-tests/main.rs b/bin/tests/integration-tests/main.rs deleted file mode 100644 index fdf11b8..0000000 --- a/bin/tests/integration-tests/main.rs +++ /dev/null @@ -1,5 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -mod command_tests; diff --git a/bin/tests/manifest-tests.rs b/bin/tests/manifest-tests.rs deleted file mode 100644 index e099f31..0000000 --- a/bin/tests/manifest-tests.rs +++ /dev/null @@ -1,18 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use std::path::Path; - -use tufaceous_lib::assemble::ArtifactManifest; - -fn check_manifest(path: &Path) -> datatest_stable::Result<()> { - let path = path.try_into()?; - let manifest = - ArtifactManifest::from_path(path).expect("failed to load manifest"); - manifest.verify_all_present()?; - - Ok(()) -} - -datatest_stable::harness!(check_manifest, "manifests", r"^.*\.toml"); diff --git a/brand-metadata/Cargo.toml b/brand-metadata/Cargo.toml index 5bd92c0..5ec7cd2 100644 --- a/brand-metadata/Cargo.toml +++ b/brand-metadata/Cargo.toml @@ -6,7 +6,6 @@ license.workspace = true publish.workspace = true [dependencies] -semver.workspace = true serde.workspace = true serde_json.workspace = true tar.workspace = true diff --git a/brand-metadata/src/lib.rs b/brand-metadata/src/lib.rs index 21af0eb..a5ed8f9 100644 --- a/brand-metadata/src/lib.rs +++ b/brand-metadata/src/lib.rs @@ -9,9 +9,14 @@ //! extended this format with additional archive types for identifying other //! types of tarballs. -use std::io::{Error, ErrorKind, Read, Result, Write}; - -use serde::{Deserialize, Serialize}; +use std::io::Error; +use std::io::ErrorKind; +use std::io::Read; +use std::io::Result; +use std::io::Write; + +use serde::Deserialize; +use serde::Serialize; use tufaceous_artifact::ArtifactVersion; #[derive(Clone, Debug, Deserialize, Serialize)] diff --git a/lib/Cargo.toml b/lib/Cargo.toml index aa1c2d9..c16340e 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -1,50 +1,38 @@ [package] -name = "tufaceous-lib" +name = "tufaceous" version.workspace = true edition.workspace = true license.workspace = true publish.workspace = true [dependencies] -anyhow.workspace = true -async-trait.workspace = true +atomicwrites.workspace = true aws-lc-rs.workspace = true -base64.workspace = true -buf-list.workspace = true bytes.workspace = true camino.workspace = true camino-tempfile.workspace = true chrono.workspace = true -daft.workspace = true -debug-ignore.workspace = true flate2.workspace = true -fs-err.workspace = true -futures.workspace = true +futures-util.workspace = true hex.workspace = true hubtools.workspace = true -indent_write.workspace = true -itertools.workspace = true -parse-size.workspace = true -rand.workspace = true +rawzip.workspace = true semver.workspace = true serde.workspace = true serde_json.workspace = true -serde_path_to_error.workspace = true sha2.workspace = true slog.workspace = true tar.workspace = true thiserror.workspace = true tokio.workspace = true -toml.workspace = true +tokio-util.workspace = true tough.workspace = true -tufaceous-artifact.workspace = true +tufaceous-artifact = { workspace = true, features = ["hubtools"] } tufaceous-brand-metadata.workspace = true url.workspace = true -zip.workspace = true [dev-dependencies] -dropshot.workspace = true -tokio = { workspace = true, features = ["macros", "test-util"] } +tokio = { workspace = true, features = ["macros"] } [lints] workspace = true diff --git a/lib/src/archive.rs b/lib/src/archive.rs deleted file mode 100644 index d209e26..0000000 --- a/lib/src/archive.rs +++ /dev/null @@ -1,297 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -//! Support for reading and writing zip archives. - -use std::fmt; -use std::io::{BufReader, BufWriter, Cursor, Read, Seek}; - -use anyhow::{Context, Result, anyhow, bail}; -use buf_list::BufList; -use bytes::Bytes; -use camino::{Utf8Component, Utf8Path, Utf8PathBuf}; -use debug_ignore::DebugIgnore; -use fs_err::File; -use zip::write::{FileOptions, SimpleFileOptions}; -use zip::{CompressionMethod, ZipArchive, ZipWriter}; - -/// A builder for TUF repo archives. -#[derive(Debug)] -pub(crate) struct ArchiveBuilder { - writer: DebugIgnore>>, - // Stored for better error messages. - output_path: Utf8PathBuf, -} - -/// Defines the base directory for TUF repo archives created by this tool. -/// -/// The usual convention is that the base dir is the name of the archive -/// (e.g. foo-1.0 for foo-1.0.zip). but just using a consistent name here -/// simplifies the code that extracts the archive. -pub const ZIP_BASE_DIR: &str = "repo"; - -impl ArchiveBuilder { - /// Creates a new `ArchiveBuilder`, writing to the given path. - pub fn new(output_path: Utf8PathBuf) -> Result { - // The filename must end with "zip". - if output_path.extension() != Some("zip") { - bail!("output path `{output_path}` must end with .zip"); - } - - let file = File::create(&output_path)?; - let writer = ZipWriter::new(BufWriter::new(file)); - Ok(Self { writer: writer.into(), output_path }) - } - - /// Writes the given path to the archive at the name `name`. - /// - /// The name has [`ZIP_BASE_DIR`] prepended to it. - pub fn write_file( - &mut self, - path: &Utf8Path, - name: &Utf8Path, - ) -> Result<()> { - let name = Utf8Path::new(ZIP_BASE_DIR).join(name); - - self.writer.start_file(name.as_str(), Self::file_options())?; - let mut reader = fs_err::File::open(path)?; - std::io::copy(&mut reader, &mut *self.writer).with_context(|| { - format!( - "error writing `{path}` to archive at `{}`", - self.output_path - ) - })?; - Ok(()) - } - - pub fn finish(self) -> Result<()> { - let Self { writer, output_path } = self; - - let zip_file = writer.0.finish().with_context(|| { - format!("error finalizing archive at `{}`", output_path) - })?; - zip_file.into_inner().with_context(|| { - format!("error writing archive at `{}`", output_path) - })?; - - Ok(()) - } - - fn file_options() -> SimpleFileOptions { - // The main purpose of the zip archive is to transmit archives that are - // already compressed, so there's no point trying to re-compress them. - FileOptions::default().compression_method(CompressionMethod::Stored) - } -} - -/// An extractor for archives created by tufaceous. -/// -/// Ideally we'd just be able to read the TUF repo out of a zip archive in -/// memory, but sadly that isn't possible today due to a missing lifetime -/// parameter on `Transport::fetch`. See [this -/// issue](https://github.com/awslabs/tough/pull/563). -#[derive(Debug)] -pub struct ArchiveExtractor { - archive: ZipArchive, -} - -impl ArchiveExtractor> { - /// Builds an extractor from the given path. - /// - /// The archive must be a zip file generated by tufaceous. - pub fn from_path(zip_path: &Utf8Path) -> Result { - let reader = BufReader::new(File::open(zip_path)?); - Self::new(reader).with_context(|| { - format!("error opening zip archive at `{zip_path}`") - }) - } -} - -impl<'a> ArchiveExtractor> { - /// Loads an archived repository from memory as borrowed bytes. - pub fn from_borrowed_bytes(archive: &'a [u8]) -> Result { - let reader = Cursor::new(archive); - Self::new(reader).context("error opening zip archive from memory") - } -} - -impl ArchiveExtractor> { - /// Loads an archived repository from memory as owned bytes. - pub fn from_owned_bytes(archive: impl Into) -> Result { - let reader = Cursor::new(archive.into()); - Self::new(reader).context("error opening zip archive from memory") - } -} - -impl<'a> ArchiveExtractor> { - /// Loads an archived repository from memory as a borrowed BufList. - pub fn from_borrowed_buf_list(archive: &'a BufList) -> Result { - let reader = buf_list::Cursor::new(archive); - Self::new(reader).context("error opening zip archive from memory") - } -} - -impl ArchiveExtractor> { - /// Loads an archived repository from memory as an owned BufList. - pub fn from_owned_buf_list(archive: BufList) -> Result { - let reader = buf_list::Cursor::new(archive); - Self::new(reader).context("error opening zip archive from memory") - } -} - -impl ArchiveExtractor -where - R: Read + Seek, -{ - /// Creates a new `ArchiveExtractor` from the given reader. - pub fn new(reader: R) -> Result> { - // Validate the archive to ensure all paths are correctly formed. - let archive = Self::validate(ZipArchive::new(reader)?)?; - - Ok(Self { archive }) - } - - fn validate(mut archive: ZipArchive) -> Result> { - for i in 0..archive.len() { - let zip_file = archive.by_index(i).with_context(|| { - format!("error reading file number `{i} from archive") - })?; - if !zip_file.is_file() { - bail!("archive must consist only of files, not directories"); - } - let path = Utf8Path::new(zip_file.name()); - validate_path(path).map_err(|error| { - anyhow!("invalid path in archive `{path}`: {error}") - })?; - } - - Ok(archive) - } - - /// Extracts this archive into the specified directory. - /// - /// Once this is completed, use - /// [`OmicronRepo::load_untrusted`](crate::OmicronRepo::load_untrusted) to - /// load the archive from `output_dir`. - /// - /// [`ZIP_BASE_DIR`] will be stripped from output paths. - pub fn extract(&mut self, output_dir: &Utf8Path) -> Result<()> { - for i in 0..self.archive.len() { - let mut zip_file = self.archive.by_index(i).with_context(|| { - format!("error reading file number `{i} from archive") - })?; - // SAFETY: file names have already been checked in `Self::validate`. - let file_name = Utf8Path::new(zip_file.name()).to_owned(); - let dest_path = output_dir.join( - file_name - .strip_prefix(ZIP_BASE_DIR) - .expect("checked in Self::validate"), - ); - // The file is in a directory. - fs_err::create_dir_all( - dest_path.parent().expect("at least 1 component"), - )?; - - let mut writer = File::create(&dest_path)?; - std::io::copy(&mut zip_file, &mut writer).with_context(|| { - format!( - "error writing `{file_name}` in archive to `{dest_path}`" - ) - })?; - } - - Ok(()) - } -} - -#[derive(Clone, Debug, Eq, PartialEq)] -enum InvalidPath<'a> { - AbsolutePath, - ExactlyBaseDir, - IncorrectBaseDir, - InvalidComponent(Utf8Component<'a>), -} - -impl fmt::Display for InvalidPath<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - InvalidPath::AbsolutePath => { - write!(f, "path is absolute -- expected relative paths") - } - InvalidPath::ExactlyBaseDir => { - write!( - f, - "path is exactly `{ZIP_BASE_DIR}` -- expected `{ZIP_BASE_DIR}/`" - ) - } - InvalidPath::IncorrectBaseDir => { - write!(f, "invalid base directory -- must be `{ZIP_BASE_DIR}`") - } - InvalidPath::InvalidComponent(component) => { - write!(f, "invalid component `{component}`") - } - } - } -} - -fn validate_path(path: &Utf8Path) -> Result<(), InvalidPath<'_>> { - if path.is_absolute() { - return Err(InvalidPath::AbsolutePath); - } - if path == ZIP_BASE_DIR { - return Err(InvalidPath::ExactlyBaseDir); - } - if !path.starts_with(ZIP_BASE_DIR) { - return Err(InvalidPath::IncorrectBaseDir); - } - - for component in path.components() { - if !matches!(component, Utf8Component::Normal(_)) { - return Err(InvalidPath::InvalidComponent(component)); - } - } - - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_validate_path() { - let valid = ["repo/foo", "repo/foo/bar", "repo/foo/./bar", "repo//foo"]; - let invalid = [ - ("repo", InvalidPath::ExactlyBaseDir), - ("repo/", InvalidPath::ExactlyBaseDir), - ("repo/.", InvalidPath::ExactlyBaseDir), - ("not-repo", InvalidPath::IncorrectBaseDir), - ("not-repo/foo", InvalidPath::IncorrectBaseDir), - ( - "repo/..", - InvalidPath::InvalidComponent(Utf8Component::ParentDir), - ), - ("/repo/foo", InvalidPath::AbsolutePath), - ]; - - for path in valid { - validate_path(Utf8Path::new(path)).unwrap_or_else(|err| { - panic!("expected path `{path}` to be valid: {err}") - }); - } - - for (path, expected) in invalid { - eprintln!("testing invalid path: `{path}`"); - let actual = match validate_path(Utf8Path::new(path)) { - Ok(()) => panic!("expected path `{path}` to be invalid"), - Err(error) => error, - }; - - assert_eq!( - actual, expected, - "for path `{path}`, InvalidPath error should match" - ); - } - } -} diff --git a/lib/src/artifact.rs b/lib/src/artifact.rs deleted file mode 100644 index 600b8a6..0000000 --- a/lib/src/artifact.rs +++ /dev/null @@ -1,638 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use std::io::{self, BufReader, Write}; -use std::path::Path; - -use anyhow::{Context, Result, bail}; -use buf_list::BufList; -use bytes::Bytes; -use camino::{Utf8Path, Utf8PathBuf}; -use fs_err::File; -use sha2::{Digest, Sha256}; -use tough::editor::RepositoryEditor; -use tufaceous_artifact::{ - ArtifactHash, ArtifactKind, ArtifactVersion, InstallinatorArtifact, - InstallinatorArtifactKind, KnownArtifactKind, -}; -use tufaceous_brand_metadata::Metadata; - -mod composite; - -pub use composite::CompositeControlPlaneArchiveBuilder; -pub use composite::CompositeEntry; -pub use composite::CompositeHostArchiveBuilder; -pub use composite::CompositeRotArchiveBuilder; -pub use composite::MtimeSource; - -use crate::assemble::ArtifactDeploymentUnits; -use crate::target::{TargetFinishWrite, TargetWriter}; - -/// The location a artifact will be obtained from. -#[derive(Clone, Debug)] -pub enum ArtifactSource { - File(Utf8PathBuf), - Memory(BufList), - // We might need to support downloading data over HTTP as well -} - -/// Describes a new artifact to be added. -pub struct AddArtifact { - kind: ArtifactKind, - name: String, - version: ArtifactVersion, - source: ArtifactSource, - deployment_units: ArtifactDeploymentUnits, -} - -impl AddArtifact { - /// Creates an [`AddArtifact`] from the provided source. - pub(crate) fn new( - kind: ArtifactKind, - name: String, - version: ArtifactVersion, - source: ArtifactSource, - deployment_units: ArtifactDeploymentUnits, - ) -> Self { - Self { kind, name, version, source, deployment_units } - } - - /// Returns the kind of artifact this is. - pub fn kind(&self) -> &ArtifactKind { - &self.kind - } - - /// Returns the name of the new artifact. - pub fn name(&self) -> &str { - &self.name - } - - /// Returns the version of the new artifact. - pub fn version(&self) -> &ArtifactVersion { - &self.version - } - - /// Returns the source for this artifact. - pub fn source(&self) -> &ArtifactSource { - &self.source - } - - /// Returns information about deployment units for this artifact. - pub fn deployment_units(&self) -> &ArtifactDeploymentUnits { - &self.deployment_units - } - - pub(crate) fn target_name(&self) -> String { - format!("{}-{}-{}.tar.gz", self.kind, self.name, self.version) - } - - /// Writes this artifact as a temporary file, returning a - /// [`TempWrittenArtifact`]. - pub(crate) fn write_temp( - &self, - targets_dir: &Utf8Path, - ) -> Result { - let target_name = self.target_name(); - let mut file = TargetWriter::new(targets_dir, &target_name)?; - self.write_to(&mut file).with_context(|| { - format!("error writing artifact `{target_name}") - })?; - let finished_file = file.finish_write(); - Ok(TempWrittenArtifact { - kind: self.kind.clone(), - name: self.name.clone(), - version: self.version.clone(), - deployment_units: self.deployment_units.clone(), - finished_file, - }) - } - - /// Writes this artifact to the specifid writer. - fn write_to(&self, writer: &mut W) -> Result<()> { - match &self.source { - ArtifactSource::File(path) => { - let mut reader = File::open(path)?; - std::io::copy(&mut reader, writer)?; - } - ArtifactSource::Memory(buf_list) => { - for chunk in buf_list { - writer.write_all(chunk)?; - } - } - } - - Ok(()) - } -} - -/// A newly-added artifact that's been written out to a temporary file. -#[must_use = "the artifact is still temporary and must be finalized"] -pub(crate) struct TempWrittenArtifact { - kind: ArtifactKind, - name: String, - version: ArtifactVersion, - deployment_units: ArtifactDeploymentUnits, - finished_file: TargetFinishWrite, -} - -impl TempWrittenArtifact { - pub(crate) fn name(&self) -> &str { - &self.name - } - - pub(crate) fn version(&self) -> &ArtifactVersion { - &self.version - } - - pub(crate) fn kind(&self) -> &ArtifactKind { - &self.kind - } - - pub(crate) fn digest(&self) -> ArtifactHash { - self.finished_file.digest() - } - - pub(crate) fn deployment_units(&self) -> &ArtifactDeploymentUnits { - &self.deployment_units - } - - /// Returns information about installinator artifacts for this newly-added - /// artifact. - pub(crate) fn installinator_artifacts( - &self, - ) -> impl Iterator + '_ { - let known = self.kind.to_known(); - - // Currently, a `TempWrittenArtifact` corresponds to zero or one - // installinator artifacts so we can just return an Option. If, in the - // future, a single `TempWrittenArtifact` corresponds to multiple - // installinator artifacts, we'd have to return a more complex iterator. - let artifact = match known { - Some(KnownArtifactKind::Host) => { - // The host phase 2 artifact is an installinator artifact. - let host_phase_2 = match &self.deployment_units { - ArtifactDeploymentUnits::SingleUnit - | ArtifactDeploymentUnits::Unknown => { - panic!( - "expected Host artifact to be Composite, found {:?}", - self.deployment_units - ); - } - ArtifactDeploymentUnits::Composite { deployment_units } => { - deployment_units - .values() - .find(|unit| { - unit.kind == ArtifactKind::HOST_PHASE_2 - }) - .unwrap_or_else(|| { - panic!( - "Host artifact must have a host phase 2 \ - deployment unit, found {:?}", - deployment_units, - ) - }) - } - }; - Some(InstallinatorArtifact { - name: host_phase_2.name.clone(), - kind: InstallinatorArtifactKind::HostPhase2, - hash: host_phase_2.hash, - }) - } - Some(KnownArtifactKind::ControlPlane) => { - Some(InstallinatorArtifact { - name: self.name.clone(), - kind: InstallinatorArtifactKind::ControlPlane, - hash: self.digest(), - }) - } - Some(KnownArtifactKind::MeasurementCorpus) => { - Some(InstallinatorArtifact { - name: self.name.clone(), - kind: InstallinatorArtifactKind::MeasurementCorpus, - hash: self.digest(), - }) - } - Some(_) | None => None, - }; - artifact.into_iter() - } - - pub(crate) fn finalize( - self, - editor: &mut RepositoryEditor, - ) -> Result { - self.finished_file.finalize(editor) - } -} - -pub(crate) fn make_filler_text_with_seed( - // composite artifact. - kind: &str, - version: &ArtifactVersion, - length: usize, - seed: &str, -) -> Vec { - // Add the kind and version to the filler text first. This ensures that - // hashes are unique by kind and version. - let mut out = Vec::with_capacity(length); - out.extend_from_slice(kind.as_bytes()); - out.extend_from_slice(b":"); - out.extend_from_slice(version.as_str().as_bytes()); - out.extend_from_slice(b":"); - out.extend_from_slice(seed.as_bytes()); - out.extend_from_slice(b":"); - let remaining = length.saturating_sub(out.len()); - out.extend( - std::iter::repeat(FILLER_TEXT).flatten().copied().take(remaining), - ); - - out -} - -pub(crate) fn make_filler_text( - // composite artifact. - kind: &str, - version: &ArtifactVersion, - length: usize, -) -> Vec { - // Add the kind and version to the filler text first. This ensures that - // hashes are unique by kind and version. - let mut out = Vec::with_capacity(length); - out.extend_from_slice(kind.as_bytes()); - out.extend_from_slice(b":"); - out.extend_from_slice(version.as_str().as_bytes()); - out.extend_from_slice(b":"); - - let remaining = length.saturating_sub(out.len()); - out.extend( - std::iter::repeat(FILLER_TEXT).flatten().copied().take(remaining), - ); - - out -} - -/// Represents host phase images. -/// -/// The host and trampoline artifacts are actually tarballs, with phase 1 and -/// phase 2 images inside them. This code extracts those images out of the -/// tarballs. -#[derive(Clone, Debug)] -pub struct HostPhaseImages { - pub gimlet_phase_1: Bytes, - pub cosmo_phase_1: Bytes, - pub phase_2: Bytes, -} - -/// File sources for extraction -/// -/// Passing three identical arguments gets confusing and error prone -pub struct HostPhaseImageSource { - pub gimlet_phase_1: W, - pub cosmo_phase_1: W, - pub phase_2: W, -} - -impl HostPhaseImages { - pub fn extract(reader: R) -> Result { - let mut gimlet_phase_1 = Vec::new(); - let mut cosmo_phase_1 = Vec::new(); - let mut phase_2 = Vec::new(); - let source = HostPhaseImageSource { - gimlet_phase_1: io::Cursor::<&mut Vec>::new( - &mut gimlet_phase_1, - ), - cosmo_phase_1: io::Cursor::<&mut Vec>::new(&mut cosmo_phase_1), - phase_2: io::Cursor::<&mut Vec>::new(&mut phase_2), - }; - - Self::extract_into(reader, source)?; - Ok(Self { - gimlet_phase_1: gimlet_phase_1.into(), - cosmo_phase_1: cosmo_phase_1.into(), - phase_2: phase_2.into(), - }) - } - - pub fn extract_into( - reader: R, - source: HostPhaseImageSource, - ) -> Result<()> { - let uncompressed = flate2::bufread::GzDecoder::new(reader); - let mut archive = tar::Archive::new(uncompressed); - - let mut oxide_json_found = false; - let mut gimlet_phase_1_writer = Some(source.gimlet_phase_1); - let mut cosmo_phase_1_writer = Some(source.cosmo_phase_1); - let mut phase_2_writer = Some(source.phase_2); - for entry in archive - .entries() - .context("error building list of entries from archive")? - { - let entry = entry.context("error reading entry from archive")?; - let path = entry - .header() - .path() - .context("error reading path from archive")?; - if path == Path::new(OXIDE_JSON_FILE_NAME) { - let json_bytes = read_entry(entry, OXIDE_JSON_FILE_NAME)?; - let metadata: Metadata = - serde_json::from_slice(&json_bytes).with_context(|| { - format!( - "error deserializing JSON from {OXIDE_JSON_FILE_NAME}" - ) - })?; - if !metadata.is_os() { - bail!( - "unexpected archive type: expected os, found {:?}", - metadata.archive_type(), - ) - } - oxide_json_found = true; - } else if path == Path::new(COSMO_HOST_PHASE_1_FILE_NAME) { - if let Some(cosmo_phase_1) = cosmo_phase_1_writer.take() { - read_entry_into( - entry, - COSMO_HOST_PHASE_1_FILE_NAME, - cosmo_phase_1, - )?; - } - } else if path == Path::new(GIMLET_HOST_PHASE_1_FILE_NAME) { - if let Some(gimlet_phase_1) = gimlet_phase_1_writer.take() { - read_entry_into( - entry, - GIMLET_HOST_PHASE_1_FILE_NAME, - gimlet_phase_1, - )?; - } - } else if path == Path::new(HOST_PHASE_2_FILE_NAME) { - if let Some(phase_2) = phase_2_writer.take() { - read_entry_into(entry, HOST_PHASE_2_FILE_NAME, phase_2)?; - } - } - - if oxide_json_found - && gimlet_phase_1_writer.is_none() - && cosmo_phase_1_writer.is_none() - && phase_2_writer.is_none() - { - break; - } - } - - let mut not_found = Vec::new(); - if !oxide_json_found { - not_found.push(OXIDE_JSON_FILE_NAME); - } - - // If we didn't `.take()` the writer out of the options, we never saw - // the expected phase1/phase2 filenames. - if cosmo_phase_1_writer.is_some() { - not_found.push(COSMO_HOST_PHASE_1_FILE_NAME); - } - if gimlet_phase_1_writer.is_some() { - not_found.push(GIMLET_HOST_PHASE_1_FILE_NAME); - } - if phase_2_writer.is_some() { - not_found.push(HOST_PHASE_2_FILE_NAME); - } - - if !not_found.is_empty() { - bail!("required files not found: {}", not_found.join(", ")) - } - - Ok(()) - } - - pub fn gimlet_phase_1_hash(&self) -> ArtifactHash { - let hash = Sha256::digest(&self.gimlet_phase_1); - ArtifactHash(hash.into()) - } - - pub fn cosmo_phase_1_hash(&self) -> ArtifactHash { - let hash = Sha256::digest(&self.cosmo_phase_1); - ArtifactHash(hash.into()) - } - - pub fn phase_2_hash(&self) -> ArtifactHash { - let hash = Sha256::digest(&self.phase_2); - ArtifactHash(hash.into()) - } -} - -fn read_entry( - entry: tar::Entry, - file_name: &str, -) -> Result { - let mut buf = Vec::new(); - read_entry_into(entry, file_name, io::Cursor::new(&mut buf))?; - Ok(buf.into()) -} - -fn read_entry_into( - mut entry: tar::Entry, - file_name: &str, - mut out: W, -) -> Result<()> { - let entry_type = entry.header().entry_type(); - if entry_type != tar::EntryType::Regular { - bail!("for {file_name}, expected regular file, found {entry_type:?}"); - } - io::copy(&mut entry, &mut out) - .with_context(|| format!("error reading {file_name} from archive"))?; - Ok(()) -} - -/// Represents RoT A/B hubris archives. -/// -/// RoT artifacts are actually tarballs, with both A and B hubris archives -/// inside them. This code extracts those archives out of the tarballs. -#[derive(Clone, Debug)] -pub struct RotArchives { - pub archive_a: Bytes, - pub archive_b: Bytes, -} - -impl RotArchives { - pub fn extract(reader: R) -> Result { - let mut archive_a = Vec::new(); - let mut archive_b = Vec::new(); - Self::extract_into( - reader, - io::Cursor::<&mut Vec>::new(&mut archive_a), - io::Cursor::<&mut Vec>::new(&mut archive_b), - )?; - Ok(Self { archive_a: archive_a.into(), archive_b: archive_b.into() }) - } - - pub fn extract_into( - reader: R, - archive_a: W, - archive_b: W, - ) -> Result<()> { - let uncompressed = flate2::bufread::GzDecoder::new(reader); - let mut archive = tar::Archive::new(uncompressed); - - let mut oxide_json_found = false; - let mut archive_a_writer = Some(archive_a); - let mut archive_b_writer = Some(archive_b); - for entry in archive - .entries() - .context("error building list of entries from archive")? - { - let entry = entry.context("error reading entry from archive")?; - let path = entry - .header() - .path() - .context("error reading path from archive")?; - if path == Path::new(OXIDE_JSON_FILE_NAME) { - let json_bytes = read_entry(entry, OXIDE_JSON_FILE_NAME)?; - let metadata: Metadata = - serde_json::from_slice(&json_bytes).with_context(|| { - format!( - "error deserializing JSON from {OXIDE_JSON_FILE_NAME}" - ) - })?; - if !metadata.is_rot() { - bail!( - "unexpected archive type: expected rot, found {:?}", - metadata.archive_type(), - ) - } - oxide_json_found = true; - } else if path == Path::new(ROT_ARCHIVE_A_FILE_NAME) { - if let Some(archive_a) = archive_a_writer.take() { - read_entry_into(entry, ROT_ARCHIVE_A_FILE_NAME, archive_a)?; - } - } else if path == Path::new(ROT_ARCHIVE_B_FILE_NAME) { - if let Some(archive_b) = archive_b_writer.take() { - read_entry_into(entry, ROT_ARCHIVE_B_FILE_NAME, archive_b)?; - } - } - - if oxide_json_found - && archive_a_writer.is_none() - && archive_b_writer.is_none() - { - break; - } - } - - let mut not_found = Vec::new(); - if !oxide_json_found { - not_found.push(OXIDE_JSON_FILE_NAME); - } - - // If we didn't `.take()` the writer out of the options, we never saw - // the expected A/B filenames. - if archive_a_writer.is_some() { - not_found.push(ROT_ARCHIVE_A_FILE_NAME); - } - if archive_b_writer.is_some() { - not_found.push(ROT_ARCHIVE_B_FILE_NAME); - } - - if !not_found.is_empty() { - bail!("required files not found: {}", not_found.join(", ")) - } - - Ok(()) - } -} - -/// Represents control plane zone images. -/// -/// The control plane artifact is actually a tarball that contains a set of zone -/// images. This code extracts those images out of the tarball. -#[derive(Clone, Debug)] -pub struct ControlPlaneZoneImages { - pub zones: Vec<(String, Bytes)>, -} - -impl ControlPlaneZoneImages { - pub fn extract(reader: R) -> Result { - let mut zones = Vec::new(); - Self::extract_into(reader, |name, reader| { - let mut buf = Vec::new(); - io::copy(reader, &mut buf)?; - zones.push((name, buf.into())); - Ok(()) - })?; - Ok(Self { zones }) - } - - pub fn extract_into(reader: R, mut handler: F) -> Result<()> - where - R: io::Read, - F: FnMut(String, &mut dyn io::Read) -> Result<()>, - { - let uncompressed = - flate2::bufread::GzDecoder::new(BufReader::new(reader)); - let mut archive = tar::Archive::new(uncompressed); - - let mut oxide_json_found = false; - let mut zone_found = false; - for entry in archive - .entries() - .context("error building list of entries from archive")? - { - let mut entry = - entry.context("error reading entry from archive")?; - let path = entry - .header() - .path() - .context("error reading path from archive")?; - if path == Path::new(OXIDE_JSON_FILE_NAME) { - let json_bytes = read_entry(entry, OXIDE_JSON_FILE_NAME)?; - let metadata: Metadata = - serde_json::from_slice(&json_bytes).with_context(|| { - format!( - "error deserializing JSON from {OXIDE_JSON_FILE_NAME}" - ) - })?; - if !metadata.is_control_plane() { - bail!( - "unexpected archive type: expected control_plane, found {:?}", - metadata.archive_type(), - ) - } - oxide_json_found = true; - } else if path.starts_with(CONTROL_PLANE_ARCHIVE_ZONE_DIRECTORY) { - if let Some(name) = path - .file_name() - .and_then(|s| s.to_str()) - .map(|s| s.to_string()) - { - handler(name, &mut entry)?; - } - zone_found = true; - } - } - - let mut not_found = Vec::new(); - if !oxide_json_found { - not_found.push(OXIDE_JSON_FILE_NAME); - } - if !not_found.is_empty() { - bail!("required files not found: {}", not_found.join(", ")) - } - if !zone_found { - bail!( - "no zone images found in `{}/`", - CONTROL_PLANE_ARCHIVE_ZONE_DIRECTORY - ); - } - - Ok(()) - } -} - -static FILLER_TEXT: &[u8; 16] = b"tufaceousfaketxt"; -static OXIDE_JSON_FILE_NAME: &str = "oxide.json"; -pub(crate) static GIMLET_HOST_PHASE_1_FILE_NAME: &str = "image/gimlet.rom"; -pub(crate) static COSMO_HOST_PHASE_1_FILE_NAME: &str = "image/cosmo.rom"; -pub(crate) static HOST_PHASE_2_FILE_NAME: &str = "image/zfs.img"; -pub(crate) static ROT_ARCHIVE_A_FILE_NAME: &str = "archive-a.zip"; -pub(crate) static ROT_ARCHIVE_B_FILE_NAME: &str = "archive-b.zip"; -static CONTROL_PLANE_ARCHIVE_ZONE_DIRECTORY: &str = "zones"; diff --git a/lib/src/artifact/composite.rs b/lib/src/artifact/composite.rs deleted file mode 100644 index 3a7eab6..0000000 --- a/lib/src/artifact/composite.rs +++ /dev/null @@ -1,220 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use std::io::{BufWriter, Write}; - -use anyhow::{Context, Result, anyhow, bail}; -use camino::Utf8Path; -use flate2::Compression; -use flate2::write::GzEncoder; -use sha2::{Digest, Sha256}; -use tufaceous_artifact::ArtifactHash; -use tufaceous_brand_metadata::{ArchiveType, Metadata}; - -use super::{ - CONTROL_PLANE_ARCHIVE_ZONE_DIRECTORY, COSMO_HOST_PHASE_1_FILE_NAME, - GIMLET_HOST_PHASE_1_FILE_NAME, HOST_PHASE_2_FILE_NAME, - ROT_ARCHIVE_A_FILE_NAME, ROT_ARCHIVE_B_FILE_NAME, -}; - -/// Represents a single entry in a composite artifact. -/// -/// A composite artifact is a tarball containing multiple artifacts. This -/// struct is intended for the insertion of one such entry into the artifact. -/// -/// At the moment it only accepts byte slices, but it could be extended to -/// support arbitrary readers in the future. -pub struct CompositeEntry<'a> { - pub data: &'a [u8], - pub mtime_source: MtimeSource, -} - -pub struct CompositeControlPlaneArchiveBuilder { - inner: CompositeTarballBuilder, -} - -impl CompositeControlPlaneArchiveBuilder { - pub fn new(writer: W, mtime_source: MtimeSource) -> Result { - let metadata = Metadata::new(ArchiveType::ControlPlane); - let inner = - CompositeTarballBuilder::new(writer, metadata, mtime_source)?; - Ok(Self { inner }) - } - - pub fn append_zone( - &mut self, - name: &str, - entry: CompositeEntry<'_>, - ) -> Result<(ArtifactHash, String)> { - let name_path = Utf8Path::new(name); - if name_path.file_name() != Some(name) { - bail!("control plane zone filenames should not contain paths"); - } - let path = - Utf8Path::new(CONTROL_PLANE_ARCHIVE_ZONE_DIRECTORY).join(name_path); - let hash = self.inner.append_file(path.as_str(), entry)?; - Ok((hash, name.to_owned())) - } - - pub fn finish(self) -> Result { - self.inner.finish() - } -} - -pub struct CompositeRotArchiveBuilder { - inner: CompositeTarballBuilder, -} - -impl CompositeRotArchiveBuilder { - pub fn new(writer: W, mtime_source: MtimeSource) -> Result { - let metadata = Metadata::new(ArchiveType::Rot); - let inner = - CompositeTarballBuilder::new(writer, metadata, mtime_source)?; - Ok(Self { inner }) - } - - pub fn append_archive_a( - &mut self, - entry: CompositeEntry<'_>, - ) -> Result { - self.inner.append_file(ROT_ARCHIVE_A_FILE_NAME, entry) - } - - pub fn append_archive_b( - &mut self, - entry: CompositeEntry<'_>, - ) -> Result { - self.inner.append_file(ROT_ARCHIVE_B_FILE_NAME, entry) - } - - pub fn finish(self) -> Result { - self.inner.finish() - } -} - -pub struct CompositeHostArchiveBuilder { - inner: CompositeTarballBuilder, -} - -impl CompositeHostArchiveBuilder { - pub fn new(writer: W, mtime_source: MtimeSource) -> Result { - let metadata = Metadata::new(ArchiveType::Os); - let inner = - CompositeTarballBuilder::new(writer, metadata, mtime_source)?; - Ok(Self { inner }) - } - - pub fn append_gimlet_phase_1( - &mut self, - entry: CompositeEntry<'_>, - ) -> Result { - self.inner.append_file(GIMLET_HOST_PHASE_1_FILE_NAME, entry) - } - - pub fn append_cosmo_phase_1( - &mut self, - entry: CompositeEntry<'_>, - ) -> Result { - self.inner.append_file(COSMO_HOST_PHASE_1_FILE_NAME, entry) - } - - pub fn append_phase_2( - &mut self, - entry: CompositeEntry<'_>, - ) -> Result { - self.inner.append_file(HOST_PHASE_2_FILE_NAME, entry) - } - - pub fn finish(self) -> Result { - self.inner.finish() - } -} - -struct CompositeTarballBuilder { - builder: tar::Builder>>, -} - -impl CompositeTarballBuilder { - fn new( - writer: W, - metadata: Metadata, - mtime_source: MtimeSource, - ) -> Result { - let mut builder = tar::Builder::new(GzEncoder::new( - BufWriter::new(writer), - Compression::fast(), - )); - metadata.append_to_tar(&mut builder, mtime_source.into_mtime())?; - Ok(Self { builder }) - } - - fn append_file( - &mut self, - path: &str, - entry: CompositeEntry<'_>, - ) -> Result { - let header = - make_tar_header(path, entry.data.len(), entry.mtime_source); - self.builder - .append(&header, entry.data) - .with_context(|| format!("error append {path:?}"))?; - let hash = Sha256::digest(entry.data); - Ok(ArtifactHash(hash.into())) - } - - fn finish(self) -> Result { - let gz_encoder = - self.builder.into_inner().context("error finalizing archive")?; - let buf_writer = - gz_encoder.finish().context("error finishing gz encoder")?; - buf_writer - .into_inner() - .map_err(|_| anyhow!("error flushing buffered archive writer")) - } -} - -fn make_tar_header( - path: &str, - size: usize, - mtime_source: MtimeSource, -) -> tar::Header { - let mtime = mtime_source.into_mtime(); - - let mut header = tar::Header::new_ustar(); - header.set_username("root").unwrap(); - header.set_uid(0); - header.set_groupname("root").unwrap(); - header.set_gid(0); - header.set_path(path).unwrap(); - header.set_size(size as u64); - header.set_mode(0o444); - header.set_entry_type(tar::EntryType::Regular); - header.set_mtime(mtime); - header.set_cksum(); - - header -} - -/// How to obtain the `mtime` field for a tar header. -#[derive(Copy, Clone, Debug)] -pub enum MtimeSource { - /// Use a fixed timestamp of zero seconds past the Unix epoch. - Zero, - - /// Use the current time. - Now, -} - -impl MtimeSource { - pub(crate) fn into_mtime(self) -> u64 { - use std::time::{SystemTime, UNIX_EPOCH}; - - match self { - Self::Zero => 0, - Self::Now => { - SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() - } - } - } -} diff --git a/lib/src/assemble/build.rs b/lib/src/assemble/build.rs deleted file mode 100644 index 947c497..0000000 --- a/lib/src/assemble/build.rs +++ /dev/null @@ -1,167 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use anyhow::{Context, Result}; -use camino::{Utf8Path, Utf8PathBuf}; -use chrono::{DateTime, Utc}; -use tough::editor::signed::SignedRole; -use tough::schema::Root; - -use crate::{ - AddArtifact, Key, OmicronRepo, OmicronRepoEditor, utils::merge_anyhow_list, -}; - -use super::ArtifactManifest; - -/// Assembles a TUF repo from a list of artifacts. -#[derive(Debug)] -pub struct OmicronRepoAssembler { - log: slog::Logger, - manifest: ArtifactManifest, - build_dir: Option, - keys: Vec, - root: Option>, - expiry: DateTime, - include_installinator_doc: bool, - output_path: Utf8PathBuf, -} - -impl OmicronRepoAssembler { - pub fn new( - log: &slog::Logger, - manifest: ArtifactManifest, - keys: Vec, - expiry: DateTime, - include_installinator_doc: bool, - output_path: Utf8PathBuf, - ) -> Self { - Self { - log: log.new(slog::o!("component" => "OmicronRepoAssembler")), - manifest, - build_dir: None, - keys, - root: None, - expiry, - include_installinator_doc, - output_path, - } - } - - pub fn set_build_dir(&mut self, build_dir: Utf8PathBuf) -> &mut Self { - self.build_dir = Some(build_dir); - self - } - - pub fn set_root_role(&mut self, root_role: SignedRole) -> &mut Self { - self.root = Some(root_role); - self - } - - pub async fn build(&self) -> Result<()> { - let (build_dir, is_temp) = match &self.build_dir { - Some(dir) => (dir.clone(), false), - None => { - // Create a new temporary directory. - let dir = camino_tempfile::Builder::new() - .prefix("tufaceous") - .tempdir()?; - // This will cause the directory to be preserved -- we're going - // to clean it up if it's successful. - let path = dir.keep(); - (path, true) - } - }; - - slog::info!(self.log, "assembling repository in `{build_dir}`"); - - match self.build_impl(&build_dir).await { - Ok(()) => { - if is_temp { - slog::debug!(self.log, "assembly successful, cleaning up"); - // Log, but otherwise ignore, errors while cleaning up. - if let Err(error) = fs_err::remove_dir_all(&build_dir) { - slog::warn!( - self.log, - "failed to clean up temporary directory {build_dir}: {error}" - ) - } - } - - slog::info!( - self.log, - "artifacts assembled and archived to `{}`", - self.output_path - ); - Ok(()) - } - Err(error) => { - slog::error!( - self.log, - "assembly failed, failing build directory preserved: `{build_dir}`" - ); - Err(error) - } - } - } - - async fn build_impl(&self, build_dir: &Utf8Path) -> Result<()> { - let root = match &self.root { - Some(root) => root.clone(), - None => { - crate::root::new_root(self.keys.clone(), self.expiry).await? - } - }; - let mut repository = OmicronRepoEditor::initialize( - build_dir.to_owned(), - root, - self.manifest.system_version.clone(), - ) - .await?; - - // Add all the artifacts. - let mut errors = Vec::new(); - - for (kind, entries) in &self.manifest.artifacts { - for data in entries { - let new_artifact = AddArtifact::new( - (*kind).into(), - data.name.clone(), - data.version.clone(), - data.source.clone(), - data.deployment_units.clone(), - ); - let res = - repository.add_artifact(&new_artifact).with_context(|| { - format!("error adding artifact with kind `{kind}`") - }); - if let Err(err) = res { - errors.push(err); - } - } - } - - if !errors.is_empty() { - return Err(merge_anyhow_list(errors)); - } - - // Write out the repository. - repository - .sign_and_finish( - self.keys.clone(), - self.expiry, - self.include_installinator_doc, - ) - .await?; - - // Now reopen the repository to archive it into a zip file. - let repo2 = OmicronRepo::load_untrusted(&self.log, build_dir) - .await - .context("error reopening repository to archive")?; - repo2 - .archive(&self.output_path) - .context("error archiving repository")?; - - Ok(()) - } -} diff --git a/lib/src/assemble/deployment_unit.rs b/lib/src/assemble/deployment_unit.rs deleted file mode 100644 index 698d397..0000000 --- a/lib/src/assemble/deployment_unit.rs +++ /dev/null @@ -1,262 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use std::{collections::BTreeMap, fmt}; - -use daft::{Diffable, Leaf}; -use thiserror::Error; -use tufaceous_artifact::{ - ArtifactHash, ArtifactHashId, ArtifactKind, ArtifactVersion, - KnownArtifactKind, -}; - -/// Information about deployment units keyed by hash ID. -/// -/// This information is used to ensure uniqueness of deployment units within a -/// particular scope. -pub type DeploymentUnitMap = BTreeMap; - -/// Information about deployment units for an artifact. -#[derive(Clone, Debug)] -pub enum ArtifactDeploymentUnits { - /// This artifact is single-unit (not composite). There is exactly one - /// deployment unit, which is the artifact itself. - SingleUnit, - - /// This is a composite artifact, with these deployment units. - Composite { deployment_units: DeploymentUnitMap }, - - /// Data was not available for this artifact. - Unknown, -} - -/// Ths scope in which deployment unit data is being gathered. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum DeploymentUnitScope { - /// Deployment units are being gathered for a single composite artifact. - Artifact { - /// The kind of the composite artifact. - composite_kind: KnownArtifactKind, - }, - - /// Deployment units are being gathered for a repository. - Repository, -} - -impl fmt::Display for DeploymentUnitScope { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - DeploymentUnitScope::Artifact { composite_kind } => { - write!(f, "{composite_kind} artifact") - } - DeploymentUnitScope::Repository => "repository".fmt(f), - } - } -} - -/// Data associated with a deployment unit. -#[derive(Clone, Debug)] -pub struct DeploymentUnitData { - /// The name of the deployment unit. - pub name: String, - - /// The version of the deployment unit. - pub version: ArtifactVersion, - - /// The kind of the deployment unit. - pub kind: ArtifactKind, - - /// The hash of the deployment unit. - pub hash: ArtifactHash, -} - -impl DeploymentUnitData { - /// Returns the [`ArtifactHashId`] of the deployment unit. - #[inline] - pub fn hash_id(&self) -> ArtifactHashId { - ArtifactHashId { kind: self.kind.clone(), hash: self.hash } - } -} - -impl fmt::Display for DeploymentUnitData { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "name: {}, version: {}", self.name, self.version) - } -} - -/// Builder for a [`DeploymentUnitMap`]. -/// -/// This builder ensures uniqueness of deployment units within a particular -/// [`DeploymentUnitScope`]. -#[derive(Clone, Debug)] -pub struct DeploymentUnitMapBuilder { - scope: DeploymentUnitScope, - // TODO: This should also check for duplicate name/version/kind. Will - // probably want to use a data structure more tailored for this. - deployment_units: DeploymentUnitMap, -} - -impl DeploymentUnitMapBuilder { - pub fn new(scope: DeploymentUnitScope) -> Self { - Self { scope, deployment_units: BTreeMap::new() } - } - - /// Starts adding a new deployment unit to `self`. - pub fn start_insert( - &mut self, - data: DeploymentUnitData, - ) -> Result, DuplicateDeploymentUnitError> { - let hash_id = data.hash_id(); - if let Some(existing) = self.deployment_units.get(&hash_id) { - return Err(DuplicateDeploymentUnitError::new_single( - self.scope, - existing.clone(), - data, - )); - }; - - Ok(NewDeploymentUnits::new_single(&mut self.deployment_units, data)) - } - - /// Starts a bulk insert of another deployment unit map into `self`. The - /// merge is not committed until `NewDeploymentUnits::commit` is called. - /// - /// Returns an error if any duplicates are found. - pub fn start_bulk_insert( - &mut self, - units: DeploymentUnitMap, - ) -> Result, DuplicateDeploymentUnitError> { - // Check that there are no duplicates. We don't expect to see any - // duplicated artifacts at all within a single artifact or repository, - // so we don't check whether the hashes are the same. - // - // In order for this check to be done now rather than at commit time, we - // rely on two things: - // - // 1. `start_bulk_insert` accepts a `&mut self` parameter. - // 2. There is no interior mutability in `DeploymentUnitMapBuilder`. - // - // Together, these two checks ensure that nothing else can modify the - // map between now and either commit or discard. - let diff = self.deployment_units.diff(&units); - - if !diff.common.is_empty() { - return Err(DuplicateDeploymentUnitError { - scope: self.scope, - duplicates: diff - .common - .into_iter() - .map(|(k, v)| (k.clone(), v.cloned())) - .collect(), - }); - } - - Ok(NewDeploymentUnits { base: &mut self.deployment_units, units }) - } - - /// Convenience method for `self.start_add_deployment_unit(..).insert()` for a - /// single deployment unit. - pub fn insert( - &mut self, - data: DeploymentUnitData, - ) -> Result<(), DuplicateDeploymentUnitError> { - self.start_insert(data)?.commit(); - Ok(()) - } - - pub fn finish_map(self) -> DeploymentUnitMap { - self.deployment_units - } - - pub fn finish_units(self) -> ArtifactDeploymentUnits { - ArtifactDeploymentUnits::Composite { - deployment_units: self.deployment_units, - } - } -} - -/// Information about new deployment units for a [`DeploymentUnitMapBuilder`]. -/// -/// This serves as a way to make new deployment units ready to be bulk-inserted -/// into the builder. -#[must_use = "NewDeploymentUnits must be committed into the builder"] -pub struct NewDeploymentUnits<'a> { - base: &'a mut DeploymentUnitMap, - units: DeploymentUnitMap, -} - -impl<'a> NewDeploymentUnits<'a> { - fn new_single( - base: &'a mut DeploymentUnitMap, - data: DeploymentUnitData, - ) -> Self { - let mut units = DeploymentUnitMap::new(); - units.insert(data.hash_id(), data); - Self { base, units } - } - - /// Inserts the deployment unit data into the builder. - pub fn commit(self) { - self.base.extend(self.units); - } -} - -#[derive(Clone, Debug, Error)] -pub struct DuplicateDeploymentUnitError { - /// The scope within which duplicates were found. - pub scope: DeploymentUnitScope, - - /// Duplicates found while inserting deployment unit data. - /// - /// For `Leaf`, `before` is the existing data and - /// `after` is the new data. - pub duplicates: BTreeMap>, -} - -impl DuplicateDeploymentUnitError { - // Note: existing and new should have the same hash_id. - fn new_single( - scope: DeploymentUnitScope, - existing: DeploymentUnitData, - new: DeploymentUnitData, - ) -> Self { - let mut duplicates = BTreeMap::new(); - let hash_id = existing.hash_id(); - duplicates.insert(hash_id, Leaf { before: existing, after: new }); - Self { scope, duplicates } - } -} - -impl fmt::Display for DuplicateDeploymentUnitError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - // For a single deployment unit, we can simply display the artifact kind and hash. - if self.duplicates.len() == 1 { - let (hash_id, data) = self.duplicates.first_key_value().unwrap(); - // XXX: should `ArtifactHashId` have a `Display` impl, or maybe a - // `.display()` or `.display_human()` method? - write!( - f, - "a deployment unit with the same kind and hash already exists in this {}:\n\ - {hash_id} (existing {}; new {})", - self.scope, data.before, data.after, - ) - } else { - writeln!( - f, - "{} deployment units with the same kind and hash already exist in this {}:", - self.duplicates.len(), - self.scope, - )?; - for (hash_id, data) in &self.duplicates { - writeln!( - f, - " - for {hash_id} (existing: {}; new: {})", - data.before, data.after, - )?; - } - - Ok(()) - } - } -} diff --git a/lib/src/assemble/manifest.rs b/lib/src/assemble/manifest.rs deleted file mode 100644 index 8fb425f..0000000 --- a/lib/src/assemble/manifest.rs +++ /dev/null @@ -1,1019 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use std::collections::{BTreeMap, BTreeSet}; -use std::io::BufReader; -use std::str::FromStr; -use std::{fmt, fs}; - -use anyhow::{Context, Result, bail, ensure}; -use camino::{Utf8Path, Utf8PathBuf}; -use itertools::Itertools; -use parse_size::parse_size; -use semver::Version; -use serde::{Deserialize, Serialize}; -use tufaceous_artifact::{ArtifactKind, ArtifactVersion, KnownArtifactKind}; - -use crate::assemble::{DeploymentUnitData, DeploymentUnitScope}; -use crate::{ - ArtifactSource, COSMO_HOST_PHASE_1_FILE_NAME, - CompositeControlPlaneArchiveBuilder, CompositeEntry, - CompositeHostArchiveBuilder, CompositeRotArchiveBuilder, - GIMLET_HOST_PHASE_1_FILE_NAME, HOST_PHASE_2_FILE_NAME, HostPhaseImages, - MtimeSource, ROT_ARCHIVE_A_FILE_NAME, ROT_ARCHIVE_B_FILE_NAME, - make_filler_text, make_filler_text_with_seed, -}; - -use super::{ArtifactDeploymentUnits, DeploymentUnitMapBuilder}; - -static FAKE_MANIFEST_TOML: &str = - include_str!("../../../bin/manifests/fake.toml"); - -/// A list of components in a TUF repo representing a single update. -#[derive(Clone, Debug)] -pub struct ArtifactManifest { - pub system_version: Version, - pub artifacts: BTreeMap>, -} - -impl ArtifactManifest { - /// Reads a manifest in from a TOML file. - pub fn from_path(path: &Utf8Path) -> Result { - let input = fs_err::read_to_string(path)?; - let base_dir = path - .parent() - .with_context(|| format!("path `{path}` did not have a parent"))?; - Self::from_str(base_dir, &input) - } - - /// Deserializes a manifest from an input string. - pub fn from_str(base_dir: &Utf8Path, input: &str) -> Result { - let manifest = DeserializedManifest::from_str(input)?; - Self::from_deserialized(base_dir, manifest) - } - - /// Creates a manifest from a [`DeserializedManifest`]. - pub fn from_deserialized( - base_dir: &Utf8Path, - manifest: DeserializedManifest, - ) -> Result { - // Replace all paths in the deserialized manifest with absolute ones, - // and do some processing to support flexible manifests: - // - // 1. assemble any composite artifacts from their pieces - // 2. replace any "fake" artifacts with in-memory buffers - // - // Currently both of those transformations produce - // `ArtifactSource::Memory(_)` variants (i.e., composite and fake - // artifacts all sit in-memory until we're done with the manifest), - // which puts some limits on how large the inputs to the manifest can - // practically be. If this becomes onerous, we could instead write the - // transformed artifacts to temporary files. - // - // We do some additional error checking here to make sure the - // `CompositeZZZ` variants are only used with their corresponding - // `KnownArtifactKind`s. It would be nicer to enforce this more - // statically and let serde do these checks, but that seems relatively - // tricky in comparison to these checks. - - Ok(ArtifactManifest { - system_version: manifest.system_version, - artifacts: manifest - .artifacts - .into_iter() - .map(|(kind, entries)| { - Self::parse_deserialized_entries(base_dir, kind, entries) - }) - .collect::>()?, - }) - } - - fn parse_deserialized_entries( - base_dir: &Utf8Path, - kind: KnownArtifactKind, - entries: Vec, - ) -> Result<(KnownArtifactKind, Vec)> { - let entries = entries - .into_iter() - .map(|artifact_data| { - let (source, deployment_units) = match artifact_data.source { - DeserializedArtifactSource::File { path } => { - let path = base_dir.join(&path); - - // Host images are actually composite artifacts, and we - // need to treat them that way for installinator. - let deployment_units = - if kind == KnownArtifactKind::Host { - let file = - fs::File::open(&path).with_context(|| { - format!( - "error opening host image at `{path}`" - ) - })?; - let reader = BufReader::new(file); - let images = HostPhaseImages::extract(reader)?; - - let mut data_builder = - DeploymentUnitMapBuilder::new( - DeploymentUnitScope::Artifact { - composite_kind: kind, - }, - ); - data_builder - .insert(DeploymentUnitData { - name: COSMO_HOST_PHASE_1_FILE_NAME - .to_owned(), - version: artifact_data.version.clone(), - kind: ArtifactKind::COSMO_HOST_PHASE_1, - hash: images.cosmo_phase_1_hash(), - }) - .expect("unique kind"); - data_builder - .insert(DeploymentUnitData { - name: GIMLET_HOST_PHASE_1_FILE_NAME - .to_owned(), - version: artifact_data.version.clone(), - kind: ArtifactKind::GIMLET_HOST_PHASE_1, - hash: images.gimlet_phase_1_hash(), - }) - .expect("unique kind"); - - data_builder - .insert(DeploymentUnitData { - name: HOST_PHASE_2_FILE_NAME.to_owned(), - version: artifact_data.version.clone(), - kind: ArtifactKind::HOST_PHASE_2, - hash: images.phase_2_hash(), - }) - .expect("unique kind"); - - data_builder.finish_units() - } else { - // It would be nice to extract other kinds of - // composite artifacts here, but (a) we don't - // have a need for that in this case and (b) the - // code for that currently lives in omicron's - // update-common. - ArtifactDeploymentUnits::SingleUnit - }; - - (ArtifactSource::File(path), deployment_units) - } - DeserializedArtifactSource::Fake { size, data_version } => { - // This test-only environment variable is used to - // simulate two artifacts with different - // name/version/kind but the same hash. - let data_version = data_version - .as_ref() - .unwrap_or(&artifact_data.version); - let fake_data = - FakeDataAttributes::new(kind, data_version) - .make_data(size as usize); - ( - ArtifactSource::Memory(fake_data.into()), - ArtifactDeploymentUnits::SingleUnit, - ) - } - DeserializedArtifactSource::CompositeHost { - gimlet_phase_1, - cosmo_phase_1, - phase_2, - } => { - ensure!( - matches!( - kind, - KnownArtifactKind::Host - | KnownArtifactKind::Trampoline - ), - "`composite_host` source cannot be used with \ - artifact kind {kind:?}" - ); - - let mtime_source = if gimlet_phase_1.is_fake() - && cosmo_phase_1.is_fake() - && phase_2.is_fake() - { - // Ensure stability of fake artifacts. - MtimeSource::Zero - } else { - MtimeSource::Now - }; - - let mut builder = CompositeHostArchiveBuilder::new( - Vec::new(), - mtime_source, - )?; - let cosmo_phase_1_hash = cosmo_phase_1.with_entry( - FakeDataAttributes::new_with_seed( - kind, - &artifact_data.version, - "cosmo".to_string(), - ), - |entry| builder.append_cosmo_phase_1(entry), - )?; - let gimlet_phase_1_hash = gimlet_phase_1.with_entry( - FakeDataAttributes::new_with_seed( - kind, - &artifact_data.version, - "gimlet".to_string(), - ), - |entry| builder.append_gimlet_phase_1(entry), - )?; - - let phase_2_hash = phase_2.with_entry( - FakeDataAttributes::new( - kind, - &artifact_data.version, - ), - |entry| builder.append_phase_2(entry), - )?; - let source = - ArtifactSource::Memory(builder.finish()?.into()); - - let mut data_builder = DeploymentUnitMapBuilder::new( - DeploymentUnitScope::Artifact { - composite_kind: kind, - }, - ); - data_builder - .insert(DeploymentUnitData { - name: GIMLET_HOST_PHASE_1_FILE_NAME.to_owned(), - version: artifact_data.version.clone(), - kind: ArtifactKind::GIMLET_HOST_PHASE_1, - hash: gimlet_phase_1_hash, - }) - .expect("unique kind"); - - data_builder - .insert(DeploymentUnitData { - name: COSMO_HOST_PHASE_1_FILE_NAME.to_owned(), - version: artifact_data.version.clone(), - kind: ArtifactKind::COSMO_HOST_PHASE_1, - hash: cosmo_phase_1_hash, - }) - .expect("unique kind"); - data_builder - .insert(DeploymentUnitData { - name: HOST_PHASE_2_FILE_NAME.to_owned(), - version: artifact_data.version.clone(), - kind: ArtifactKind::HOST_PHASE_2, - hash: phase_2_hash, - }) - .expect("unique kind"); - - (source, data_builder.finish_units()) - } - DeserializedArtifactSource::CompositeRot { - archive_a, - archive_b, - } => { - let (a_kind, b_kind) = - kind.rot_a_and_b_kinds().context( - "`composite_rot` source cannot be used with \ - non-RoT artifact kind", - )?; - - let mtime_source = - if archive_a.is_fake() && archive_b.is_fake() { - // Ensure stability of fake artifacts. - MtimeSource::Zero - } else { - MtimeSource::Now - }; - - let mut builder = CompositeRotArchiveBuilder::new( - Vec::new(), - mtime_source, - )?; - let archive_a_hash = archive_a.with_entry( - FakeDataAttributes::new( - kind, - &artifact_data.version, - ), - |entry| builder.append_archive_a(entry), - )?; - let archive_b_hash = archive_b.with_entry( - FakeDataAttributes::new( - kind, - &artifact_data.version, - ), - |entry| builder.append_archive_b(entry), - )?; - - let mut data_builder = DeploymentUnitMapBuilder::new( - DeploymentUnitScope::Artifact { - composite_kind: kind, - }, - ); - data_builder - .insert(DeploymentUnitData { - name: ROT_ARCHIVE_A_FILE_NAME.to_owned(), - version: artifact_data.version.clone(), - kind: a_kind, - hash: archive_a_hash, - }) - .expect("unique kind in empty map"); - data_builder - .insert(DeploymentUnitData { - name: ROT_ARCHIVE_B_FILE_NAME.to_owned(), - version: artifact_data.version.clone(), - kind: b_kind, - hash: archive_b_hash, - }) - .expect("unique kind in empty map"); - - ( - ArtifactSource::Memory(builder.finish()?.into()), - data_builder.finish_units(), - ) - } - DeserializedArtifactSource::CompositeControlPlane { - zones, - } => { - ensure!( - kind == KnownArtifactKind::ControlPlane, - "`composite_control_plane` source cannot be \ - used with artifact kind {kind:?}" - ); - - // Ensure stability of fake artifacts. - let mtime_source = if zones.iter().all(|z| z.is_fake()) - { - MtimeSource::Zero - } else { - MtimeSource::Now - }; - - let data = Vec::new(); - let mut builder = - CompositeControlPlaneArchiveBuilder::new( - data, - mtime_source, - )?; - - let zone_kind = - ArtifactKind::from(KnownArtifactKind::Zone); - let mut data_builder = DeploymentUnitMapBuilder::new( - DeploymentUnitScope::Artifact { - composite_kind: kind, - }, - ); - - for zone in zones { - let (hash, name) = zone.with_name_and_entry( - &artifact_data.version, - |name, entry| builder.append_zone(name, entry), - )?; - data_builder.insert(DeploymentUnitData { - name: name.to_owned(), - version: artifact_data.version.clone(), - kind: zone_kind.clone(), - hash, - })?; - } - ( - ArtifactSource::Memory(builder.finish()?.into()), - data_builder.finish_units(), - ) - } - }; - let data = ArtifactData { - name: artifact_data.name, - version: artifact_data.version, - source, - deployment_units, - }; - Ok(data) - }) - .collect::>()?; - Ok((kind, entries)) - } - - /// Returns a fake manifest. Useful for testing. - pub fn new_fake() -> Self { - // The base directory doesn't matter for fake manifests. - Self::from_str(".".into(), FAKE_MANIFEST_TOML) - .expect("the fake manifest is a valid manifest") - } - - /// Checks that all versions are valid semver. - pub fn verify_all_semver(&self) -> Result<()> { - let mut non_semver = Vec::new(); - for artifacts in self.artifacts.values() { - for artifact in artifacts { - if artifact.version.as_str().parse::().is_err() { - non_semver.push(artifact); - } - } - } - - if !non_semver.is_empty() { - bail!( - "non-semver versions found: {}", - non_semver.iter().map(|d| d.display()).join(", "), - ); - } - Ok(()) - } - - /// Checks that all expected artifacts are present, returning an error with - /// details if any artifacts are missing. - pub fn verify_all_present(&self) -> Result<()> { - let all_artifacts: BTreeSet<_> = KnownArtifactKind::iter() - .filter(|k| { - // Installinator documents are generated by tufaceous and should - // not be part of the manifest. - !matches!( - k, - KnownArtifactKind::Zone - | KnownArtifactKind::InstallinatorDocument - | KnownArtifactKind::MeasurementCorpus - ) - }) - .collect(); - let present_artifacts: BTreeSet<_> = - self.artifacts.keys().copied().collect(); - - let missing = &all_artifacts - &present_artifacts; - if !missing.is_empty() { - bail!( - "manifest has missing artifacts: {}", - itertools::join(missing, ", ") - ); - } - - Ok(()) - } -} - -#[derive(Debug)] -struct FakeDataAttributes<'a> { - kind: KnownArtifactKind, - version: &'a ArtifactVersion, - seed: String, -} - -impl<'a> FakeDataAttributes<'a> { - fn new(kind: KnownArtifactKind, version: &'a ArtifactVersion) -> Self { - Self { kind, version, seed: "".to_string() } - } - - fn new_with_seed( - kind: KnownArtifactKind, - version: &'a ArtifactVersion, - seed: String, - ) -> Self { - Self { kind, version, seed } - } - - fn make_data(&self, size: usize) -> Vec { - use hubtools::{CabooseBuilder, HubrisArchiveBuilder}; - - let board = match self.kind { - KnownArtifactKind::Host | KnownArtifactKind::Trampoline => { - // need to use the extra seed to be able to generate different gimlet vs - // cosmo images - return make_filler_text_with_seed( - &self.kind.to_string(), - self.version, - size, - &self.seed, - ); - } - // non-Hubris artifacts: just make fake data - KnownArtifactKind::ControlPlane | KnownArtifactKind::Zone => { - return make_filler_text( - &self.kind.to_string(), - self.version, - size, - ); - } - KnownArtifactKind::InstallinatorDocument => { - panic!( - "fake manifest should not have an installinator document" - ); - } - KnownArtifactKind::MeasurementCorpus => { - return make_filler_text( - &self.kind.to_string(), - self.version, - size, - ); - } - - // In production, all the bootloaders and RoTs claim to have the - // same board (currently: `oxide-rot-1`). Let's do that here too. - KnownArtifactKind::GimletRotBootloader - | KnownArtifactKind::PscRotBootloader - | KnownArtifactKind::SwitchRotBootloader - | KnownArtifactKind::GimletRot - | KnownArtifactKind::PscRot - | KnownArtifactKind::SwitchRot => "SimRot", - - // SP artifacts: build a fake archive (SimGimletSp and - // SimGimletRot are used by sp-sim) - KnownArtifactKind::GimletSp => "SimGimletSp", - KnownArtifactKind::SwitchSp => "SimSidecarSp", - KnownArtifactKind::PscSp => "SimPscSp", - }; - - let caboose = { - // We use a fake git commit that contains `self.kind` to ensure that - // the artifacts we produce are distinct for each `kind`, even if - // all the other caboose fields are identical. - let mut builder = CabooseBuilder::default() - .git_commit(format!("this-is-a-fake-{}", self.kind)) - .board(board) - .name(board) - .version(self.version.to_string()); - - if let Some(sign) = self.kind.fake_artifact_hubris_sign() { - builder = builder.sign(sign); - } - - builder.build() - }; - - let mut builder = HubrisArchiveBuilder::with_fake_image(); - builder.write_caboose(caboose.as_slice()).unwrap(); - builder.build_to_vec().unwrap() - } -} - -/// Information about an individual artifact. -#[derive(Clone, Debug)] -pub struct ArtifactData { - pub name: String, - pub version: ArtifactVersion, - pub source: ArtifactSource, - pub deployment_units: ArtifactDeploymentUnits, -} - -impl ArtifactData { - /// Returns a displayer for the name and version of the artifact. - pub fn display(&self) -> ArtifactDataDisplay<'_> { - ArtifactDataDisplay { data: self } - } -} - -pub struct ArtifactDataDisplay<'a> { - data: &'a ArtifactData, -} - -impl fmt::Display for ArtifactDataDisplay<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{} ({})", self.data.name, self.data.version) - } -} - -/// Deserializable version of [`ArtifactManifest`]. -/// -/// Since manifests require a base directory to be deserialized properly, -/// we don't expose the `Deserialize` impl on `ArtifactManifest, forcing -/// consumers to go through [`ArtifactManifest::from_path`] or -/// [`ArtifactManifest::from_str`]. -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] -#[serde(rename_all = "snake_case")] -pub struct DeserializedManifest { - pub system_version: Version, - #[serde(rename = "artifact")] - pub artifacts: BTreeMap>, -} - -impl DeserializedManifest { - pub fn from_path(path: &Utf8Path) -> Result { - let input = fs_err::read_to_string(path)?; - Self::from_str(&input).with_context(|| { - format!("error deserializing manifest from {path}") - }) - } - - pub fn to_toml(&self) -> Result { - toml::to_string(self).context("error serializing manifest to TOML") - } - - /// For fake manifests, applies a set of changes to them. - /// - /// Intended for testing. - pub fn apply_tweaks(&mut self, tweaks: &[ManifestTweak]) -> Result<()> { - for tweak in tweaks { - match tweak { - ManifestTweak::SystemVersion(version) => { - self.system_version = version.clone(); - } - ManifestTweak::ArtifactVersion { kind, version } => { - let entries = - self.artifacts.get_mut(kind).with_context(|| { - format!( - "manifest does not have artifact kind \ - {kind}", - ) - })?; - for entry in entries { - entry.version = version.clone(); - } - } - ManifestTweak::ArtifactSize { kind, size_delta } => { - let entries = - self.artifacts.get_mut(kind).with_context(|| { - format!( - "manifest does not have artifact kind \ - {kind}", - ) - })?; - - for entry in entries { - entry.source.apply_size_delta(*size_delta)?; - } - } - ManifestTweak::ArtifactDataVersion { kind, data_version } => { - let entries = - self.artifacts.get_mut(kind).with_context(|| { - format!( - "manifest does not have artifact kind \ - {kind}", - ) - })?; - - for entry in entries { - entry - .source - .apply_data_version(data_version.as_ref())?; - } - } - } - } - - Ok(()) - } - - /// Returns the fake manifest. - pub fn fake() -> Self { - Self::from_str(FAKE_MANIFEST_TOML).unwrap() - } - - /// Returns a version of the fake manifest with a set of changes applied. - /// - /// This is primarily intended for testing. - pub fn tweaked_fake(tweaks: &[ManifestTweak]) -> Self { - let mut manifest = Self::fake(); - manifest - .apply_tweaks(tweaks) - .expect("builtin fake manifest should accept all tweaks"); - - manifest - } -} - -impl FromStr for DeserializedManifest { - type Err = anyhow::Error; - - fn from_str(input: &str) -> Result { - let de = toml::Deserializer::new(input); - serde_path_to_error::deserialize(de) - .context("error deserializing manifest") - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] -#[serde(rename_all = "snake_case")] -pub struct DeserializedArtifactData { - pub name: String, - pub version: ArtifactVersion, - pub source: DeserializedArtifactSource, -} - -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] -#[serde(tag = "kind", rename_all = "kebab-case")] -pub enum DeserializedArtifactSource { - File { - path: Utf8PathBuf, - }, - Fake { - #[serde(deserialize_with = "deserialize_byte_size")] - size: u64, - /// The internal version to use while constructing the fake artifact - /// data. - /// - /// If not set, the artifact's version is used. - #[serde(default)] - data_version: Option, - }, - CompositeHost { - gimlet_phase_1: DeserializedFileArtifactSource, - cosmo_phase_1: DeserializedFileArtifactSource, - phase_2: DeserializedFileArtifactSource, - }, - CompositeRot { - archive_a: DeserializedFileArtifactSource, - archive_b: DeserializedFileArtifactSource, - }, - CompositeControlPlane { - zones: Vec, - }, -} - -impl DeserializedArtifactSource { - fn apply_size_delta(&mut self, size_delta: i64) -> Result<()> { - match self { - DeserializedArtifactSource::File { .. } => { - bail!("cannot apply size delta to `file` source") - } - DeserializedArtifactSource::Fake { size, data_version: _ } => { - *size = (*size).saturating_add_signed(size_delta); - Ok(()) - } - DeserializedArtifactSource::CompositeHost { - gimlet_phase_1, - cosmo_phase_1, - phase_2, - } => { - gimlet_phase_1.apply_size_delta(size_delta)?; - cosmo_phase_1.apply_size_delta(size_delta)?; - phase_2.apply_size_delta(size_delta)?; - Ok(()) - } - DeserializedArtifactSource::CompositeRot { - archive_a, - archive_b, - } => { - archive_a.apply_size_delta(size_delta)?; - archive_b.apply_size_delta(size_delta)?; - Ok(()) - } - DeserializedArtifactSource::CompositeControlPlane { zones } => { - for zone in zones { - zone.apply_size_delta(size_delta)?; - } - Ok(()) - } - } - } - - fn apply_data_version( - &mut self, - new_data_version: Option<&ArtifactVersion>, - ) -> Result<()> { - match self { - DeserializedArtifactSource::File { .. } => { - bail!("cannot apply data version to `file` source") - } - DeserializedArtifactSource::Fake { data_version, .. } => { - *data_version = new_data_version.cloned(); - Ok(()) - } - DeserializedArtifactSource::CompositeHost { .. } => { - bail!( - "cannot yet apply data version to `composite_host` source" - ) - } - DeserializedArtifactSource::CompositeRot { .. } => { - bail!("cannot yet apply data version to `composite_rot` source") - } - DeserializedArtifactSource::CompositeControlPlane { .. } => { - bail!( - "cannot yet apply data version to `composite_control_plane` source" - ) - } - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum DeserializedFileArtifactSource { - File { - path: Utf8PathBuf, - }, - Fake { - #[serde(deserialize_with = "deserialize_byte_size")] - size: u64, - }, -} - -impl DeserializedFileArtifactSource { - fn is_fake(&self) -> bool { - matches!(self, DeserializedFileArtifactSource::Fake { .. }) - } - - fn with_entry(&self, fake_attr: FakeDataAttributes, f: F) -> Result - where - F: FnOnce(CompositeEntry<'_>) -> Result, - { - let (data, mtime_source) = match self { - DeserializedFileArtifactSource::File { path } => { - let data = std::fs::read(path) - .with_context(|| format!("failed to read {path}"))?; - // For now, always use the current time as the source. (Maybe - // change this to use the mtime on disk in the future?) - (data, MtimeSource::Now) - } - DeserializedFileArtifactSource::Fake { size } => { - (fake_attr.make_data(*size as usize), MtimeSource::Zero) - } - }; - let entry = CompositeEntry { data: &data, mtime_source }; - f(entry) - } - - fn apply_size_delta(&mut self, size_delta: i64) -> Result<()> { - match self { - DeserializedFileArtifactSource::File { .. } => { - bail!("cannot apply size delta to `file` source") - } - DeserializedFileArtifactSource::Fake { size } => { - *size = (*size).saturating_add_signed(size_delta); - Ok(()) - } - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum DeserializedControlPlaneZoneSource { - File { - path: Utf8PathBuf, - #[serde(skip_serializing_if = "Option::is_none")] - file_name: Option, - }, - #[serde(rename_all = "snake_case")] - Fake { - artifact_name: String, - file_name: String, - #[serde(deserialize_with = "deserialize_byte_size")] - size: u64, - }, -} - -impl DeserializedControlPlaneZoneSource { - fn is_fake(&self) -> bool { - matches!(self, DeserializedControlPlaneZoneSource::Fake { .. }) - } - - fn with_name_and_entry( - &self, - version: &ArtifactVersion, - f: F, - ) -> Result - where - F: FnOnce(&str, CompositeEntry<'_>) -> Result, - { - let (name, data, mtime_source) = match self { - DeserializedControlPlaneZoneSource::File { path, file_name } => { - let data = std::fs::read(path) - .with_context(|| format!("failed to read {path}"))?; - let name = file_name - .as_deref() - .or_else(|| path.file_name()) - .with_context(|| { - format!("zone path missing file name: {path}") - })?; - // For now, always use the current time as the source. (Maybe - // change this to use the mtime on disk in the future?) - (name.to_owned(), data, MtimeSource::Now) - } - DeserializedControlPlaneZoneSource::Fake { - artifact_name, - file_name, - size, - } => { - use flate2::{Compression, write::GzEncoder}; - use tufaceous_brand_metadata::{ - ArchiveType, LayerInfo, Metadata, - }; - - let mut tar = tar::Builder::new(GzEncoder::new( - Vec::new(), - Compression::fast(), - )); - - let metadata = Metadata::new(ArchiveType::Layer(LayerInfo { - pkg: artifact_name.clone(), - version: version.clone(), - })); - metadata.append_to_tar(&mut tar, 0)?; - - let mut h = tar::Header::new_ustar(); - h.set_entry_type(tar::EntryType::Regular); - h.set_path("fake")?; - h.set_mode(0o444); - h.set_size(*size); - h.set_mtime(0); - h.set_cksum(); - tar.append( - &h, - make_filler_text(artifact_name, version, *size as usize) - .as_slice(), - )?; - - let data = tar.into_inner()?.finish()?; - (file_name.clone(), data, MtimeSource::Zero) - } - }; - let entry = CompositeEntry { data: &data, mtime_source }; - f(&name, entry) - } - - fn apply_size_delta(&mut self, size_delta: i64) -> Result<()> { - match self { - DeserializedControlPlaneZoneSource::File { .. } => { - bail!("cannot apply size delta to `file` source") - } - DeserializedControlPlaneZoneSource::Fake { size, .. } => { - (*size) = (*size).saturating_add_signed(size_delta); - Ok(()) - } - } - } -} -/// A change to apply to a manifest. -#[derive(Clone, Debug)] -pub enum ManifestTweak { - /// Update the system version. - SystemVersion(Version), - - /// Update the versions for this artifact. - ArtifactVersion { kind: KnownArtifactKind, version: ArtifactVersion }, - - /// Update the size of this fake artifact. - ArtifactSize { kind: KnownArtifactKind, size_delta: i64 }, - - /// Update the data version of this fake artifact. - /// - /// This version is typically the same as the artifact version, but it can - /// be changed for testing purposes. - ArtifactDataVersion { - kind: KnownArtifactKind, - /// Setting this to `None` resets the data version to the artifact - /// version. - data_version: Option, - }, -} - -fn deserialize_byte_size<'de, D>(deserializer: D) -> Result -where - D: serde::Deserializer<'de>, -{ - // Attempt to deserialize the size as either a string or an integer. - - struct Visitor; - - impl serde::de::Visitor<'_> for Visitor { - type Value = u64; - - fn expecting( - &self, - formatter: &mut std::fmt::Formatter, - ) -> std::fmt::Result { - formatter - .write_str("a string representing a byte size or an integer") - } - - fn visit_str(self, value: &str) -> Result - where - E: serde::de::Error, - { - parse_size(value).map_err(|_| { - serde::de::Error::invalid_value( - serde::de::Unexpected::Str(value), - &self, - ) - }) - } - - // TOML uses i64, not u64 - fn visit_i64(self, value: i64) -> Result - where - E: serde::de::Error, - { - Ok(value as u64) - } - - fn visit_u64(self, value: u64) -> Result - where - E: serde::de::Error, - { - Ok(value) - } - } - - deserializer.deserialize_any(Visitor) -} - -#[cfg(test)] -mod tests { - use super::*; - - // Ensure that the fake manifest roundtrips after serialization and - // deserialization. - #[test] - fn fake_roundtrip() { - let manifest = DeserializedManifest::fake(); - let toml = toml::to_string(&manifest).unwrap(); - let deserialized = DeserializedManifest::from_str(&toml) - .expect("fake manifest is a valid manifest"); - assert_eq!(manifest, deserialized); - } -} diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs new file mode 100644 index 0000000..e2157cf --- /dev/null +++ b/lib/src/edit/editor.rs @@ -0,0 +1,498 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::BTreeMap; +use std::collections::HashMap; +use std::collections::HashSet; + +use camino::Utf8Path; +use camino::Utf8PathBuf; +use futures_util::FutureExt; +use futures_util::TryFutureExt; +use futures_util::TryStreamExt; +use hubtools::Caboose; +use semver::Version; +use tokio::fs::File; +use tokio::task::JoinSet; +use tufaceous_artifact::ArtifactVersion; +use tufaceous_artifact::InstallinatorArtifact; +use tufaceous_artifact::InstallinatorDocument; +use tufaceous_artifact::KnownArtifactTags; +use tufaceous_artifact::OsBoard; +use tufaceous_artifact::OsVariant; +use tufaceous_artifact::RotSlot; +use tufaceous_artifact::hubris::ReadCabooseError; + +use crate::Repository; +use crate::edit::UnsignedRepository; +use crate::edit::source::BytesSource; +use crate::edit::source::FakeSource; +use crate::edit::source::FileSource; +use crate::edit::source::RepositorySource; +use crate::edit::source::Target; +use crate::edit::source::TargetSource; +use crate::error::Error; +use crate::error::ErrorKind; +use crate::schema::ArtifactSchema; +use crate::schema::ArtifactsSchema; + +const KIB: usize = 1024; +const MIB: usize = 1024 * KIB; + +#[derive(Debug)] +pub struct RepositoryEditor<'a> { + system_version: Version, + targets: HashMap>>, + artifacts: HashMap>, + generate_installinator_document: bool, +} + +impl<'a> RepositoryEditor<'a> { + pub fn new(system_version: Version) -> Self { + Self { + system_version, + targets: HashMap::new(), + artifacts: HashMap::new(), + generate_installinator_document: false, + } + } + + pub fn system_version(self, system_version: Version) -> Self { + Self { system_version, ..self } + } + + pub async fn add_artifact( + mut self, + target_name: String, + version: ArtifactVersion, + tags: KnownArtifactTags, + path: impl Into, + ) -> Result { + self.targets + .entry(target_name.clone()) + .or_default() + .push(TargetSource::File(FileSource::open(path).await?)); + self.artifacts.entry(target_name.clone()).or_default().insert( + ArtifactSchema { target_name, version, tags: tags.to_tags() }, + ); + Ok(self) + } + + pub async fn add_extra_target( + mut self, + target_name: String, + path: impl Into, + ) -> Result { + self.targets + .entry(target_name.clone()) + .or_default() + .push(TargetSource::File(FileSource::open(path).await?)); + Ok(self) + } + + pub fn add_fake_artifact( + mut self, + target_name: String, + version: ArtifactVersion, + tags: KnownArtifactTags, + length: usize, + ) -> Self { + let prefix = format!("{target_name}\n{version}\n{tags:?}\n"); + self.targets + .entry(target_name.clone()) + .or_default() + .push(TargetSource::Fake(FakeSource::new(prefix, length))); + self.artifacts.entry(target_name.clone()).or_default().insert( + ArtifactSchema { target_name, version, tags: tags.to_tags() }, + ); + self + } + + pub fn fake(system_version: Version) -> Result { + let version = ArtifactVersion::new(system_version.to_string())?; + let mut editor = + Self::new(system_version).generate_installinator_document(); + + for hash in ["123abc", "def456"] { + editor = editor.add_fake_artifact( + format!("measurement-corpus-{hash}.corim"), + version.clone(), + KnownArtifactTags::MeasurementCorpus {}, + 4 * KIB, + ); + } + for variant in [OsVariant::Host, OsVariant::Recovery] { + for board in [OsBoard::Gimlet, OsBoard::Cosmo] { + editor = editor.add_fake_artifact( + format!("{variant}-os/{board}.rom"), + version.clone(), + KnownArtifactTags::OsPhase1 { variant, board }, + MIB, + ); + } + editor = editor.add_fake_artifact( + format!("{variant}-os/zfs.img"), + version.clone(), + KnownArtifactTags::OsPhase2 { variant }, + 4 * MIB, + ); + } + for slot in [RotSlot::A, RotSlot::B] { + editor = editor.add_fake_artifact( + format!("rot-fake-slot{slot}.zip"), + version.clone(), + KnownArtifactTags::Rot { + board: "fake".into(), + sign: None, + slot, + }, + 256 * KIB, + ); + } + editor = editor.add_fake_artifact( + "rot-bootloader-fake.zip".into(), + version.clone(), + KnownArtifactTags::RotBootloader { + board: "fake".into(), + sign: None, + }, + 64 * KIB, + ); + editor = editor.add_fake_artifact( + "sp-fake.zip".into(), + version.clone(), + KnownArtifactTags::Sp { board: "fake".into() }, + MIB, + ); + for name in ["zone1", "zone2"] { + editor = editor.add_fake_artifact( + format!("zones/{name}.tar.gz"), + version.clone(), + KnownArtifactTags::Zone { name: name.into() }, + MIB, + ); + } + + Ok(editor) + } + + pub fn from_repo(repo: &'a mut Repository) -> Result { + Self::new(repo.system_version().clone()).import_repo(repo) + } + + pub fn import_repo( + mut self, + repo: &'a mut Repository, + ) -> Result { + if repo.is_v1() { + return Err(ErrorKind::ImportV1Repo.into()); + } + + for (target_name, target) in repo.targets() { + if target_name.raw() == ArtifactsSchema::TARGET_NAME { + continue; + } + self.targets.entry(target_name.raw().to_owned()).or_default().push( + TargetSource::Repository(RepositorySource { + repo, + target_name: target_name.raw().to_owned(), + length: target.length, + sha256: target.hashes.sha256.to_vec(), + }), + ); + } + for artifact in repo.artifacts() { + self.artifacts + .entry(artifact.target_name.clone()) + .or_default() + .insert(ArtifactSchema { + target_name: artifact.target_name.clone(), + version: artifact.version.clone(), + tags: artifact.tags.clone(), + }); + } + Ok(self) + } + + pub fn generate_installinator_document(self) -> Self { + Self { generate_installinator_document: true, ..self } + } + + pub async fn finish(self) -> Result, Error> { + // Un-nest `self.artifacts`, returning an error if we have multiple + // artifact definitions for a single target name. + let mut artifacts = self + .artifacts + .into_iter() + .filter_map(|(target_name, entries)| { + Some(if entries.len() > 1 { + Err(ErrorKind::TargetNameCollision { target_name }.into()) + } else { + Ok((target_name, entries.into_iter().next()?)) + }) + }) + .collect::, Error>>()?; + + // Collect all the sha256 hashes and lengths for each source. For file + // and fake sources, we want to calculate the hashes in parallel, so + // we spawn their calculation tasks on a JoinSet. Sources from borrowed + // repositories can't be moved into a task, but we already know their + // hash. + let mut all_targets = Vec::new(); + let mut tasks = JoinSet::new(); + for (target_name, sources) in self.targets { + for source in sources { + let target_name = target_name.clone(); + match source { + TargetSource::Bytes(source) => { + all_targets.push((target_name, source.into_target())); + } + TargetSource::File(source) => { + let future = source + .into_target() + .map_ok(|target| (target_name, target)); + tasks.spawn(future); + } + TargetSource::Repository(source) => { + all_targets.push((target_name, source.into_target())); + } + TargetSource::Fake(source) => { + let future = source + .into_target() + .map(|target| Ok((target_name, target))); + tasks.spawn(future); + } + } + } + } + while let Some(result) = tasks.join_next().await { + all_targets.push(result??); + } + // Fold the targets back into a map, checking that duplicates have the + // same sha256 hash and length. + let mut targets = TargetMap::default(); + for (target_name, target) in all_targets { + targets.insert(target_name, target)?; + } + + if self.generate_installinator_document { + let version = + ArtifactVersion::new(self.system_version.to_string())?; + let target_name = format!("installinator_document-{version}.json"); + let artifact = ArtifactSchema { + target_name: target_name.clone(), + version, + tags: KnownArtifactTags::InstallinatorDocument {}.to_tags(), + }; + if let Some(existing) = artifacts.get(&target_name) { + if existing != &artifact { + return Err( + ErrorKind::TargetNameCollision { target_name }.into() + ); + } + } else { + artifacts.insert(target_name.clone(), artifact); + } + + let mut document = InstallinatorDocument { artifacts: Vec::new() }; + for artifact in artifacts.values() { + if let Some(kind) = KnownArtifactTags::from_tags(&artifact.tags) + .ok() + .and_then(|tags| tags.to_installinator()) + && let Some(target) = targets.0.get(&artifact.target_name) + && let Ok(sha256) = target.sha256.as_slice().try_into() + { + document.artifacts.push(InstallinatorArtifact { + name: artifact.target_name.clone(), + kind, + sha256, + }); + } + } + let target = BytesSource::json(&document) + .map_err(ErrorKind::SerializeInstallinator)? + .into_target(); + targets.insert(target_name, target)?; + } + + let document = ArtifactsSchema { + system_version: self.system_version, + artifacts: artifacts.into_values().collect(), + }; + let target = BytesSource::json(&document) + .map_err(ErrorKind::SerializeArtifacts)? + .into_target(); + targets.insert(ArtifactsSchema::TARGET_NAME.to_owned(), target)?; + Ok(UnsignedRepository::from_targets(targets.0)) + } + + pub async fn add_os_artifacts( + mut self, + variant: OsVariant, + build_output_path: impl Into, + ) -> Result { + let base = Utf8Path::new(match variant { + OsVariant::Host => "host-os", + OsVariant::Recovery => "recovery-os", + }); + let version = ArtifactVersion::new(self.system_version.to_string())?; + let mut read_dir = crate::util::read_dir(build_output_path).await?; + while let Some(entry) = read_dir.try_next().await? { + let target_name = base.join(entry.file_name()); + let tags = match entry.file_name() { + "cosmo.rom" => KnownArtifactTags::OsPhase1 { + variant, + board: OsBoard::Cosmo, + }, + "gimlet.rom" => KnownArtifactTags::OsPhase1 { + variant, + board: OsBoard::Gimlet, + }, + "zfs.img" => KnownArtifactTags::OsPhase2 { variant }, + + _ => { + self = self + .add_extra_target(target_name.into(), entry.into_path()) + .await?; + continue; + } + }; + self = self + .add_artifact( + target_name.into(), + version.clone(), + tags, + entry.into_path(), + ) + .await?; + } + Ok(self) + } + + async fn add_hubris_image( + mut self, + target_name: String, + path: impl Into, + f: F, + ) -> Result + where + F: FnOnce(&Caboose) -> Result, + { + let path = path.into(); + let mut source = FileSource::open(path.clone()).await?; + let caboose = source.read_hubris_caboose().await?; + self.targets + .entry(target_name.clone()) + .or_default() + .push(TargetSource::File(source)); + let (tags, version) = f(&caboose) + .and_then(|tags| { + Ok((tags, tufaceous_artifact::hubris::read_version(&caboose)?)) + }) + .map_err(|source| ErrorKind::ReadCaboose { source, path })?; + let version = ArtifactVersion::new(version)?; + self.artifacts.entry(target_name.clone()).or_default().insert( + ArtifactSchema { target_name, version, tags: tags.to_tags() }, + ); + Ok(self) + } + + pub async fn add_rot_image( + self, + target_name: String, + slot: RotSlot, + path: impl Into, + ) -> Result { + self.add_hubris_image(target_name, path, |caboose| { + KnownArtifactTags::from_rot_caboose(caboose, slot) + }) + .await + } + + pub async fn add_rot_bootloader_image( + self, + target_name: String, + path: impl Into, + ) -> Result { + self.add_hubris_image( + target_name, + path, + KnownArtifactTags::from_rot_bootloader_caboose, + ) + .await + } + + pub async fn add_sp_image( + self, + target_name: String, + path: impl Into, + ) -> Result { + self.add_hubris_image( + target_name, + path, + KnownArtifactTags::from_sp_caboose, + ) + .await + } + + pub async fn add_zone_image( + mut self, + file_name: &str, + path: impl Into, + ) -> Result { + let path = path.into(); + let file = File::open(&path) + .await + .map_err(|source| ErrorKind::OpenFile { + source, + path: path.clone(), + })? + .into_std() + .await; + let (file, layer_info) = + crate::util::read_zone_layer_info(file, path.clone()).await?; + let source = FileSource::from_file(file.into(), path); + let target_name = format!("zones/{file_name}"); + self.targets + .entry(target_name.clone()) + .or_default() + .push(TargetSource::File(source)); + self.artifacts.entry(target_name.clone()).or_default().insert( + ArtifactSchema { + target_name, + version: layer_info.version, + tags: KnownArtifactTags::Zone { name: layer_info.pkg } + .to_tags(), + }, + ); + Ok(self) + } +} + +#[derive(Default)] +struct TargetMap<'a>(BTreeMap>); + +impl<'a> TargetMap<'a> { + fn insert( + &mut self, + target_name: String, + target: Target<'a>, + ) -> Result<(), Error> { + if let Some(existing) = self.0.get(&target_name) { + if existing.length == target.length + && existing.sha256 == target.sha256 + { + if existing.source.cost() <= target.source.cost() { + return Ok(()); + } + } else { + return Err( + ErrorKind::TargetNameCollision { target_name }.into() + ); + } + } + self.0.insert(target_name, target); + Ok(()) + } +} diff --git a/lib/src/edit/key.rs b/lib/src/edit/key.rs new file mode 100644 index 0000000..ef256ad --- /dev/null +++ b/lib/src/edit/key.rs @@ -0,0 +1,62 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::sync::Arc; + +use aws_lc_rs::error::Unspecified; +use aws_lc_rs::rand::SecureRandom; +use aws_lc_rs::signature::Ed25519KeyPair; +use tough::async_trait; +use tough::key_source::KeySource; +use tough::schema::key::Key; +use tough::sign::Sign; + +use crate::error::Error; +use crate::error::ErrorKind; + +#[derive(Debug, Clone)] +pub struct Ed25519Key(Arc); + +impl Ed25519Key { + pub fn generate() -> Result { + Ok(Self(Arc::new( + Ed25519KeyPair::generate() + .map_err(|Unspecified| ErrorKind::Ed25519Generate)?, + ))) + } +} + +#[async_trait] +impl Sign for Ed25519Key { + fn tuf_key(&self) -> Key { + self.0.tuf_key() + } + + async fn sign( + &self, + msg: &[u8], + rng: &(dyn SecureRandom + Sync), + ) -> Result, Box> + { + Sign::sign(self.0.as_ref(), msg, rng).await + } +} + +#[async_trait] +impl KeySource for Ed25519Key { + async fn as_sign( + &self, + ) -> Result, Box> + { + Ok(Box::new(self.clone())) + } + + async fn write( + &self, + _value: &str, + _key_id_hex: &str, + ) -> Result<(), Box> { + Ok(()) + } +} diff --git a/lib/src/assemble/mod.rs b/lib/src/edit/mod.rs similarity index 62% rename from lib/src/assemble/mod.rs rename to lib/src/edit/mod.rs index 51542e2..839a057 100644 --- a/lib/src/assemble/mod.rs +++ b/lib/src/edit/mod.rs @@ -2,10 +2,13 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -mod build; -mod deployment_unit; -mod manifest; +mod editor; +mod key; +mod root; +mod sign; +mod source; -pub use build::*; -pub use deployment_unit::*; -pub use manifest::*; +pub use editor::*; +pub use key::*; +pub use root::*; +pub use sign::*; diff --git a/lib/src/root.rs b/lib/src/edit/root.rs similarity index 54% rename from lib/src/root.rs rename to lib/src/edit/root.rs index dc64d37..eebcae0 100644 --- a/lib/src/root.rs +++ b/lib/src/edit/root.rs @@ -3,54 +3,59 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use std::collections::HashMap; -use std::num::NonZeroU64; +use std::num::NonZero; -use anyhow::Result; use aws_lc_rs::rand::SystemRandom; -use chrono::{DateTime, Utc}; +use chrono::DateTime; +use chrono::Utc; use tough::editor::signed::SignedRole; -use tough::schema::{KeyHolder, RoleKeys, RoleType, Root}; +use tough::key_source::KeySource; +use tough::schema::KeyHolder; +use tough::schema::RoleKeys; +use tough::schema::RoleType; +use tough::schema::Root; -use crate::key::Key; +use crate::error::Error; +use crate::error::ErrorKind; -pub async fn new_root( - keys: Vec, +pub async fn generate_root( + keys: &[Box], expires: DateTime, -) -> Result> { +) -> Result, Error> { let mut root = Root { - spec_version: "1.0.0".to_string(), - consistent_snapshot: true, - version: NonZeroU64::new(1).unwrap(), + spec_version: "1.0.0".into(), + consistent_snapshot: false, + version: NonZero::::MIN, expires, keys: HashMap::new(), roles: HashMap::new(), _extra: HashMap::new(), }; - for key in &keys { - let key = key.as_tuf_key()?; - root.keys.insert(key.key_id()?, key); + + for key in keys { + let key = key.as_sign().await.map_err(ErrorKind::ToughKey)?.tuf_key(); + root.keys.insert(key.key_id().map_err(ErrorKind::KeyId)?, key); } - for kind in [ + for role_type in [ RoleType::Root, RoleType::Snapshot, RoleType::Targets, RoleType::Timestamp, ] { root.roles.insert( - kind, + role_type, RoleKeys { keyids: root.keys.keys().cloned().collect(), - threshold: NonZeroU64::new(1).unwrap(), + threshold: NonZero::::MIN, _extra: HashMap::new(), }, ); } - let keys = crate::key::boxed_keys(keys); Ok(SignedRole::new( root.clone(), &KeyHolder::Root(root), - &keys, + keys, &SystemRandom::new(), ) .await?) diff --git a/lib/src/edit/sign.rs b/lib/src/edit/sign.rs new file mode 100644 index 0000000..bbbedc5 --- /dev/null +++ b/lib/src/edit/sign.rs @@ -0,0 +1,293 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::BTreeMap; +use std::collections::HashMap; +use std::io::Write; +use std::num::NonZero; +use std::time::Duration; + +use camino::Utf8Path; +use chrono::DateTime; +use chrono::SubsecRound; +use chrono::Utc; +use flate2::Compression; +use futures_util::TryStreamExt; +use rawzip::time::UtcDateTime; +use tough::key_source::KeySource; +use tough::schema::Root; +use tough::schema::Signed; + +use crate::edit::Ed25519Key; +use crate::edit::source::FileSource; +use crate::edit::source::Target; +use crate::edit::source::TargetSource; +use crate::error::Error; +use crate::error::ErrorKind; +use crate::zip_writer::ZipWriter; + +pub(crate) const DEFAULT_VALIDITY: Duration = + Duration::from_secs(60 * 60 * 24 * 7 /* 1 week */); + +#[derive(Debug)] +pub struct UnsignedRepository<'a> { + targets: BTreeMap>, + root: Option>, + keys: Vec>, + generate_root: bool, + snapshot_version: NonZero, + snapshot_expires: DateTime, + targets_version: NonZero, + targets_expires: DateTime, + timestamp_version: NonZero, + timestamp_expires: DateTime, +} + +impl<'a> UnsignedRepository<'a> { + pub(crate) fn from_targets(targets: BTreeMap>) -> Self { + let now = Utc::now().trunc_subsecs(0); + let version = NonZero::try_from(now.timestamp()) + .and_then(NonZero::::try_from) + .unwrap_or(NonZero::::MIN); + let expires = now + DEFAULT_VALIDITY; + + Self { + targets, + root: None, + keys: Vec::new(), + generate_root: false, + snapshot_version: version, + snapshot_expires: expires, + targets_version: version, + targets_expires: expires, + timestamp_version: version, + timestamp_expires: expires, + } + } + + pub fn root(self, root: impl AsRef<[u8]>) -> Self { + Self { + root: Some(root.as_ref().to_vec()), + generate_root: false, + ..self + } + } + + pub fn key(mut self, key: impl KeySource + 'static) -> Self { + self.keys.push(Box::new(key)); + self + } + + pub fn generate_root(self) -> Self { + Self { root: None, generate_root: true, ..self } + } + + pub fn snapshot_version(self, snapshot_version: NonZero) -> Self { + Self { snapshot_version, ..self } + } + + pub fn snapshot_expires(self, snapshot_expires: DateTime) -> Self { + Self { snapshot_expires, ..self } + } + + pub fn targets_version(self, targets_version: NonZero) -> Self { + Self { targets_version, ..self } + } + + pub fn targets_expires(self, targets_expires: DateTime) -> Self { + Self { targets_expires, ..self } + } + + pub fn timestamp_version(self, timestamp_version: NonZero) -> Self { + Self { timestamp_version, ..self } + } + + pub fn timestamp_expires(self, timestamp_expires: DateTime) -> Self { + Self { timestamp_expires, ..self } + } + + pub async fn sign(mut self) -> Result, Error> { + let (root, consistent_snapshot) = if let Some(root) = self.root { + let parsed_root: Signed = serde_json::from_slice(&root) + .map_err(ErrorKind::ParseSigningRoot)?; + (root, parsed_root.signed.consistent_snapshot) + } else if self.generate_root { + if self.keys.is_empty() { + self.keys.push(Box::new(Ed25519Key::generate()?)); + } + let expires = self + .snapshot_expires + .min(self.targets_expires) + .min(self.timestamp_expires); + let root = crate::edit::generate_root(&self.keys, expires).await?; + (root.buffer().clone(), root.signed().signed.consistent_snapshot) + } else { + return Err(ErrorKind::NoSigningRoot.into()); + }; + + let tempdir = tokio::task::spawn_blocking(camino_tempfile::tempdir) + .await? + .map_err(ErrorKind::CreateTempDir)?; + + // tough's RepositoryEditor can only be constructed with a path to a + // root role, which is mildly silly; we need to write the root out to a + // temporary directory so it can read it back in again. + let root_path = tempdir.path().join("root.json"); + tokio::fs::write(&root_path, &root).await.map_err(|source| { + ErrorKind::WriteFile { source, path: root_path.clone() } + })?; + let mut editor = + tough::editor::RepositoryEditor::new(&root_path).await?; + editor + .snapshot_version(self.snapshot_version) + .snapshot_expires(self.snapshot_expires) + .targets_version(self.targets_version)? + .targets_expires(self.targets_expires)? + .timestamp_version(self.timestamp_version) + .timestamp_expires(self.timestamp_expires); + + let mut sources = BTreeMap::new(); + for (target_name, Target { length, sha256, source }) in self.targets { + let path = if consistent_snapshot { + format!("{}.{target_name}", hex::encode(&sha256)) + } else { + target_name.clone() + }; + let hashes = tough::schema::Hashes { + sha256: sha256.into(), + _extra: HashMap::new(), + }; + let target = tough::schema::Target { + length, + hashes, + custom: HashMap::new(), + _extra: HashMap::new(), + }; + editor.add_target(target_name.as_str(), target)?; + + sources.insert((FilePrefix::Targets, path), source); + } + + let signed = editor.sign(&self.keys).await?; + // We can't read the metadata directly, so we again need to write it to + // our temporary directory and then read the data back in. + let metadata_dir = tempdir.path().join("metadata"); + signed.write(&metadata_dir).await?; + let mut read_dir = crate::util::read_dir(metadata_dir).await?; + while let Some(entry) = read_dir.try_next().await? { + // This is opening a file within the tempdir that is about to be + // deleted. This is expected to be fine. + let source = + TargetSource::File(FileSource::open(entry.path()).await?); + sources.insert( + (FilePrefix::Metadata, entry.file_name().into()), + source, + ); + } + + Ok(SignedRepository { sources }) + } +} + +#[derive(Debug)] +pub struct SignedRepository<'a> { + sources: BTreeMap<(FilePrefix, String), TargetSource<'a>>, +} + +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] +enum FilePrefix { + Metadata, + Targets, +} + +impl SignedRepository<'_> { + pub async fn write_zip( + &mut self, + writer: W, + modification_time: DateTime, + ) -> Result { + self.write_zip_impl(ZipWriter::new(writer), modification_time, None) + .await + } + + pub async fn write_zip_file( + &mut self, + path: impl AsRef, + modification_time: DateTime, + ) -> Result<(), Error> { + let path = path.as_ref(); + self.write_zip_impl( + ZipWriter::create(path), + modification_time, + Some(path), + ) + .await + } + + async fn write_zip_impl( + &mut self, + mut writer: ZipWriter, + modification_time: DateTime, + archive_path: Option<&Utf8Path>, + ) -> Result { + let last_modified = + UtcDateTime::from_unix(modification_time.timestamp()); + 'outer: for ((prefix, name), source) in &mut self.sources { + let prefix = Utf8Path::new(match prefix { + FilePrefix::Metadata => "repo/metadata", + FilePrefix::Targets => "repo/targets", + }); + let mut stream = source.stream(); + let first_chunk = stream.try_next().await?; + let compression = first_chunk + .as_deref() + .map_or(Compression::none(), deflate_heuristic); + let Ok(mut entry) = writer + .new_file(prefix.join(name)) + .compression(compression) + .last_modified(last_modified) + .unix_permissions(0o644) + .start() + .await + else { + break 'outer; + }; + let Some(first_chunk) = first_chunk else { continue }; + let Ok(()) = entry.write(first_chunk).await else { break }; + while let Some(item) = stream.try_next().await? { + let Ok(()) = entry.write(item).await else { break 'outer }; + } + } + writer.finish().await.map_err(|source| { + let archive_path = archive_path.map(|p| p.to_owned()); + ErrorKind::WriteZip { source, archive_path }.into() + }) + } +} + +fn deflate_heuristic(buf: &[u8]) -> Compression { + if buf.is_empty() { + // probably an empty file! + Compression::none() + } else if buf.starts_with(b"\x1f\x8b") { + // gzip, e.g. illumos zone tarball + Compression::none() + } else if buf.starts_with(b"PK\x03\x04") { + // ZIP archive, e.g. hubris archive. not necessarily compressed based on + // this heuristic alone but in our case it's very likely. + Compression::none() + } else if buf.starts_with(&0x1DEB0075_u32.to_le_bytes()) { + // oxide phase 2 OS image. images are zlib-compressed after the header + // if the least-significant bit of the flags starting at byte 8 is set: + if buf.get(8).is_some_and(|b| *b & 0x1 == 0x1) { + Compression::none() + } else { + // not zlib-compressed, but the ZFS dataset likely has compression + // enabled. use fast compression to deflate unused blocks + Compression::fast() + } + } else { + Compression::best() + } +} diff --git a/lib/src/edit/source.rs b/lib/src/edit/source.rs new file mode 100644 index 0000000..0d95f84 --- /dev/null +++ b/lib/src/edit/source.rs @@ -0,0 +1,257 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::convert::Infallible; +use std::pin::Pin; + +use bytes::Bytes; +use camino::Utf8PathBuf; +use futures_util::Stream; +use futures_util::StreamExt; +use futures_util::TryStreamExt; +use futures_util::stream; +use hubtools::Caboose; +use hubtools::RawHubrisArchive; +use serde::Serialize; +use sha2::Digest; +use sha2::Sha256; +use tokio::fs::File; +use tokio::io::AsyncSeekExt; +use tokio_util::io::ReaderStream; +use tufaceous_artifact::ArtifactHash; + +use crate::Repository; +use crate::error::Error; +use crate::error::ErrorKind; + +#[derive(Debug)] +pub(crate) struct Target<'a> { + pub(crate) length: u64, + pub(crate) sha256: Vec, + pub(crate) source: TargetSource<'a>, +} + +#[derive(Debug)] +pub(crate) enum TargetSource<'a> { + Bytes(BytesSource), + File(FileSource), + Repository(RepositorySource<'a>), + Fake(FakeSource), +} + +impl TargetSource<'_> { + pub(crate) fn cost(&self) -> usize { + match self { + TargetSource::Bytes(_) => 0, + TargetSource::Fake(_) => 1, + TargetSource::File(_) => 2, + TargetSource::Repository(_) => 3, + } + } + + pub(crate) fn stream( + &mut self, + ) -> Pin> + '_>> { + match self { + TargetSource::Bytes(BytesSource(bytes)) => { + Box::pin(stream::once(std::future::ready(Ok(bytes.clone())))) + } + TargetSource::File(source) => Box::pin(source.stream()), + TargetSource::Repository(source) => Box::pin(source.stream()), + TargetSource::Fake(source) => { + Box::pin(source.stream().err_into::()) + } + } + } +} + +#[derive(Debug, Clone)] +pub(crate) struct BytesSource(pub(crate) Bytes); + +impl BytesSource { + pub(crate) fn json( + data: &T, + ) -> Result { + let mut s = serde_json::to_string_pretty(data)?; + s.push('\n'); + Ok(Self(s.into())) + } + + pub(crate) fn into_target(self) -> Target<'static> { + Target { + length: self.0.len().try_into().unwrap(), + sha256: Sha256::digest(&self.0).to_vec(), + source: TargetSource::Bytes(self), + } + } +} + +#[derive(Debug)] +pub(crate) struct FileSource { + file: File, + path: Utf8PathBuf, + length_sha256: Option<(u64, ArtifactHash)>, +} + +impl FileSource { + pub(crate) async fn open( + path: impl Into, + ) -> Result { + let path = path.into(); + let file = File::open(&path).await.map_err(|source| { + ErrorKind::OpenFile { source, path: path.clone() } + })?; + Ok(Self::from_file(file, path)) + } + + pub(crate) fn from_file(file: File, path: Utf8PathBuf) -> Self { + Self { file, path, length_sha256: None } + } + + async fn read( + &mut self, + mut vec: Option<&mut Vec>, + ) -> Result<(u64, ArtifactHash), Error> { + let (length, hasher) = self + .stream() + .try_fold( + (0u64, Sha256::new()), + |(mut length, mut hasher), bytes| { + length += u64::try_from(bytes.len()).unwrap(); + hasher.update(&bytes); + if let Some(vec) = vec.as_mut() { + vec.extend_from_slice(&bytes) + } + std::future::ready(Ok((length, hasher))) + }, + ) + .await?; + let sha256 = ArtifactHash(hasher.finalize().into()); + Ok(*self.length_sha256.insert((length, sha256))) + } + + pub(crate) async fn into_target( + mut self, + ) -> Result, Error> { + let (length, sha256) = match self.length_sha256 { + Some(inner) => inner, + None => self.read(None).await?, + }; + Ok(Target { + length, + sha256: sha256.0.to_vec(), + source: TargetSource::File(self), + }) + } + + pub(crate) async fn read_hubris_caboose( + &mut self, + ) -> Result { + let mut image = Vec::new(); + self.read(Some(&mut image)).await?; + RawHubrisArchive::from_vec(image) + .and_then(|image| image.read_caboose()) + .map_err(|source| { + ErrorKind::ReadHubrisArchive { source, path: self.path.clone() } + .into() + }) + } + + pub(crate) fn stream( + &mut self, + ) -> impl Stream> { + stream::once(async { + if let Err(source) = self.file.rewind().await { + return Err(Error::from(ErrorKind::SeekFile { + source, + path: self.path.clone(), + })); + } + Ok(ReaderStream::new(&mut self.file) + .map_err(|source| ErrorKind::ReadFile { + source, + path: self.path.clone(), + }) + .err_into::()) + }) + .try_flatten() + } +} + +#[derive(Debug)] +pub(crate) struct RepositorySource<'a> { + pub(crate) repo: &'a Repository, + pub(crate) target_name: String, + pub(crate) length: u64, + pub(crate) sha256: Vec, +} + +impl<'a> RepositorySource<'a> { + pub(crate) fn into_target(self) -> Target<'a> { + Target { + length: self.length, + sha256: self.sha256.clone(), + source: TargetSource::Repository(self), + } + } + + pub(crate) fn stream(&self) -> impl Stream> { + stream::once(self.repo.read_target(&self.target_name)).try_flatten() + } +} + +#[derive(Debug)] +pub(crate) struct FakeSource { + prefix: Bytes, + length: usize, + sha256: Option, +} + +impl FakeSource { + pub(crate) fn new(prefix: String, length: usize) -> Self { + Self { prefix: prefix.into(), length, sha256: None } + } + + async fn sha256(&mut self) -> ArtifactHash { + if let Some(sha256) = self.sha256 { + return sha256; + } + let mut stream = self.stream(); + let mut hasher = Sha256::new(); + while let Some(Ok(bytes)) = stream.next().await { + hasher.update(&bytes); + } + *self.sha256.insert(ArtifactHash(hasher.finalize().into())) + } + + pub(crate) async fn into_target(mut self) -> Target<'static> { + Target { + length: self.length.try_into().unwrap(), + sha256: self.sha256().await.0.to_vec(), + source: TargetSource::Fake(self), + } + } + + pub(crate) fn stream( + &self, + ) -> impl Stream> + 'static { + const ZERO: Bytes = Bytes::from_static(&[0; 8192]); + + let mut prefix = self.prefix.clone(); + prefix.truncate(self.length); + let suffix = stream::unfold( + self.length.saturating_sub(prefix.len()), + |remaining| { + std::future::ready(if remaining == 0 { + None + } else if remaining < ZERO.len() { + Some((Ok(ZERO.slice(..remaining)), 0)) + } else { + Some((Ok(ZERO), remaining - ZERO.len())) + }) + }, + ); + stream::once(std::future::ready(Ok(prefix))).chain(suffix) + } +} diff --git a/lib/src/error.rs b/lib/src/error.rs new file mode 100644 index 0000000..6de161b --- /dev/null +++ b/lib/src/error.rs @@ -0,0 +1,215 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::convert::Infallible; +use std::fmt; +use std::fmt::Debug; +use std::fmt::Display; +use std::ops::Range; + +use camino::Utf8PathBuf; + +#[derive(Debug, thiserror::Error)] +#[error(transparent)] +pub struct Error(pub Box); + +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum ErrorKind { + #[error(transparent)] + Fetch(tough::TransportError), + #[error(transparent)] + Join(#[from] tokio::task::JoinError), + #[error(transparent)] + Tough(#[from] tough::error::Error), + #[error("error while manipulating key")] + ToughKey(#[source] Box), + + #[error( + "failed to read zip archive{archive_path}", + archive_path = SpacePath(archive_path), + )] + ReadZip { source: rawzip::Error, archive_path: Option }, + #[error( + "failed to write zip archive{archive_path}", + archive_path = SpacePath(archive_path), + )] + WriteZip { source: rawzip::Error, archive_path: Option }, + #[error( + "zip archive{archive_path}'s end of central directory record \ + expects {expected} entries, but found {actual} entries", + archive_path = SpacePath(archive_path) + )] + ZipEntryCount { + expected: u64, + actual: u64, + archive_path: Option, + }, + #[error( + "zip archive{archive_path} has overlapping compressed data ranges: \ + {earlier_path:?} ({earlier:?}) and {later_path:?} ({later:?})", + archive_path = SpacePath(archive_path), + earlier_path = DebugByteString(earlier_path), + later_path = DebugByteString(later_path), + )] + ZipOverlappingRanges { + earlier_path: Vec, + earlier: Range, + later_path: Vec, + later: Range, + archive_path: Option, + }, + #[error( + "zip archive{archive_path} path name mismatch between central directory \ + and local header ({central:?} != {local:?})", + archive_path = SpacePath(archive_path), + central = DebugByteString(central), + local = DebugByteString(local), + )] + ZipPathMismatch { + central: Vec, + local: Vec, + archive_path: Option, + }, + #[error( + "zip archive{archive_path} compressed data range for {file_path:?} \ + ({data_range:?}) overruns central directory", + archive_path = SpacePath(archive_path), + file_path = DebugByteString(file_path), + )] + ZipRangeOverrun { + file_path: Vec, + data_range: Range, + archive_path: Option, + }, + + #[error("failed to open file {path}")] + OpenFile { source: std::io::Error, path: Utf8PathBuf }, + #[error("failed to read from file {path}")] + ReadFile { source: std::io::Error, path: Utf8PathBuf }, + #[error("failed to write to file {path}")] + WriteFile { source: std::io::Error, path: Utf8PathBuf }, + #[error("failed to seek in file {path}")] + SeekFile { source: std::io::Error, path: Utf8PathBuf }, + #[error("failed to read directory {path}")] + ReadDir { source: std::io::Error, path: Utf8PathBuf }, + + #[error("failed to create temporary file")] + CreateTempFile(#[source] std::io::Error), + #[error("failed to read from temporary file")] + ReadTempFile(#[source] std::io::Error), + #[error("failed to write to temporary file")] + WriteTempFile(#[source] std::io::Error), + #[error("failed to create temporary directory")] + CreateTempDir(#[source] std::io::Error), + + #[error("failed to read hubris archive {path}")] + ReadHubrisArchive { source: hubtools::Error, path: Utf8PathBuf }, + #[error("failed to read caboose from {path}")] + ReadCaboose { + source: tufaceous_artifact::hubris::ReadCabooseError, + path: Utf8PathBuf, + }, + + #[error("unspecified failure when generating ed25519 key")] + Ed25519Generate, + #[error("failed to calculate TUF key ID")] + KeyId(#[source] tough::schema::Error), + + #[error("metadata base URL unset")] + MetadataBaseUrlUnset, + #[error("targets base URL unset")] + TargetsBaseUrlUnset, + #[error("failed to join {url} onto {base}")] + UrlJoin { source: url::ParseError, url: &'static str, base: String }, + #[error("no trust roots provided to load repository")] + NoTrustRoots, + #[error("target {target_name} not found")] + TargetNotFound { target_name: String }, + #[error("failed to parse target {target}")] + ParseTargetJson { source: serde_json::Error, target: String }, + + #[error(transparent)] + ArtifactVersion(#[from] tufaceous_artifact::ArtifactVersionError), + #[error("target name collision on {target_name}")] + TargetNameCollision { target_name: String }, + #[error("failed to serialize artifacts document")] + SerializeArtifacts(#[source] serde_json::Error), + #[error("failed to serialize installinator document")] + SerializeInstallinator(#[source] serde_json::Error), + #[error("no root provided to sign repository")] + NoSigningRoot, + #[error("failed to parse signing root")] + ParseSigningRoot(#[source] serde_json::Error), + + #[error("failed to read composite artifact {target}")] + ReadCompositeArtifact { source: std::io::Error, target: String }, + #[error("failed to read oxide.json from {path}")] + ReadZoneOxideJson { source: std::io::Error, path: Utf8PathBuf }, + #[error("importing v1 repository to an editor is not supported")] + ImportV1Repo, +} + +impl> From for Error { + fn from(kind: T) -> Self { + Error(Box::new(kind.into())) + } +} + +impl From for ErrorKind { + fn from(error: Infallible) -> Self { + match error {} + } +} + +struct SpacePath<'a>(&'a Option); + +impl Display for SpacePath<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.0 { + Some(path) => write!(f, " {path}"), + None => Ok(()), + } + } +} + +struct DebugByteString<'a>(&'a [u8]); + +impl Debug for DebugByteString<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if let Ok(s) = std::str::from_utf8(self.0) { + write!(f, "{s:?}") + } else { + write!(f, "b\"{}\"", self.0.escape_ascii()) + } + } +} + +#[cfg(test)] +mod tests { + use std::error::Error as _; + use std::fmt::Write; + + use crate::error::Error; + use crate::error::ErrorKind; + + #[test] + fn error_display_chain_doesnt_repeat() { + let err = Error::from(ErrorKind::OpenFile { + source: std::io::Error::from(std::io::ErrorKind::NotFound), + path: "/nowhere/in/particular".into(), + }); + let mut chain = err.to_string(); + let mut source = err.source(); + while let Some(err) = source { + write!(chain, ": {err}").unwrap(); + source = err.source(); + } + + assert_eq!( + chain, + "failed to open file /nowhere/in/particular: entity not found" + ); + } +} diff --git a/lib/src/key.rs b/lib/src/key.rs deleted file mode 100644 index 2703d82..0000000 --- a/lib/src/key.rs +++ /dev/null @@ -1,84 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use std::fmt::{self, Display}; -use std::str::FromStr; - -use anyhow::{Result, bail}; -use aws_lc_rs::rand::SystemRandom; -use aws_lc_rs::signature::Ed25519KeyPair; -use base64::{Engine, engine::general_purpose::URL_SAFE}; -use tough::async_trait; -use tough::key_source::KeySource; -use tough::sign::{Sign, SignKeyPair}; - -pub(crate) fn boxed_keys(keys: Vec) -> Vec> { - keys.into_iter().map(|k| Box::new(k) as Box).collect() -} - -#[derive(Debug, Clone)] -pub enum Key { - Ed25519 { pkcs8: Vec }, -} - -impl Key { - pub fn generate_ed25519() -> Result { - let pkcs8 = Ed25519KeyPair::generate_pkcs8(&SystemRandom::new())?; - Ok(Key::Ed25519 { pkcs8: pkcs8.as_ref().to_vec() }) - } - - fn as_sign_key_pair(&self) -> Result { - match self { - Key::Ed25519 { pkcs8 } => { - Ok(SignKeyPair::ED25519(Ed25519KeyPair::from_pkcs8(pkcs8)?)) - } - } - } - - pub(crate) fn as_tuf_key(&self) -> Result { - Ok(self.as_sign_key_pair()?.tuf_key()) - } -} - -#[async_trait] -impl KeySource for Key { - async fn as_sign( - &self, - ) -> Result, Box> - { - Ok(Box::new(self.as_sign_key_pair()?)) - } - - async fn write( - &self, - _value: &str, - _key_id_hex: &str, - ) -> Result<(), Box> { - Err("cannot write key back to key source".into()) - } -} - -impl FromStr for Key { - type Err = anyhow::Error; - - fn from_str(s: &str) -> Result { - match s.split_once(':') { - Some(("ed25519", base64)) => { - Ok(Key::Ed25519 { pkcs8: URL_SAFE.decode(base64)? }) - } - Some((kind, _)) => bail!("Invalid key source kind: {}", kind), - None => bail!("Invalid key source (format is `kind:data`)"), - } - } -} - -impl Display for Key { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Key::Ed25519 { pkcs8 } => { - write!(f, "ed25519:{}", URL_SAFE.encode(pkcs8)) - } - } - } -} diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 9dc1854..1fc75bf 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -2,16 +2,15 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -mod archive; -mod artifact; -pub mod assemble; -mod key; -mod repository; -pub mod root; -mod target; -mod utils; +pub mod edit; +pub mod error; +mod loader; +mod repo; +mod schema; +mod util; +mod zip_transport; +mod zip_writer; -pub use archive::*; -pub use artifact::*; -pub use key::*; -pub use repository::*; +pub use loader::*; +pub use repo::*; +pub use zip_transport::*; diff --git a/lib/src/loader.rs b/lib/src/loader.rs new file mode 100644 index 0000000..832ce2b --- /dev/null +++ b/lib/src/loader.rs @@ -0,0 +1,216 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::fmt::Debug; + +use camino::Utf8PathBuf; +use futures_util::TryStreamExt; +use slog::Logger; +use url::Url; + +use crate::Repository; +use crate::ZipTransport; +use crate::error::Error; +use crate::error::ErrorKind; + +/// Controls how repository trust is established. +/// +/// Used in [`RepositoryLoader::trust_store_behavior`]. +#[derive(Debug, Clone, Copy, Default)] +pub enum TrustStoreBehavior { + /// Verification by the provided trust roots is required. + /// + /// This is the default behavior. + #[default] + Required, + /// The trust root is established by fetching `metadata/1.root.json` from + /// the repository. Verification then proceeds normally. + UnsafeBlindFaith, +} + +#[derive(Debug, Clone, Default)] +pub struct RepositoryLoader { + expiration_enforcement: tough::ExpirationEnforcement, + limits: tough::Limits, + metadata_base_url: Option, + targets_base_url: Option, + trust_roots: Vec>, + trust_store_behavior: TrustStoreBehavior, + v1_compatibility: bool, +} + +impl RepositoryLoader { + /// Constructs a new `RepositoryLoader`. + /// + /// This is the same as `Repository::loader()`. + pub fn new() -> Self { + Self::default() + } + + /// Set whether metadata expiration times are enforced. + pub fn expiration_enforcement( + self, + expiration_enforcement: tough::ExpirationEnforcement, + ) -> Self { + Self { expiration_enforcement, ..self } + } + + /// Set limits used while fetching repository metadata. + pub fn limits(self, limits: tough::Limits) -> Self { + Self { limits, ..self } + } + + /// Set the metadata base URL. + pub fn metadata_base_url(self, metadata_base_url: Url) -> Self { + Self { metadata_base_url: Some(metadata_base_url), ..self } + } + + /// Set the targets base URL. + pub fn targets_base_url(self, targets_base_url: Url) -> Self { + Self { targets_base_url: Some(targets_base_url), ..self } + } + + /// Add a trusted root role to the trust store. + pub fn trust_root(mut self, trust_root: impl AsRef) -> Self { + self.trust_roots.push(trust_root.as_ref().into()); + self + } + + /// Add additional trusted root roles to the trust store. + pub fn trust_roots( + mut self, + trust_roots: impl IntoIterator>, + ) -> Self { + let iter = trust_roots.into_iter().map(|root| root.as_ref().into()); + self.trust_roots.extend(iter); + self + } + + /// Set how repository trust is established. + pub fn trust_store_behavior( + self, + trust_store_behavior: TrustStoreBehavior, + ) -> Self { + Self { trust_store_behavior, ..self } + } + + /// Enable compatibility with v1-format repositories. + /// + /// If a v1-format repository is encountered, composite artifacts are + /// extracted into temporary files in [`std::env::temp_dir()`]. Artifacts + /// are not extracted in parallel to avoid unexpectedly using too many + /// resources. Reading a v1-format repository takes on the order of about 10 + /// seconds on a 2025-era CPU. + pub fn v1_compatibility(self, v1_compatibility: bool) -> Self { + Self { v1_compatibility, ..self } + } + + fn zip_base_urls(self) -> Self { + self.metadata_base_url("zip:///repo/metadata/".parse().unwrap()) + .targets_base_url("zip:///repo/targets/".parse().unwrap()) + } + + /// Load a Tufaceous-generated ZIP archive from an owned buffer. + pub async fn load_zip_buffer( + self, + data: T, + log: &Logger, + ) -> Result + where + T: AsRef<[u8]> + Debug + Send + Sync + 'static, + { + let transport = ZipTransport::from_slice(data, log)?; + self.zip_base_urls().load(transport, log).await + } + + /// Load a Tufaceous-generated ZIP archive from a file. + pub async fn load_zip_file( + self, + archive_path: impl Into, + log: &Logger, + ) -> Result { + let transport = ZipTransport::from_file(archive_path, log).await?; + self.zip_base_urls().load(transport, log).await + } + + /// Load a repository from the configured metadata and targets base URLs + /// using the given transport. + pub async fn load( + self, + transport: impl tough::Transport + Clone + 'static, + log: &Logger, + ) -> Result { + let Some(metadata_base_url) = self.metadata_base_url else { + return Err(ErrorKind::MetadataBaseUrlUnset.into()); + }; + let Some(targets_base_url) = self.targets_base_url else { + return Err(ErrorKind::TargetsBaseUrlUnset.into()); + }; + + let trust_roots = match self.trust_store_behavior { + TrustStoreBehavior::Required => self.trust_roots, + TrustStoreBehavior::UnsafeBlindFaith => { + let root_path = "1.root.json"; + let root_url = + metadata_base_url.join(root_path).map_err(|source| { + ErrorKind::UrlJoin { + source, + url: root_path, + base: metadata_base_url.to_string(), + } + })?; + let mut stream = transport + .fetch(root_url.clone()) + .await + .map_err(ErrorKind::Fetch)?; + let mut root = Vec::new(); + while let Some(bytes) = + stream.try_next().await.map_err(ErrorKind::Fetch)? + { + root.extend_from_slice(&bytes); + } + vec![root] + } + }; + + let mut last_error = ErrorKind::NoTrustRoots; + for trust_root in trust_roots { + match tough::RepositoryLoader::new( + &trust_root, + metadata_base_url.clone(), + targets_base_url.clone(), + ) + .expiration_enforcement(self.expiration_enforcement) + .limits(self.limits) + .transport(transport.clone()) + .load() + .await + { + Ok(repo) => { + return Repository::from_loaded( + repo, + log, + trust_root, + self.v1_compatibility, + ) + .await; + } + Err(error) => { + if matches!( + error, + tough::error::Error::VerifyMetadata { .. } + | tough::error::Error::VerifyTrustedMetadata { .. } + ) { + // failed to verify, try the next trust root + last_error = ErrorKind::from(error); + } else { + // other errors are fatal + return Err(Error::from(error)); + } + } + } + } + Err(last_error.into()) + } +} diff --git a/lib/src/repo.rs b/lib/src/repo.rs new file mode 100644 index 0000000..11e6004 --- /dev/null +++ b/lib/src/repo.rs @@ -0,0 +1,204 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +mod v1; + +use std::collections::HashMap; +use std::pin::Pin; + +use bytes::Bytes; +use futures_util::Stream; +use futures_util::TryStreamExt; +use semver::Version; +use serde::de::DeserializeOwned; +use slog::Logger; +use slog::warn; +use tough::TargetName; +use tough::schema::Hashes; +use tough::schema::Target; +use tufaceous_artifact::Artifact; +use tufaceous_artifact::ArtifactHash; +use tufaceous_artifact::Artifacts; + +use crate::RepositoryLoader; +use crate::error::Error; +use crate::error::ErrorKind; +use crate::schema::ArtifactSchema; +use crate::schema::ArtifactsSchema; + +/// A loaded TUF repository. +#[derive(Debug, Clone)] +pub struct Repository { + inner: tough::Repository, + system_version: Version, + trust_root: Vec, + artifacts: Artifacts, + v1_unpacked: Option, +} + +impl Repository { + pub fn loader() -> RepositoryLoader { + RepositoryLoader::new() + } + + pub(crate) async fn from_loaded( + repo: tough::Repository, + log: &Logger, + trust_root: Vec, + v1_compatibility: bool, + ) -> Result { + let Some(ArtifactsSchema { system_version, artifacts }) = + read_target_json(&repo, ArtifactsSchema::TARGET_NAME).await? + else { + if v1_compatibility + && let Some(repo) = + v1::from_loaded(repo, log, trust_root).await? + { + return Ok(repo); + } + + return Err(ErrorKind::TargetNotFound { + target_name: ArtifactsSchema::TARGET_NAME.to_owned(), + } + .into()); + }; + + let artifacts = artifacts + .into_iter() + .filter_map(|ArtifactSchema { target_name, version, tags }| { + let (sha256, length) = sha256_length(&repo, log, &target_name)?; + Some(Artifact { target_name, version, tags, sha256, length }) + }) + .collect(); + Ok(Repository { + inner: repo, + trust_root, + system_version, + artifacts, + v1_unpacked: None, + }) + } + + pub fn system_version(&self) -> &Version { + &self.system_version + } + + pub fn trust_root(&self) -> &[u8] { + &self.trust_root + } + + pub fn targets(&self) -> &HashMap { + &self.inner.targets().signed.targets + } + + pub fn artifacts(&self) -> &Artifacts { + &self.artifacts + } + + pub fn is_v1(&self) -> bool { + self.v1_unpacked.is_some() + } + + pub async fn read_target<'a>( + &'a self, + target: &str, + ) -> Result> + use<'a>, Error> { + if let Some(stream) = read_target(&self.inner, target).await? { + return Ok(Box::pin(stream) + as Pin>>>); + } + + if let Some(unpacked) = &self.v1_unpacked + && let Some(entry) = unpacked.entries.get(target) + { + return Ok(Box::pin(entry.stream())); + } + + Err(ErrorKind::TargetNotFound { target_name: target.to_owned() }.into()) + } +} + +async fn read_target<'a>( + repo: &'a tough::Repository, + target: &str, +) -> Result> + use<'a>>, Error> { + let target_name = target.parse()?; + // Ensure the target is in the top-level targets.json role and not a + // delegated target; we don't permit the use of delegated targets in + // Tufaceous currently. + if !repo.targets().signed.targets.contains_key(&target_name) { + return Ok(None); + } + Ok(repo + .read_target(&target_name) + .await? + .map(TryStreamExt::err_into::)) +} + +async fn read_target_vec( + repo: &tough::Repository, + target: &str, +) -> Result>, Error> { + let Some(mut stream) = read_target(repo, target).await? else { + return Ok(None); + }; + let mut buf = Vec::new(); + while let Some(item) = stream.try_next().await? { + buf.extend_from_slice(item.as_ref()); + } + Ok(Some(buf)) +} + +async fn read_target_json( + repo: &tough::Repository, + target: &str, +) -> Result, Error> { + let Some(vec) = read_target_vec(repo, target).await? else { + return Ok(None); + }; + Ok(serde_json::from_slice(&vec).map_err(|source| { + ErrorKind::ParseTargetJson { source, target: target.into() } + })?) +} + +fn sha256_length( + repo: &tough::Repository, + log: &Logger, + target_name: &str, +) -> Option<(ArtifactHash, u64)> { + let parsed_name = TargetName::new(target_name) + .inspect_err(|error| { + warn!( + log, + "skipping artifact"; + "target_name" => &target_name, + "error" => error.to_string(), + ); + }) + .ok()?; + let Some(target) = repo.targets().signed.targets.get(&parsed_name) else { + warn!( + log, + "skipping artifact"; + "target_name" => &target_name, + "error" => "target not found", + ); + return None; + }; + let Hashes { sha256, .. } = &target.hashes; + let sha256 = sha256 + .as_ref() + .try_into() + .inspect_err(|_| { + warn!( + log, + "skipping artifact"; + "target_name" => &target_name, + "error" => "incorrect checksum length", + "sha256" => hex::encode(sha256), + ); + }) + .ok()?; + Some((sha256, target.length)) +} diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs new file mode 100644 index 0000000..85989dd --- /dev/null +++ b/lib/src/repo/v1.rs @@ -0,0 +1,506 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::HashMap; +use std::io::BufRead; +use std::io::Read; +use std::io::Write; +use std::sync::Arc; + +use bytes::Buf; +use bytes::Bytes; +use bytes::BytesMut; +use camino::FromPathBufError; +use camino::Utf8PathBuf; +use flate2::read::GzDecoder; +use futures_util::Stream; +use futures_util::TryStreamExt; +use futures_util::pin_mut; +use futures_util::stream; +use hubtools::RawHubrisArchive; +use rawzip::FileReader; +use rawzip::RangeReader; +use rawzip::ReaderAt; +use semver::Version; +use serde::Deserialize; +use sha2::Digest; +use sha2::Sha256; +use slog::Logger; +use slog::warn; +use tokio::sync::mpsc; +use tufaceous_artifact::Artifact; +use tufaceous_artifact::ArtifactHash; +use tufaceous_artifact::ArtifactVersion; +use tufaceous_artifact::Artifacts; +use tufaceous_artifact::KnownArtifactTags; +use tufaceous_artifact::OsBoard; +use tufaceous_artifact::OsVariant; +use tufaceous_artifact::RotSlot; + +use crate::Repository; +use crate::error::Error; +use crate::error::ErrorKind; +use crate::repo::read_target; +use crate::repo::read_target_json; +use crate::repo::read_target_vec; +use crate::repo::sha256_length; + +#[derive(Debug, Clone)] +pub(crate) struct Unpacked { + pub(crate) entries: HashMap, +} + +pub(crate) async fn from_loaded( + repo: tough::Repository, + log: &Logger, + trust_root: Vec, +) -> Result, Error> { + let Some(V1ArtifactsSchema { system_version, artifacts: v1_artifacts }) = + read_target_json(&repo, V1ArtifactsSchema::TARGET_NAME).await? + else { + return Ok(None); + }; + + let mut artifacts = Artifacts::default(); + let mut entries = HashMap::new(); + for artifact in v1_artifacts { + let kind = match artifact.kind { + V1ArtifactKind::Known(kind) => kind, + V1ArtifactKind::Unknown(kind) => { + warn!( + log, + "skipping artifact"; + "target_name" => &artifact.target, + "error" => "unknown v1 kind", + "kind" => kind, + ); + continue; + } + }; + let Some((sha256, length)) = + sha256_length(&repo, log, &artifact.target) + else { + continue; + }; + + match kind { + V1KnownArtifactKind::GimletSp + | V1KnownArtifactKind::PscSp + | V1KnownArtifactKind::SwitchSp => { + let Some(image) = + read_target_vec(&repo, &artifact.target).await? + else { + continue; + }; + let caboose = RawHubrisArchive::from_vec(image) + .and_then(|image| image.read_caboose()) + .map_err(|source| ErrorKind::ReadHubrisArchive { + source, + path: artifact.target.clone().into(), + })?; + let tags = KnownArtifactTags::from_sp_caboose(&caboose) + .map_err(|source| ErrorKind::ReadCaboose { + source, + path: artifact.target.clone().into(), + })?; + artifacts.insert(Artifact { + target_name: artifact.target, + version: artifact.version, + tags: tags.to_tags(), + sha256, + length, + }); + } + V1KnownArtifactKind::GimletRotBootloader + | V1KnownArtifactKind::PscRotBootloader + | V1KnownArtifactKind::SwitchRotBootloader => { + let Some(image) = + read_target_vec(&repo, &artifact.target).await? + else { + continue; + }; + let caboose = RawHubrisArchive::from_vec(image) + .and_then(|image| image.read_caboose()) + .map_err(|source| ErrorKind::ReadHubrisArchive { + source, + path: artifact.target.clone().into(), + })?; + let tags = + KnownArtifactTags::from_rot_bootloader_caboose(&caboose) + .map_err(|source| ErrorKind::ReadCaboose { + source, + path: artifact.target.clone().into(), + })?; + artifacts.insert(Artifact { + target_name: artifact.target, + version: artifact.version, + tags: tags.to_tags(), + sha256, + length, + }); + } + + V1KnownArtifactKind::GimletRot + | V1KnownArtifactKind::PscRot + | V1KnownArtifactKind::SwitchRot => { + let target = artifact.target.clone(); + let unpacked = + CompositeArtifact::unpack(&repo, &target).await?; + for (tar_path, inner) in unpacked.entries { + let UnpackedArtifact { file, sha256, length } = inner; + let slot = match tar_path.as_ref() { + "archive-a.zip" => RotSlot::A, + "archive-b.zip" => RotSlot::B, + _ => continue, + }; + let mut reader = RangeReader::new(file.clone(), 0..length); + let mut vec = length + .try_into() + .map(Vec::with_capacity) + .unwrap_or_default(); + let image = tokio::task::spawn_blocking(move || { + reader + .read_to_end(&mut vec) + .map(|_| vec) + .map_err(ErrorKind::ReadTempFile) + }) + .await??; + let caboose = RawHubrisArchive::from_vec(image) + .and_then(|image| image.read_caboose()) + .map_err(|source| ErrorKind::ReadHubrisArchive { + source, + path: target.clone().into(), + })?; + let tags = + KnownArtifactTags::from_rot_caboose(&caboose, slot) + .map_err(|source| ErrorKind::ReadCaboose { + source, + path: target.clone().into(), + })?; + let target_name = format!("{}/{tar_path}", artifact.target); + entries.insert( + target_name.clone(), + UnpackedArtifact { file, sha256, length }, + ); + artifacts.insert(Artifact { + target_name, + version: artifact.version.clone(), + tags: tags.to_tags(), + sha256, + length, + }); + } + } + + V1KnownArtifactKind::Host => { + unpack_os( + CompositeArtifact::unpack(&repo, &artifact.target).await?, + &mut entries, + &mut artifacts, + &artifact, + OsVariant::Host, + ); + } + V1KnownArtifactKind::Trampoline => { + unpack_os( + CompositeArtifact::unpack(&repo, &artifact.target).await?, + &mut entries, + &mut artifacts, + &artifact, + OsVariant::Recovery, + ); + } + + V1KnownArtifactKind::InstallinatorDocument => { + artifacts.insert(Artifact { + target_name: artifact.target, + version: artifact.version, + tags: KnownArtifactTags::InstallinatorDocument {}.to_tags(), + sha256, + length, + }); + } + + V1KnownArtifactKind::ControlPlane => { + let unpacked = + CompositeArtifact::unpack(&repo, &artifact.target).await?; + for (tar_path, inner) in unpacked.entries { + let UnpackedArtifact { file, sha256, length } = inner; + if tar_path.starts_with("zones/") { + let path = tar_path.to_string(); + let (file, layer_info) = + crate::util::read_zone_layer_info( + RangeReader::new(file, 0..length), + path.into(), + ) + .await?; + let file = file.into_inner(); + let target_name = + format!("{}/{tar_path}", artifact.target); + entries.insert( + target_name.clone(), + UnpackedArtifact { file, sha256, length }, + ); + artifacts.insert(Artifact { + target_name, + version: layer_info.version, + tags: KnownArtifactTags::Zone { + name: layer_info.pkg, + } + .to_tags(), + sha256, + length, + }); + } + } + } + + V1KnownArtifactKind::MeasurementCorpus => { + artifacts.insert(Artifact { + target_name: artifact.target, + version: artifact.version, + tags: KnownArtifactTags::MeasurementCorpus {}.to_tags(), + sha256, + length, + }); + } + } + } + + Ok(Some(Repository { + inner: repo, + system_version, + trust_root, + artifacts: Artifacts::new(artifacts), + v1_unpacked: Some(Unpacked { entries }), + })) +} + +#[derive(Debug, Clone)] +pub(crate) struct UnpackedArtifact { + file: Arc, + sha256: ArtifactHash, + length: u64, +} + +impl UnpackedArtifact { + pub(crate) fn stream(&self) -> impl Stream> { + stream::try_unfold( + (BytesMut::new(), Sha256::new(), 0), + async |(mut buf, mut hasher, mut bytes_read)| { + if buf.capacity() == 0 { + buf.reserve(8192); + } + buf.resize(buf.capacity(), 0); + let file = self.file.clone(); + let mut buf = tokio::task::spawn_blocking(move || { + let n = file + .read_at(&mut buf, bytes_read) + .map_err(ErrorKind::ReadTempFile)?; + buf.truncate(n); + Ok::<_, Error>(buf) + }) + .await??; + let bytes = buf.split().freeze(); + if bytes.is_empty() { + let msg = if self.sha256 + != ArtifactHash(hasher.finalize().into()) + { + "invalid checksum" + } else if self.length != bytes_read { + "invalid length" + } else { + return Ok(None); + }; + let source = std::io::Error::new( + std::io::ErrorKind::InvalidData, + msg, + ); + return Err(ErrorKind::ReadTempFile(source).into()); + } + hasher.update(&bytes); + bytes_read += u64::try_from(bytes.len()).unwrap(); + Ok(Some((bytes, (buf, hasher, bytes_read)))) + }, + ) + } +} + +#[derive(Debug)] +struct CompositeArtifact { + entries: HashMap, +} + +impl CompositeArtifact { + async fn unpack( + repo: &tough::Repository, + target_name: &str, + ) -> Result { + let stream = + read_target(repo, target_name).await?.ok_or_else(|| { + ErrorKind::TargetNotFound { + target_name: target_name.to_owned(), + } + })?; + pin_mut!(stream); + let (tx, rx) = mpsc::channel(1); + let target_name = target_name.to_owned(); + let task = tokio::task::spawn_blocking(move || { + let mut archive = + tar::Archive::new(GzDecoder::new(MpscReader::new(rx))); + let mut entries = HashMap::new(); + for entry in archive.entries().map_err(|source| { + ErrorKind::ReadCompositeArtifact { + source, + target: target_name.clone(), + } + })? { + let (mut entry, path) = entry + .and_then(|entry| { + let path = entry.header().path()?.into_owned(); + let path = Utf8PathBuf::try_from(path) + .map_err(FromPathBufError::into_io_error)?; + Ok((entry, path)) + }) + .map_err(|source| ErrorKind::ReadCompositeArtifact { + source, + target: target_name.clone(), + })?; + let mut file = camino_tempfile::tempfile() + .map_err(ErrorKind::CreateTempFile)?; + let mut hasher = Sha256::new(); + let mut length = 0u64; + let mut buf = [0; 8192]; + while let n = entry.read(&mut buf).map_err(|source| { + ErrorKind::ReadCompositeArtifact { + source, + target: target_name.clone(), + } + })? && n > 0 + { + file.write_all(&buf[..n]) + .map_err(ErrorKind::WriteTempFile)?; + hasher.update(&buf[..n]); + length += u64::try_from(n).unwrap(); + } + let file = Arc::new(file.into()); + let sha256 = ArtifactHash(hasher.finalize().into()); + entries.insert(path, UnpackedArtifact { file, sha256, length }); + } + Ok(Self { entries }) + }); + while let Some(item) = stream.try_next().await? { + let Ok(()) = tx.send(item).await else { break }; + } + task.await? + } +} + +fn unpack_os( + unpacked: CompositeArtifact, + entries: &mut HashMap, + artifacts: &mut Artifacts, + artifact: &V1Artifact, + variant: OsVariant, +) { + for (tar_path, inner) in unpacked.entries { + let tags = match tar_path.as_ref() { + "image/cosmo.rom" => { + KnownArtifactTags::OsPhase1 { variant, board: OsBoard::Cosmo } + } + "image/gimlet.rom" => { + KnownArtifactTags::OsPhase1 { variant, board: OsBoard::Gimlet } + } + "image/zfs.img" => KnownArtifactTags::OsPhase2 { variant }, + _ => continue, + }; + let target_name = format!("{}/{tar_path}", artifact.target); + artifacts.insert(Artifact { + target_name: target_name.clone(), + version: artifact.version.clone(), + tags: tags.to_tags(), + sha256: inner.sha256, + length: inner.length, + }); + entries.insert(target_name, inner); + } +} + +struct MpscReader { + rx: mpsc::Receiver, + buf: Bytes, +} + +impl MpscReader { + fn new(rx: mpsc::Receiver) -> Self { + Self { rx, buf: Bytes::new() } + } +} + +impl Read for MpscReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.fill_buf()?; + let len = self.buf.len().min(buf.len()); + self.buf.split_to(len).copy_to_slice(buf); + Ok(len) + } +} + +impl BufRead for MpscReader { + fn fill_buf(&mut self) -> std::io::Result<&[u8]> { + if self.buf.is_empty() + && let Some(next) = self.rx.blocking_recv() + { + self.buf = next; + } + Ok(&self.buf) + } + + fn consume(&mut self, amount: usize) { + self.buf.advance(amount); + } +} + +#[derive(Debug, Deserialize)] +pub(crate) struct V1ArtifactsSchema { + system_version: Version, + artifacts: Vec, +} + +impl V1ArtifactsSchema { + pub(crate) const TARGET_NAME: &str = "artifacts.json"; +} + +#[derive(Debug, Deserialize)] +struct V1Artifact { + version: ArtifactVersion, + kind: V1ArtifactKind, + target: String, +} + +#[derive(Debug, Deserialize)] +#[serde(untagged)] +enum V1ArtifactKind { + Known(V1KnownArtifactKind), + Unknown(String), +} + +#[derive(Debug, Deserialize, Clone, Copy)] +#[serde(rename_all = "snake_case")] +enum V1KnownArtifactKind { + GimletSp, + GimletRot, + GimletRotBootloader, + Host, + Trampoline, + InstallinatorDocument, + ControlPlane, + MeasurementCorpus, + PscSp, + PscRot, + PscRotBootloader, + SwitchSp, + SwitchRot, + SwitchRotBootloader, +} diff --git a/lib/src/repository.rs b/lib/src/repository.rs deleted file mode 100644 index f9c99fd..0000000 --- a/lib/src/repository.rs +++ /dev/null @@ -1,670 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use std::collections::BTreeSet; -use std::num::NonZeroU64; - -use anyhow::{Context, Result, anyhow}; -use buf_list::BufList; -use camino::{Utf8Path, Utf8PathBuf}; -use chrono::{DateTime, Utc}; -use fs_err as fs; -use futures::TryStreamExt; -use semver::Version; -use tough::editor::RepositoryEditor; -use tough::editor::signed::SignedRole; -use tough::error::Error; -use tough::schema::{Root, Target}; -use tough::{ExpirationEnforcement, Repository, RepositoryLoader, TargetName}; -use tufaceous_artifact::{ - Artifact, ArtifactHash, ArtifactVersion, ArtifactsDocument, - InstallinatorDocument, KnownArtifactKind, -}; -use url::Url; - -use crate::assemble::{ - ArtifactDeploymentUnits, DeploymentUnitData, DeploymentUnitMapBuilder, - DeploymentUnitScope, -}; -use crate::key::Key; -use crate::target::TargetWriter; -use crate::utils::merge_anyhow_list; -use crate::{AddArtifact, ArchiveBuilder}; - -/// A TUF repository describing Omicron. -#[derive(Debug)] -pub struct OmicronRepo { - log: slog::Logger, - repo: Repository, - repo_path: Utf8PathBuf, -} - -impl OmicronRepo { - /// Loads a repository from the given path. - /// - #[cfg(test)] - /// This method enforces expirations. To load without expiration enforcement, use - /// [`Self::load_ignore_expiration`]. - pub async fn load( - log: &slog::Logger, - repo_path: &Utf8Path, - trusted_roots: impl IntoIterator>, - ) -> Result { - Self::load_impl( - log, - repo_path, - trusted_roots, - ExpirationEnforcement::Safe, - ) - .await - } - - /// Loads a repository from the given path, ignoring expiration. - /// - /// Use cases for this include: - /// - /// 1. When you're editing an existing repository and will re-sign it afterwards. - /// 2. When you're reading a repository that was uploaded out-of-band, - /// instead of fetched from a network-accessible repository - /// 3. In an environment in which time isn't available. - pub async fn load_ignore_expiration( - log: &slog::Logger, - repo_path: &Utf8Path, - trusted_roots: impl IntoIterator>, - ) -> Result { - Self::load_impl( - log, - repo_path, - trusted_roots, - ExpirationEnforcement::Unsafe, - ) - .await - } - - async fn load_impl( - log: &slog::Logger, - repo_path: &Utf8Path, - trusted_roots: impl IntoIterator>, - exp: ExpirationEnforcement, - ) -> Result { - let repo_path = repo_path.canonicalize_utf8()?; - let mut verify_error = None; - for root in trusted_roots { - match RepositoryLoader::new( - &root, - Url::from_file_path(repo_path.join("metadata")) - .expect("the canonical path is not absolute?"), - Url::from_file_path(repo_path.join("targets")) - .expect("the canonical path is not absolute?"), - ) - .expiration_enforcement(exp) - .load() - .await - { - Ok(repo) => { - return Ok(Self { - log: log.new(slog::o!("component" => "OmicronRepo")), - repo, - repo_path, - }); - } - Err( - err @ (Error::VerifyMetadata { .. } - | Error::VerifyTrustedMetadata { .. }), - ) => { - verify_error = Some(err.into()); - continue; - } - Err(err) => return Err(err.into()), - } - } - Err(verify_error.unwrap_or_else(|| anyhow!("trust store is empty"))) - } - - /// Loads a repository from the given path. - /// - /// This method enforces expirations. To load without expiration enforcement, use - /// [`Self::load_untrusted_ignore_expiration`]. - pub async fn load_untrusted( - log: &slog::Logger, - repo_path: &Utf8Path, - ) -> Result { - Self::load_untrusted_impl(log, repo_path, ExpirationEnforcement::Safe) - .await - } - - /// Loads a repository from the given path, ignoring expiration. - /// - /// Use cases for this include: - /// - /// 1. When you're editing an existing repository and will re-sign it afterwards. - /// 2. When you're reading a repository that was uploaded out-of-band, - /// instead of fetched from a network-accessible repository - /// 3. In an environment in which time isn't available. - pub async fn load_untrusted_ignore_expiration( - log: &slog::Logger, - repo_path: &Utf8Path, - ) -> Result { - Self::load_untrusted_impl(log, repo_path, ExpirationEnforcement::Unsafe) - .await - } - - async fn load_untrusted_impl( - log: &slog::Logger, - repo_path: &Utf8Path, - exp: ExpirationEnforcement, - ) -> Result { - let repo_path = repo_path.canonicalize_utf8()?; - let root_json = repo_path.join("metadata").join("1.root.json"); - let root = tokio::fs::read(&root_json) - .await - .with_context(|| format!("error reading from {root_json}"))?; - Self::load_impl(log, &repo_path, &[root], exp).await - } - - /// Returns a canonicalized form of the repository path. - pub fn repo_path(&self) -> &Utf8Path { - &self.repo_path - } - - /// Returns the repository. - pub fn repo(&self) -> &Repository { - &self.repo - } - - /// Reads the artifacts document from the repo. - pub async fn read_artifacts(&self) -> Result { - self.read_json(ArtifactsDocument::FILE_NAME).await - } - - /// Reads the installinator document from the repo. - pub async fn read_installinator_document( - &self, - file_name: &str, - ) -> Result { - self.read_json(file_name).await - } - - /// Reads a JSON document from the repo by target name. - async fn read_json(&self, file_name: &str) -> Result - where - T: serde::de::DeserializeOwned, - { - let reader = self - .repo - .read_target(&file_name.try_into()?) - .await? - .ok_or_else(|| anyhow!("{} should be present", file_name))?; - let buf_list = reader - .try_collect::() - .await - .with_context(|| format!("error reading from {}", file_name))?; - serde_json::from_reader(buf_list::Cursor::new(&buf_list)) - .with_context(|| format!("error deserializing {}", file_name)) - } - - /// Archives the repository to the given path as a zip file. - /// - /// ## Why zip and not tar? - /// - /// The main reason is that zip supports random access to files and tar does - /// not. - /// - /// In principle it should be possible to read the repository out of a zip - /// file from memory, but we ran into [this - /// issue](https://github.com/awslabs/tough/pull/563) while implementing it. - /// Once that is resolved (or we write our own TUF crate) it should be - /// possible to do that. - /// - /// Regardless of this roadblock, we don't want to foreclose that option - /// forever, so this code uses zip rather than having to deal with a - /// migration in the future. - pub(crate) fn archive(&self, output_path: &Utf8Path) -> Result<()> { - let mut builder = ArchiveBuilder::new(output_path.to_owned())?; - - let metadata_dir = self.repo_path.join("metadata"); - - // Gather metadata files. - for entry in metadata_dir.read_dir_utf8().with_context(|| { - format!("error reading entries from {metadata_dir}") - })? { - let entry = - entry.context("error reading entry from {metadata_dir}")?; - let file_name = entry.file_name(); - if file_name.ends_with(".root.json") - || file_name == "timestamp.json" - || file_name.ends_with(".snapshot.json") - || file_name.ends_with(".targets.json") - { - // This is a valid metadata file. - builder.write_file( - entry.path(), - &Utf8Path::new("metadata").join(file_name), - )?; - } - } - - let targets_dir = self.repo_path.join("targets"); - - // Gather all targets. - for (name, target) in self.repo.targets().signed.targets_iter() { - let target_filename = self.target_filename(target, name); - let target_path = targets_dir.join(&target_filename); - slog::trace!(self.log, "adding {} to archive", name.resolved()); - builder.write_file( - &target_path, - &Utf8Path::new("targets").join(&target_filename), - )?; - } - - builder.finish()?; - - Ok(()) - } - - /// Prepends the target digest to the name if using consistent snapshots. Returns both the - /// digest and the filename. - /// - /// Adapted from tough's source. - fn target_filename(&self, target: &Target, name: &TargetName) -> String { - let sha256 = &target.hashes.sha256.clone().into_vec(); - if self.repo.root().signed.consistent_snapshot { - format!("{}.{}", hex::encode(sha256), name.resolved()) - } else { - name.resolved().to_owned() - } - } -} - -/// An editable TUF repository, used to construct new ones. -pub(crate) struct OmicronRepoEditor { - editor: RepositoryEditor, - repo_path: Utf8PathBuf, - artifacts: ArtifactsDocument, - installinator_document: InstallinatorDocument, - - // Set of `TargetName::resolved()` names for every target that existed when - // the repo was opened. We use this to ensure we don't overwrite an existing - // target when adding new artifacts. - existing_target_names: BTreeSet, - // Set of (kind, hash) pairs for every artifact and deployment unit known to - // the repo. Used to ensure (kind, hash) pairs are unique within this repo. - existing_deployment_units: DeploymentUnitMapBuilder, -} - -impl OmicronRepoEditor { - pub(crate) async fn initialize( - repo_path: Utf8PathBuf, - root: SignedRole, - system_version: Version, - ) -> Result { - let metadata_dir = repo_path.join("metadata"); - let targets_dir = repo_path.join("targets"); - let root_path = metadata_dir - .join(format!("{}.root.json", root.signed().signed.version)); - - fs::create_dir_all(&metadata_dir)?; - fs::create_dir_all(&targets_dir)?; - fs::write(&root_path, root.buffer())?; - - let editor = RepositoryEditor::new(&root_path).await?; - - Ok(Self { - editor, - repo_path, - artifacts: ArtifactsDocument::empty(system_version.clone()), - installinator_document: InstallinatorDocument::empty( - system_version, - ), - existing_target_names: BTreeSet::new(), - existing_deployment_units: DeploymentUnitMapBuilder::new( - DeploymentUnitScope::Repository, - ), - }) - } - - /// Adds an artifact to the repository. - pub(crate) fn add_artifact( - &mut self, - new_artifact: &AddArtifact, - ) -> Result { - let target_name = format!( - "{}-{}-{}.tar.gz", - new_artifact.kind(), - new_artifact.name(), - new_artifact.version(), - ); - - let mut errors = Vec::new(); - - // Make sure we're not overwriting an existing target (either one that - // existed when we opened the repo, or one that's been added via this - // method) - if self.existing_target_names.contains(&target_name) { - errors.push(anyhow!( - "a target named {target_name} already exists in the repository", - )); - } - - // Start writing the target out to a temporary path, catching errors - // that might happen. - let targets_dir = self.repo_path.join("targets"); - let new_artifact = new_artifact.write_temp(&targets_dir)?; - - // Make sure we're not adding a new deployment unit with the same - // kind/hash as an existing one. - let res = match new_artifact.deployment_units() { - ArtifactDeploymentUnits::SingleUnit - | ArtifactDeploymentUnits::Unknown => { - // For single-unit artifacts, the artifact itself is the - // deployment unit. For unknown artifacts, we don't know, but - // treat them as single-unit. - self.existing_deployment_units.start_insert( - DeploymentUnitData { - name: new_artifact.name().to_owned(), - version: new_artifact.version().clone(), - kind: new_artifact.kind().clone(), - hash: new_artifact.digest(), - }, - ) - } - ArtifactDeploymentUnits::Composite { deployment_units } => { - // For composite artifacts, merge the deployment units. - self.existing_deployment_units - .start_bulk_insert(deployment_units.clone()) - } - }; - let new_units = match res { - Ok(units) => Some(units), - Err(error) => { - errors.push(anyhow!(error)); - None - } - }; - - let version = - match ArtifactVersion::new(new_artifact.version().to_string()) { - Ok(version) => Some(version), - Err(error) => { - errors.push( - anyhow!(error).context("invalid artifact version"), - ); - None - } - }; - - if !errors.is_empty() { - return Err(merge_anyhow_list(errors)); - } - - // --- - // No errors past this point. - // --- - - self.existing_target_names.insert(target_name.clone()); - self.artifacts.artifacts.push(Artifact { - name: new_artifact.name().to_owned(), - version: version.expect("version is None => errors handled above"), - kind: new_artifact.kind().clone(), - target: target_name, - }); - self.installinator_document - .artifacts - .extend(new_artifact.installinator_artifacts()); - // The host phase 2 image is part of the host image. - new_units.expect("new_units is None => errors handled above").commit(); - - new_artifact.finalize(&mut self.editor) - } - - /// Consumes self, signing the repository and writing out this repository to - /// disk. - pub(crate) async fn sign_and_finish( - mut self, - keys: Vec, - expiry: DateTime, - include_installinator_doc: bool, - ) -> Result<()> { - let targets_dir = self.repo_path.join("targets"); - - if include_installinator_doc { - let mut installinator_doc_writer = TargetWriter::new( - &targets_dir, - self.installinator_document.file_name(), - )?; - serde_json::to_writer_pretty( - &mut installinator_doc_writer, - &self.installinator_document, - )?; - installinator_doc_writer - .finish_write() - .finalize(&mut self.editor)?; - - // Add the installinator document in the artifacts.json if it's missing. - // - // TODO: our production users don't add artifacts incrementally to a TUF - // repo -- rather, they generate a manifest and create the TUF repo in a - // one-shot fashion. We should clean up any code we have to handle - // incremental updates of TUF repos, and remove this scan as part of - // that. - let system_version = self.artifacts.system_version.clone(); - let artifact_version = - ArtifactVersion::new(system_version.to_string()) - .expect("system versions are usable as artifact versions"); - if !self.artifacts.artifacts.iter().any(|artifact| { - artifact.kind.to_known() - == Some(KnownArtifactKind::InstallinatorDocument) - }) { - self.artifacts.artifacts.push(Artifact { - name: KnownArtifactKind::InstallinatorDocument.to_string(), - version: artifact_version, - kind: KnownArtifactKind::InstallinatorDocument.into(), - target: self.installinator_document.file_name(), - }); - } - } - - let mut artifacts_doc_writer = - TargetWriter::new(&targets_dir, ArtifactsDocument::FILE_NAME)?; - serde_json::to_writer_pretty( - &mut artifacts_doc_writer, - &self.artifacts, - )?; - artifacts_doc_writer.finish_write().finalize(&mut self.editor)?; - - update_versions(&mut self.editor, expiry)?; - - let signed = self - .editor - .sign(&crate::key::boxed_keys(keys)) - .await - .context("error signing keys")?; - signed - .write(self.repo_path.join("metadata")) - .await - .context("error writing repository")?; - Ok(()) - } -} - -fn update_versions( - editor: &mut RepositoryEditor, - expiry: DateTime, -) -> Result<()> { - let version = u64::try_from(Utc::now().timestamp()) - .and_then(NonZeroU64::try_from) - .expect("bad epoch"); - editor.snapshot_version(version); - editor.targets_version(version)?; - editor.timestamp_version(version); - editor.snapshot_expires(expiry); - editor.targets_expires(expiry)?; - editor.timestamp_expires(expiry); - Ok(()) -} - -#[cfg(test)] -mod tests { - use buf_list::BufList; - use camino_tempfile::Utf8TempDir; - use chrono::Days; - use dropshot::test_util::LogContext; - use dropshot::{ConfigLogging, ConfigLoggingIfExists, ConfigLoggingLevel}; - - use crate::assemble::{ - ArtifactDeploymentUnits, ArtifactManifest, OmicronRepoAssembler, - }; - use crate::{ArchiveExtractor, ArtifactSource}; - - use super::*; - - #[tokio::test] - async fn load_trusted() { - let log_config = ConfigLogging::File { - level: ConfigLoggingLevel::Trace, - path: "UNUSED".into(), - if_exists: ConfigLoggingIfExists::Fail, - }; - let logctx = LogContext::new( - "reject_artifacts_with_the_same_filename", - &log_config, - ); - - // Generate a "trusted" root and an "untrusted" root. - let expiry = Utc::now() + Days::new(1); - let trusted_key = Key::generate_ed25519().unwrap(); - let trusted_root = - crate::root::new_root(vec![trusted_key.clone()], expiry) - .await - .unwrap(); - let untrusted_key = Key::generate_ed25519().unwrap(); - let untrusted_root = - crate::root::new_root(vec![untrusted_key], expiry).await.unwrap(); - - // Generate a repository using the trusted root. - let tempdir = Utf8TempDir::new().unwrap(); - let archive_path = tempdir.path().join("repo.zip"); - let mut assembler = OmicronRepoAssembler::new( - &logctx.log, - ArtifactManifest::new_fake(), - vec![trusted_key], - expiry, - true, - archive_path.clone(), - ); - assembler.set_root_role(trusted_root.clone()); - assembler.build().await.unwrap(); - // And now that we've created an archive and cleaned up the build - // directory, immediately unarchive it... this is a bit silly, huh? - let repo_dir = tempdir.path().join("repo"); - ArchiveExtractor::from_path(&archive_path) - .unwrap() - .extract(&repo_dir) - .unwrap(); - - // If the trust store contains the root we generated the repo from, we - // should successfully load it. - for trust_store in [ - vec![trusted_root.buffer()], - vec![trusted_root.buffer(), untrusted_root.buffer()], - vec![untrusted_root.buffer(), trusted_root.buffer()], - vec![trusted_root.buffer(), trusted_root.buffer()], - vec![ - untrusted_root.buffer(), - untrusted_root.buffer(), - trusted_root.buffer(), - ], - ] { - OmicronRepo::load(&logctx.log, &repo_dir, trust_store) - .await - .unwrap(); - } - // If the trust store is empty, we should fail. - assert_eq!( - OmicronRepo::load(&logctx.log, &repo_dir, [] as [Vec; 0]) - .await - .unwrap_err() - .to_string(), - "trust store is empty" - ); - // If the trust store otherwise does not contain the root we generated - // the repo from, we should also fail. - for trust_store in [ - vec![untrusted_root.buffer()], - vec![untrusted_root.buffer(), untrusted_root.buffer()], - ] { - assert_eq!( - OmicronRepo::load(&logctx.log, &repo_dir, trust_store) - .await - .unwrap_err() - .to_string(), - "Failed to verify timestamp metadata: \ - Signature threshold of 1 not met for role timestamp \ - (0 valid signatures)" - ) - } - - logctx.cleanup_successful(); - } - - #[tokio::test] - async fn reject_artifacts_with_the_same_filename() { - let log_config = ConfigLogging::File { - level: ConfigLoggingLevel::Trace, - path: "UNUSED".into(), - if_exists: ConfigLoggingIfExists::Fail, - }; - let logctx = LogContext::new( - "reject_artifacts_with_the_same_filename", - &log_config, - ); - let tempdir = Utf8TempDir::new().unwrap(); - let keys = vec![Key::generate_ed25519().unwrap()]; - let expiry = Utc::now() + Days::new(1); - let root = crate::root::new_root(keys.clone(), expiry).await.unwrap(); - let mut repo = OmicronRepoEditor::initialize( - tempdir.path().to_owned(), - root, - "0.0.0".parse().unwrap(), - ) - .await - .unwrap(); - - // Targets are uniquely identified by their kind/name/version triple; - // trying to add two artifacts with identical triples should fail. - let kind = "test-kind"; - let name = "test-artifact-name"; - let version = "1.0.0"; - - repo.add_artifact(&AddArtifact::new( - kind.parse().unwrap(), - name.to_string(), - version.parse().unwrap(), - ArtifactSource::Memory(BufList::from("first")), - ArtifactDeploymentUnits::Unknown, - )) - .unwrap(); - - let err = repo - .add_artifact(&AddArtifact::new( - kind.parse().unwrap(), - name.to_string(), - version.parse().unwrap(), - ArtifactSource::Memory(BufList::from("second")), - ArtifactDeploymentUnits::Unknown, - )) - .unwrap_err(); - - println!("error: {:?}", err); - let err = err.to_string(); - - assert!(err.contains("a target named")); - assert!(err.contains(kind)); - assert!(err.contains(name)); - assert!(err.contains(version)); - assert!(err.contains("already exists")); - - logctx.cleanup_successful(); - } -} diff --git a/lib/src/schema.rs b/lib/src/schema.rs new file mode 100644 index 0000000..0f4e4c4 --- /dev/null +++ b/lib/src/schema.rs @@ -0,0 +1,27 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::BTreeMap; + +use semver::Version; +use serde::Deserialize; +use serde::Serialize; +use tufaceous_artifact::ArtifactVersion; + +#[derive(Debug, Deserialize, Serialize)] +pub(crate) struct ArtifactsSchema { + pub(crate) system_version: Version, + pub(crate) artifacts: Vec, +} + +impl ArtifactsSchema { + pub(crate) const TARGET_NAME: &str = "artifacts-v2.json"; +} + +#[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Hash)] +pub(crate) struct ArtifactSchema { + pub(crate) target_name: String, + pub(crate) version: ArtifactVersion, + pub(crate) tags: BTreeMap, +} diff --git a/lib/src/target.rs b/lib/src/target.rs deleted file mode 100644 index f6e4054..0000000 --- a/lib/src/target.rs +++ /dev/null @@ -1,106 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use std::collections::HashMap; -use std::io::Write; - -use anyhow::Result; -use camino::Utf8PathBuf; -use camino_tempfile::NamedUtf8TempFile; -use sha2::{Digest, Sha256}; -use tough::editor::RepositoryEditor; -use tough::schema::{Hashes, Target}; -use tufaceous_artifact::ArtifactHash; - -pub(crate) struct TargetWriter { - file: NamedUtf8TempFile, - targets_dir: Utf8PathBuf, - name: String, - length: u64, - hasher: Sha256, -} - -impl TargetWriter { - pub(crate) fn new( - targets_dir: impl Into, - name: impl Into, - ) -> Result { - let targets_dir = targets_dir.into(); - Ok(TargetWriter { - file: NamedUtf8TempFile::new_in(&targets_dir)?, - targets_dir, - name: name.into(), - length: 0, - hasher: Sha256::default(), - }) - } - - /// Marks that writing has been completed, though the file is still - /// temporary and not persisted yet. - /// - /// The main goal is to provide a way to obtain the hash of the file without - /// persisting it. - pub(crate) fn finish_write(self) -> TargetFinishWrite { - let digest = self.hasher.finalize(); - TargetFinishWrite { - file: self.file, - targets_dir: self.targets_dir, - name: self.name, - length: self.length, - digest: ArtifactHash(digest.into()), - } - } -} - -#[must_use = "the file is still temporary and must be finalized"] -pub(crate) struct TargetFinishWrite { - file: NamedUtf8TempFile, - targets_dir: Utf8PathBuf, - name: String, - length: u64, - digest: ArtifactHash, -} - -impl TargetFinishWrite { - pub(crate) fn digest(&self) -> ArtifactHash { - self.digest - } - - pub(crate) fn finalize( - self, - editor: &mut RepositoryEditor, - ) -> Result { - self.file.persist(self.targets_dir.join(format!( - "{}.{}", - hex::encode(self.digest.0), - self.name - )))?; - editor.add_target( - self.name, - Target { - length: self.length, - hashes: Hashes { - sha256: Vec::from(self.digest.0).into(), - _extra: HashMap::new(), - }, - custom: HashMap::new(), - _extra: HashMap::new(), - }, - )?; - Ok(self.digest) - } -} - -impl Write for TargetWriter { - fn write(&mut self, buf: &[u8]) -> std::io::Result { - let n = self.file.write(buf)?; - self.length += u64::try_from(n).unwrap(); - self.hasher.update(&buf[..n]); - Ok(n) - } - - fn flush(&mut self) -> std::io::Result<()> { - self.file.flush() - } -} diff --git a/lib/src/util.rs b/lib/src/util.rs new file mode 100644 index 0000000..d3b5ab1 --- /dev/null +++ b/lib/src/util.rs @@ -0,0 +1,93 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::io::Read; +use std::pin::Pin; +use std::task::Context; +use std::task::Poll; + +use camino::FromPathBufError; +use camino::Utf8Path; +use camino::Utf8PathBuf; +use flate2::read::GzDecoder; +use futures_util::Stream; +use tufaceous_brand_metadata::LayerInfo; +use tufaceous_brand_metadata::Metadata; + +use crate::error::Error; +use crate::error::ErrorKind; + +pub(crate) async fn read_zone_layer_info( + reader: R, + path: Utf8PathBuf, +) -> Result<(R, LayerInfo), Error> { + tokio::task::spawn_blocking(move || { + let mut archive = tar::Archive::new(GzDecoder::new(reader)); + let layer_info = Metadata::read_from_tar(&mut archive) + .and_then(|metadata| metadata.layer_info().cloned()) + .map_err(|source| ErrorKind::ReadZoneOxideJson { source, path })?; + Ok((archive.into_inner().into_inner(), layer_info)) + }) + .await? +} + +pub(crate) async fn read_dir( + path: impl Into, +) -> Result { + let path = path.into(); + match tokio::fs::read_dir(&path).await { + Ok(inner) => Ok(ReadDir { inner, path }), + Err(source) => Err(ErrorKind::ReadDir { source, path }.into()), + } +} + +pub(crate) struct ReadDir { + inner: tokio::fs::ReadDir, + path: Utf8PathBuf, +} + +impl Stream for ReadDir { + type Item = Result; + + fn poll_next( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + self.inner.poll_next_entry(cx).map(|result| { + result + .and_then(|option| option.map(DirEntry::new).transpose()) + .map_err(|source| { + ErrorKind::ReadDir { source, path: self.path.clone() } + .into() + }) + .transpose() + }) + } +} + +pub(crate) struct DirEntry { + path: Utf8PathBuf, +} + +impl DirEntry { + fn new(inner: tokio::fs::DirEntry) -> Result { + let path = + inner.path().try_into().map_err(FromPathBufError::into_io_error)?; + Ok(Self { path }) + } + + pub(crate) fn path(&self) -> &Utf8Path { + &self.path + } + + pub(crate) fn into_path(self) -> Utf8PathBuf { + self.path + } + + pub(crate) fn file_name(&self) -> &str { + self.path + .file_name() + .expect("path created through DirEntry must have a filename") + } +} diff --git a/lib/src/utils.rs b/lib/src/utils.rs deleted file mode 100644 index c2ae66b..0000000 --- a/lib/src/utils.rs +++ /dev/null @@ -1,60 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use std::fmt::Write; - -use anyhow::anyhow; -use indent_write::fmt::IndentWriter; - -/// A temporary hack to convert a list of anyhow errors into a single -/// `anyhow::Error`. If no errors are provided, panic (this should be handled -/// at a higher level). -/// -/// Eventually we should gain first-class support for representing errors as -/// trees, but this will do for now. -pub(crate) fn merge_anyhow_list(errors: I) -> anyhow::Error -where - I: IntoIterator, -{ - let mut iter = errors.into_iter().peekable(); - // How many errors are there? - let Some(first_error) = iter.next() else { - // No errors: panic. - panic!("error_list_to_anyhow called with no errors"); - }; - - if iter.peek().is_none() { - // One error. - return first_error; - } - - // Multiple errors. - let mut out = String::new(); - let mut nerrors = 0; - for error in std::iter::once(first_error).chain(iter) { - if nerrors > 0 { - // Separate errors with a newline (we want there to not be a - // trailing newline to match anyhow generally). - writeln!(&mut out).unwrap(); - } - nerrors += 1; - let mut current: &dyn std::error::Error = error.as_ref(); - - let mut writer = IndentWriter::new_skip_initial(" ", &mut out); - write!(writer, "Error: {current}").unwrap(); - - while let Some(cause) = current.source() { - // This newline is not part of the `IndentWriter`'s output so that - // it is unaffected by the indent logic. - writeln!(&mut out).unwrap(); - - // The spaces align the causes with the "Error: " above. - let mut writer = - IndentWriter::new_skip_initial(" ", &mut out); - write!(writer, " - {cause}").unwrap(); - current = cause; - } - } - anyhow!(out).context(format!("{nerrors} errors encountered")) -} diff --git a/lib/src/zip_transport.rs b/lib/src/zip_transport.rs new file mode 100644 index 0000000..52d8c0a --- /dev/null +++ b/lib/src/zip_transport.rs @@ -0,0 +1,457 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::HashMap; +use std::fmt::Debug; +use std::fs::File; +use std::io::Cursor; +use std::io::Read; +use std::pin::Pin; +use std::sync::Arc; + +use bytes::Bytes; +use bytes::BytesMut; +use camino::Utf8PathBuf; +use flate2::read::DeflateDecoder; +use futures_util::FutureExt; +use futures_util::Stream; +use futures_util::StreamExt; +use futures_util::stream; +use rawzip::CompressionMethod; +use rawzip::FileReader; +use rawzip::ReaderAt; +use rawzip::ZipArchive; +use rawzip::ZipArchiveEntryWayfinder; +use rawzip::ZipReader; +use rawzip::ZipVerifier; +use rawzip::path::RawPath; +use rawzip::path::ZipFilePath; +use slog::Logger; +use slog::warn; +use tokio::sync::mpsc; +use tough::Transport; +use tough::TransportError; +use tough::TransportErrorKind; +use tough::async_trait; +use url::Url; + +use crate::error::Error; +use crate::error::ErrorKind; + +const MAX_SYMLINK_TARGET_LEN: usize = 1024; +const MAX_SYMLINK_TRAVERSAL: usize = 8; + +/// Implementation of [`tough::Transport`] that operates on a Zip archive. +/// +/// URLs used with this transport must use the `zip:///` protocol. For example, +/// if your metadata is found inside `repo/metadata` within the archive, use +/// `zip:///repo/metadata/` as the metadata base URL. +/// +/// Convenience methods for setting the correct transport +/// and base URLs for loading Tufaceous-generated +/// repositories are [`RepositoryLoader::load_zip_slice`] and +/// [`RepositoryLoader::load_zip_file`]. +/// +/// [`RepositoryLoader::load_zip_slice`]: [`crate::RepositoryLoader::load_zip_slice`] +/// [`RepositoryLoader::load_zip_file`]: [`crate::RepositoryLoader::load_zip_file`] +#[derive(Debug)] +pub struct ZipTransport { + inner: Arc>, +} + +#[derive(Debug)] +struct Inner { + archive: ZipArchive, + entries: HashMap, +} + +#[derive(Debug, Clone, Copy)] +enum Entry { + File(EntryData), + Symlink(EntryData), + Dir, + Duplicate, +} + +#[derive(Debug, Clone, Copy)] +struct EntryData { + compression_method: CompressionMethod, + wayfinder: ZipArchiveEntryWayfinder, +} + +// Manually implemented, as the derive macro adds an unnecessary `T: Clone`. +impl Clone for ZipTransport { + fn clone(&self) -> Self { + Self { inner: self.inner.clone() } + } +} + +impl + Debug + Send + Sync + 'static> ZipTransport> { + pub fn from_slice(data: T, log: &Logger) -> Result { + let archive = ZipArchive::from_slice(data).upgrade(None)?; + Self::from_impl_blocking(archive.into_zip_archive(), None, None, log) + } +} + +impl ZipTransport { + pub async fn from_file( + archive_path: impl Into, + log: &Logger, + ) -> Result { + let archive_path = archive_path.into(); + let log = log.clone(); + tokio::task::spawn_blocking(move || { + let archive_path = archive_path; + let file = match File::open(&archive_path) { + Ok(file) => file, + Err(source) => { + return Err(ErrorKind::OpenFile { + source, + path: archive_path, + } + .into()); + } + }; + let mut buffer = vec![0; rawzip::RECOMMENDED_BUFFER_SIZE]; + let archive_path = Some(archive_path); + let archive = ZipArchive::from_file(file, &mut buffer) + .upgrade(archive_path.as_ref())?; + Self::from_impl_blocking(archive, archive_path, Some(buffer), &log) + }) + .await? + } +} + +impl ZipTransport { + fn from_impl_blocking( + archive: ZipArchive, + archive_path: Option, + buffer: Option>, + log: &Logger, + ) -> Result { + let mut buffer = + buffer.unwrap_or_else(|| vec![0; rawzip::RECOMMENDED_BUFFER_SIZE]); + let expected = archive.entries_hint(); + let mut actual: u64 = 0; + let mut all_entries = Vec::new(); + let mut entries = HashMap::new(); + let mut records = archive.entries(&mut buffer); + while let Some(record) = + records.next_entry().upgrade(archive_path.as_ref())? + { + actual += 1; + all_entries.push(( + record.wayfinder(), + record.file_path().as_bytes().to_vec(), + )); + + let Some(url) = path_to_url(record.file_path(), log) else { + continue; + }; + match entries.entry(url) { + std::collections::hash_map::Entry::Occupied(mut e) => { + e.insert(Entry::Duplicate); + } + std::collections::hash_map::Entry::Vacant(e) => { + if record.is_dir() { + e.insert(Entry::Dir); + } else { + let data = EntryData { + compression_method: record.compression_method(), + wayfinder: record.wayfinder(), + }; + if record.mode().is_symlink() { + e.insert(Entry::Symlink(data)); + } else { + e.insert(Entry::File(data)); + } + } + } + } + } + + if expected != actual { + return Err(ErrorKind::ZipEntryCount { + expected, + actual, + archive_path, + } + .into()); + } + + let mut ranges = Vec::new(); + for (wayfinder, raw_path) in all_entries { + let entry = + archive.get_entry(wayfinder).upgrade(archive_path.as_ref())?; + let header = entry + .local_header(&mut buffer) + .upgrade(archive_path.as_ref())?; + if raw_path != header.file_path().as_bytes() { + return Err(ErrorKind::ZipPathMismatch { + central: raw_path, + local: header.file_path().as_bytes().to_vec(), + archive_path, + } + .into()); + } + + let (start, end) = entry.compressed_data_range(); + ranges.push((start..end, raw_path)); + // Check that no file ranges overlap with or come after the central + // directory. + if end > archive.directory_offset() { + return Err(ErrorKind::ZipRangeOverrun { + file_path: header.file_path().as_bytes().to_vec(), + data_range: start..end, + archive_path, + } + .into()); + } + } + // Check that no file ranges overlap with each other. + ranges.sort_by_key(|(range, _)| range.start); + for window in ranges.windows(2) { + let [(earlier, earlier_path), (later, later_path)] = window else { + panic!("slice::windows is broken") + }; + if earlier.end > later.start { + return Err(ErrorKind::ZipOverlappingRanges { + earlier_path: earlier_path.clone(), + earlier: earlier.clone(), + later_path: later_path.clone(), + later: later.clone(), + archive_path, + } + .into()); + } + } + + Ok(Self { inner: Arc::new(Inner { archive, entries }) }) + } + + fn reader_blocking( + &self, + EntryData { compression_method, wayfinder }: EntryData, + ) -> Result, ZipTransportError> { + let entry = self.inner.archive.get_entry(wayfinder)?; + let reader = entry.reader(); + match compression_method { + CompressionMethod::Store => { + Ok(Reader::Store(entry.verifying_reader(reader))) + } + CompressionMethod::Deflate => Ok(Reader::Deflate( + entry.verifying_reader(DeflateDecoder::new(reader)), + )), + other => Err(ZipTransportError::CompressionMethod(other)), + } + } + + fn stream( + self, + entry_data: EntryData, + url: Url, + ) -> impl Stream> { + let (tx, mut rx) = mpsc::channel(1); + let task = tokio::task::spawn_blocking(move || { + let mut reader = self.reader_blocking(entry_data)?; + let mut buf = BytesMut::zeroed(8192); + while let n = reader.read(&mut buf)? + && n > 0 + { + buf.truncate(n); + let Ok(()) = tx.blocking_send(buf.split().freeze()) else { + break; + }; + if buf.capacity() == 0 { + buf.reserve(8192); + } + buf.resize(buf.capacity(), 0); + } + Ok::<_, ZipTransportError>(()) + }); + stream::poll_fn(move |cx| rx.poll_recv(cx)).map(Ok).chain( + task.into_stream().filter_map(move |result| { + let error = match result { + Ok(Ok(())) => return std::future::ready(None), + Ok(Err(error)) => error, + Err(join_error) => join_error.into(), + }; + std::future::ready(Some(Err(error.upgrade(url.clone())))) + }), + ) + } + + async fn read_symlink_target( + self, + entry_data: EntryData, + base: &Url, + ) -> Result { + let target = tokio::task::spawn_blocking(move || { + let mut reader = self.reader_blocking(entry_data)?; + let mut v = vec![0; MAX_SYMLINK_TARGET_LEN]; + let len = + std::io::copy(&mut reader, &mut Cursor::new(v.as_mut_slice()))?; + v.truncate(len.try_into().unwrap()); + String::from_utf8(v).map_err(|source| { + ZipTransportError::SymlinkUtf8(source.utf8_error()) + }) + }) + .await??; + base.join(&target).map_err(|source| ZipTransportError::UrlJoin { + source, + url: target, + base: base.to_string(), + }) + } +} + +#[async_trait] +impl Transport + for ZipTransport +{ + async fn fetch( + &self, + original_url: Url, + ) -> Result< + Pin> + Send>>, + TransportError, + > { + let mut url = original_url.clone(); + if url.scheme() != "zip" { + return Err(TransportError::new( + TransportErrorKind::UnsupportedUrlScheme, + url, + )); + } + for _ in 0..MAX_SYMLINK_TRAVERSAL { + match self.inner.entries.get(&url).copied() { + Some(Entry::File(entry_data)) => { + return Ok(Box::pin(self.clone().stream(entry_data, url))); + } + Some(Entry::Symlink(entry_data)) => { + url = self + .clone() + .read_symlink_target(entry_data, &url) + .await + .map_err(|error| error.upgrade(url))?; + } + Some(Entry::Dir) => { + return Err(ZipTransportError::IsADirectory.upgrade(url)); + } + Some(Entry::Duplicate) => { + return Err(ZipTransportError::Duplicate.upgrade(url)); + } + None => { + return Err(ZipTransportError::FileNotFound.upgrade(url)); + } + } + } + Err(ZipTransportError::SymlinkTraversalLimit.upgrade(original_url)) + } +} + +enum Reader<'a, T: ReaderAt> { + Store(ZipVerifier, &'a T>), + Deflate(ZipVerifier>, &'a T>), +} + +impl Read for Reader<'_, T> { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + match self { + Reader::Store(reader) => reader.read(buf), + Reader::Deflate(reader) => reader.read(buf), + } + } +} + +trait ResultExt { + fn upgrade(self, archive_path: Option<&Utf8PathBuf>) -> Result; +} + +impl ResultExt for Result { + fn upgrade(self, archive_path: Option<&Utf8PathBuf>) -> Result { + match self { + Ok(v) => Ok(v), + Err(source) => Err(ErrorKind::ReadZip { + source, + archive_path: archive_path.cloned(), + })?, + } + } +} + +/// Possible source errors of [`ZipTransport::fetch`]. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum ZipTransportError { + #[error(transparent)] + Io(#[from] std::io::Error), + #[error(transparent)] + Join(#[from] tokio::task::JoinError), + #[error("failed to join {url} onto {base}")] + UrlJoin { source: url::ParseError, url: String, base: String }, + #[error(transparent)] + Zip(#[from] rawzip::Error), + + #[error("unsupported compression method {0:?}")] + CompressionMethod(CompressionMethod), + #[error("multiple entries found for file")] + Duplicate, + #[error("file not found")] + FileNotFound, + #[error("is a directory")] + IsADirectory, + #[error("reached symlink traversal limit")] + SymlinkTraversalLimit, + #[error("symlink target is not valid UTF-8")] + SymlinkUtf8(std::str::Utf8Error), +} + +impl ZipTransportError { + fn upgrade(self, url: Url) -> TransportError { + let kind = match &self { + ZipTransportError::Io(_) + | ZipTransportError::Join(_) + | ZipTransportError::Zip(_) => TransportErrorKind::Other, + + ZipTransportError::UrlJoin { .. } + | ZipTransportError::CompressionMethod(_) + | ZipTransportError::Duplicate + | ZipTransportError::FileNotFound + | ZipTransportError::IsADirectory + | ZipTransportError::SymlinkTraversalLimit + | ZipTransportError::SymlinkUtf8(_) => { + TransportErrorKind::FileNotFound + } + }; + TransportError::new_with_cause(kind, url, self) + } +} + +fn path_to_url(path: ZipFilePath>, log: &Logger) -> Option { + path.try_normalize() + .inspect_err(|err| { + warn!( + log, + "ignoring invalid path in zip archive"; + "path" => path.as_bytes().escape_ascii().to_string(), + "error" => err.to_string(), + ); + }) + .ok() + .and_then(|path| { + Url::parse("zip:///") + .unwrap() + .join(path.as_str()) + .inspect_err(|err| { + warn!( + log, + "ignoring invalid path in zip archive"; + "path" => path.as_str(), + "error" => err.to_string(), + ); + }) + .ok() + }) +} diff --git a/lib/src/zip_writer.rs b/lib/src/zip_writer.rs new file mode 100644 index 0000000..636030e --- /dev/null +++ b/lib/src/zip_writer.rs @@ -0,0 +1,199 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::io::Write; + +use atomicwrites::AtomicFile; +use atomicwrites::OverwriteBehavior; +use bytes::Bytes; +use camino::Utf8Path; +use flate2::Compression; +use flate2::write::DeflateEncoder; +use rawzip::CompressionMethod; +use rawzip::ZipArchiveWriter; +use rawzip::time::ZipDateTime; +use tokio::sync::mpsc; +use tokio::task::JoinHandle; + +pub(crate) struct ZipWriter { + task: JoinHandle>, + file_tx: mpsc::Sender, +} + +impl ZipWriter { + pub(crate) fn new(writer: W) -> Self { + let (file_tx, file_rx) = mpsc::channel(1); + let task = + tokio::task::spawn_blocking(move || write_task(writer, file_rx)); + Self { task, file_tx } + } +} + +impl ZipWriter<()> { + pub(crate) fn create(path: impl AsRef) -> Self { + let file = + AtomicFile::new(path.as_ref(), OverwriteBehavior::AllowOverwrite); + let (file_tx, file_rx) = mpsc::channel(1); + let task = tokio::task::spawn_blocking(move || { + file.write(|file| write_task(file, file_rx).map(|_| ())).map_err( + |error| match error { + atomicwrites::Error::User(error) => error, + atomicwrites::Error::Internal(error) => error.into(), + }, + ) + }); + Self { task, file_tx } + } +} + +impl ZipWriter { + pub(crate) fn new_file( + &mut self, + name: impl Into, + ) -> ZipFileBuilder<'_, W> { + let (bytes_tx, bytes_rx) = mpsc::channel(1); + ZipFileBuilder { + writer: self, + inner: ZipFile { + name: name.into(), + compression: Compression::none(), + last_modified: None, + unix_permissions: None, + bytes_rx, + }, + bytes_tx, + } + } + + pub(crate) async fn finish(self) -> Result { + drop(self.file_tx); + self.task.await.map_err(std::io::Error::from)? + } +} + +pub(crate) struct ZipFileBuilder<'a, W> { + writer: &'a mut ZipWriter, + inner: ZipFile, + bytes_tx: mpsc::Sender, +} + +impl<'a, W> ZipFileBuilder<'a, W> { + pub(crate) fn compression(mut self, compression: Compression) -> Self { + self.inner.compression = compression; + self + } + + pub(crate) fn last_modified(mut self, last_modified: ZipDateTime) -> Self { + self.inner.last_modified = Some(last_modified); + self + } + + pub(crate) fn unix_permissions(mut self, unix_permissions: u32) -> Self { + self.inner.unix_permissions = Some(unix_permissions); + self + } + + pub(crate) async fn start(self) -> Result, SendError> { + self.writer.file_tx.send(self.inner).await?; + Ok(ZipDataWriter { _writer: self.writer, bytes_tx: self.bytes_tx }) + } +} + +pub(crate) struct ZipDataWriter<'a, W> { + _writer: &'a mut ZipWriter, + bytes_tx: mpsc::Sender, +} + +impl ZipDataWriter<'_, W> { + pub(crate) async fn write( + &mut self, + bytes: Bytes, + ) -> Result<(), SendError> { + self.bytes_tx.send(bytes).await?; + Ok(()) + } +} + +struct ZipFile { + name: String, + compression: Compression, + last_modified: Option, + unix_permissions: Option, + bytes_rx: mpsc::Receiver, +} + +fn write_task( + writer: W, + mut file_rx: mpsc::Receiver, +) -> Result { + let mut archive = ZipArchiveWriter::new(writer); + while let Some(mut file) = file_rx.blocking_recv() { + let mut builder = archive.new_file(&file.name); + if file.compression != Compression::none() { + builder = builder.compression_method(CompressionMethod::Deflate); + } + if let Some(last_modified) = file.last_modified { + builder = builder.last_modified(last_modified); + } + if let Some(unix_permissions) = file.unix_permissions { + builder = builder.unix_permissions(unix_permissions); + } + let (mut entry, config) = builder.start()?; + let encoder = ZipEncoder::new(&mut entry, file.compression); + let mut writer = config.wrap(encoder); + while let Some(bytes) = file.bytes_rx.blocking_recv() { + writer.write_all(&bytes)?; + } + let (encoder, output) = writer.finish()?; + encoder.finish()?; + entry.finish(output)?; + } + archive.finish() +} + +enum ZipEncoder { + Store(W), + Deflate(DeflateEncoder), +} + +impl ZipEncoder { + fn new(writer: W, compression: Compression) -> Self { + if compression == Compression::none() { + Self::Store(writer) + } else { + Self::Deflate(DeflateEncoder::new(writer, compression)) + } + } + + fn finish(self) -> std::io::Result { + match self { + Self::Store(writer) => Ok(writer), + Self::Deflate(encoder) => encoder.finish(), + } + } +} + +impl Write for ZipEncoder { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + match self { + ZipEncoder::Store(writer) => writer.write(buf), + ZipEncoder::Deflate(writer) => writer.write(buf), + } + } + + fn flush(&mut self) -> std::io::Result<()> { + match self { + ZipEncoder::Store(writer) => writer.flush(), + ZipEncoder::Deflate(writer) => writer.flush(), + } + } +} + +pub(crate) struct SendError; + +impl From> for SendError { + fn from(_value: mpsc::error::SendError) -> Self { + Self + } +} diff --git a/lib/tests/tests.rs b/lib/tests/tests.rs new file mode 100644 index 0000000..4f102c6 --- /dev/null +++ b/lib/tests/tests.rs @@ -0,0 +1,71 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use chrono::Utc; +use futures_util::TryStreamExt; +use semver::Version; +use tufaceous::RepositoryLoader; +use tufaceous::TrustStoreBehavior; +use tufaceous::edit::RepositoryEditor; +use tufaceous::error::Error; + +const VERSION: Version = Version::new(1, 0, 0); + +#[tokio::test] +async fn it_works() -> Result<(), Error> { + let log = slog::Logger::root(slog::Discard, slog::o!()); + let zip = RepositoryEditor::fake(VERSION)? + .finish() + .await? + .generate_root() + .sign() + .await? + .write_zip(Vec::new(), Utc::now()) + .await?; + let repo = RepositoryLoader::new() + .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .load_zip_buffer(zip, &log) + .await?; + for artifact in repo.artifacts() { + repo.read_target(&artifact.target_name) + .await? + .map_ok(|bytes| bytes.to_vec()) + .try_concat() + .await?; + } + Ok(()) +} + +#[tokio::test] +async fn empty_artifact() -> Result<(), Error> { + let log = slog::Logger::root(slog::Discard, slog::o!()); + let zip = RepositoryEditor::new(VERSION) + .add_fake_artifact( + "empty.img".to_owned(), + "1.0.0".parse().unwrap(), + tufaceous_artifact::KnownArtifactTags::InstallinatorDocument {}, + 0, + ) + .finish() + .await? + .generate_root() + .sign() + .await? + .write_zip(Vec::new(), Utc::now()) + .await?; + let repo = RepositoryLoader::new() + .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .load_zip_buffer(zip, &log) + .await?; + let artifacts = repo.artifacts().iter().collect::>(); + assert_eq!(artifacts.len(), 1); + let data = repo + .read_target(&artifacts[0].target_name) + .await? + .map_ok(|bytes| bytes.to_vec()) + .try_concat() + .await?; + assert!(data.is_empty()); + Ok(()) +} diff --git a/rust-toolchain.toml b/rust-toolchain.toml index ec68e8a..9788fa4 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -2,5 +2,5 @@ # We choose a specific toolchain (rather than "stable") for repeatability. The # intent is to keep this up-to-date with recently-released stable Rust. -channel = "1.85.0" +channel = "1.91.1" profile = "default" From ece6da94ee214f4434cbacddfda6eae8d7e7ecac Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Wed, 17 Dec 2025 18:20:28 -0800 Subject: [PATCH 002/137] define and wire up metadata --- artifact/src/artifact.rs | 9 +++++++++ lib/src/edit/editor.rs | 17 +++++++++++++++++ lib/src/repo.rs | 9 ++++++++- lib/src/repo/v1.rs | 2 ++ lib/src/schema.rs | 3 +++ lib/tests/tests.rs | 20 ++++++++++++++++++++ 6 files changed, 59 insertions(+), 1 deletion(-) diff --git a/artifact/src/artifact.rs b/artifact/src/artifact.rs index 27f11aa..03024bc 100644 --- a/artifact/src/artifact.rs +++ b/artifact/src/artifact.rs @@ -37,6 +37,15 @@ impl Artifacts { Self::from_iter(iter) } + pub fn len(&self) -> usize { + self.inner.values().map(Vec::len).sum() + } + + pub fn is_empty(&self) -> bool { + debug_assert_eq!(self.len(), 0); + self.inner.is_empty() + } + pub fn insert(&mut self, artifact: Artifact) { self.inner.entry(artifact.known_tags()).or_default().push(artifact); } diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index e2157cf..75652c8 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -45,6 +45,7 @@ pub struct RepositoryEditor<'a> { system_version: Version, targets: HashMap>>, artifacts: HashMap>, + metadata: BTreeMap, generate_installinator_document: bool, } @@ -54,6 +55,7 @@ impl<'a> RepositoryEditor<'a> { system_version, targets: HashMap::new(), artifacts: HashMap::new(), + metadata: BTreeMap::new(), generate_installinator_document: false, } } @@ -319,6 +321,7 @@ impl<'a> RepositoryEditor<'a> { let document = ArtifactsSchema { system_version: self.system_version, artifacts: artifacts.into_values().collect(), + metadata: self.metadata, }; let target = BytesSource::json(&document) .map_err(ErrorKind::SerializeArtifacts)? @@ -468,6 +471,20 @@ impl<'a> RepositoryEditor<'a> { ); Ok(self) } + + pub async fn insert_metadata( + mut self, + key: String, + value: serde_json::Value, + ) -> Self { + self.metadata.insert(key, value); + self + } + + pub async fn remove_metadata(mut self, key: &str) -> Self { + self.metadata.remove(key); + self + } } #[derive(Default)] diff --git a/lib/src/repo.rs b/lib/src/repo.rs index 11e6004..e30a9c6 100644 --- a/lib/src/repo.rs +++ b/lib/src/repo.rs @@ -4,6 +4,7 @@ mod v1; +use std::collections::BTreeMap; use std::collections::HashMap; use std::pin::Pin; @@ -34,6 +35,7 @@ pub struct Repository { system_version: Version, trust_root: Vec, artifacts: Artifacts, + metadata: BTreeMap, v1_unpacked: Option, } @@ -48,7 +50,7 @@ impl Repository { trust_root: Vec, v1_compatibility: bool, ) -> Result { - let Some(ArtifactsSchema { system_version, artifacts }) = + let Some(ArtifactsSchema { system_version, artifacts, metadata }) = read_target_json(&repo, ArtifactsSchema::TARGET_NAME).await? else { if v1_compatibility @@ -76,6 +78,7 @@ impl Repository { trust_root, system_version, artifacts, + metadata, v1_unpacked: None, }) } @@ -96,6 +99,10 @@ impl Repository { &self.artifacts } + pub fn metadata(&self) -> &BTreeMap { + &self.metadata + } + pub fn is_v1(&self) -> bool { self.v1_unpacked.is_some() } diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs index 85989dd..48f1d08 100644 --- a/lib/src/repo/v1.rs +++ b/lib/src/repo/v1.rs @@ -2,6 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use std::collections::BTreeMap; use std::collections::HashMap; use std::io::BufRead; use std::io::Read; @@ -273,6 +274,7 @@ pub(crate) async fn from_loaded( system_version, trust_root, artifacts: Artifacts::new(artifacts), + metadata: BTreeMap::new(), v1_unpacked: Some(Unpacked { entries }), })) } diff --git a/lib/src/schema.rs b/lib/src/schema.rs index 0f4e4c4..b13c965 100644 --- a/lib/src/schema.rs +++ b/lib/src/schema.rs @@ -12,7 +12,10 @@ use tufaceous_artifact::ArtifactVersion; #[derive(Debug, Deserialize, Serialize)] pub(crate) struct ArtifactsSchema { pub(crate) system_version: Version, + #[serde(default)] pub(crate) artifacts: Vec, + #[serde(default)] + pub(crate) metadata: BTreeMap, } impl ArtifactsSchema { diff --git a/lib/tests/tests.rs b/lib/tests/tests.rs index 4f102c6..8d846c4 100644 --- a/lib/tests/tests.rs +++ b/lib/tests/tests.rs @@ -37,6 +37,26 @@ async fn it_works() -> Result<(), Error> { Ok(()) } +#[tokio::test] +async fn no_artifacts() -> Result<(), Error> { + let log = slog::Logger::root(slog::Discard, slog::o!()); + let zip = RepositoryEditor::new(VERSION) + .finish() + .await? + .generate_root() + .sign() + .await? + .write_zip(Vec::new(), Utc::now()) + .await?; + let repo = RepositoryLoader::new() + .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .load_zip_buffer(zip, &log) + .await?; + assert_eq!(repo.artifacts().len(), 0); + assert!(repo.artifacts().is_empty()); + Ok(()) +} + #[tokio::test] async fn empty_artifact() -> Result<(), Error> { let log = slog::Logger::root(slog::Discard, slog::o!()); From bc122f27252ddb2f53b92d55ab094435c4fc7ddf Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 18 Dec 2025 22:53:26 -0800 Subject: [PATCH 003/137] editor refactor; allow the CLI to guess kinds --- Cargo.lock | 233 +++++++++++- Cargo.toml | 4 +- artifact/src/hubris.rs | 9 +- artifact/src/installinator.rs | 3 +- artifact/src/lib.rs | 2 + artifact/src/sign.rs | 122 +++++++ artifact/src/tags.rs | 10 +- bin/src/assemble.rs | 105 ------ bin/src/build.rs | 42 +++ bin/src/main.rs | 42 +-- lib/Cargo.toml | 2 + lib/src/edit/editor.rs | 642 ++++++++++++++++++++++++---------- lib/src/edit/sign.rs | 16 +- lib/src/edit/source.rs | 66 ++-- lib/src/error.rs | 4 + lib/src/loader.rs | 4 +- lib/src/util.rs | 9 +- lib/src/zip_transport.rs | 3 +- lib/src/zip_writer.rs | 8 +- lib/tests/tests.rs | 4 +- 20 files changed, 969 insertions(+), 361 deletions(-) create mode 100644 artifact/src/sign.rs delete mode 100644 bin/src/assemble.rs create mode 100644 bin/src/build.rs diff --git a/Cargo.lock b/Cargo.lock index 2f87fa2..a83a3cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -401,6 +401,33 @@ dependencies = [ "windows-link", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "clang-sys" version = "1.8.1" @@ -547,6 +574,12 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.1.6" @@ -578,6 +611,41 @@ dependencies = [ "syn 2.0.96", ] +[[package]] +name = "darling" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.96", +] + +[[package]] +name = "darling_macro" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.96", +] + [[package]] name = "der" version = "0.7.9" @@ -609,6 +677,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ "powerfmt", + "serde", ] [[package]] @@ -913,6 +982,17 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy 0.8.27", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -1131,6 +1211,12 @@ dependencies = [ "syn 2.0.96", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.1.0" @@ -1160,6 +1246,7 @@ checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown 0.12.3", + "serde", ] [[package]] @@ -1170,6 +1257,7 @@ checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" dependencies = [ "equivalent", "hashbrown 0.15.1", + "serde", ] [[package]] @@ -1734,6 +1822,21 @@ dependencies = [ "rand_core", ] +[[package]] +name = "rats-corim" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/rats-corim.git?branch=main#f0d5d5168d3d31487a56df32c676b0c6240bcc6b" +dependencies = [ + "ciborium", + "ciborium-io", + "clap", + "hex", + "serde", + "serde_with", + "strum", + "thiserror 2.0.12", +] + [[package]] name = "rawzip" version = "0.4.2" @@ -1749,6 +1852,26 @@ dependencies = [ "bitflags", ] +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] + [[package]] name = "regex" version = "1.11.1" @@ -1897,6 +2020,12 @@ dependencies = [ "untrusted 0.9.0", ] +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + [[package]] name = "rusty-fork" version = "0.3.0" @@ -1936,6 +2065,30 @@ dependencies = [ "serde_json", ] +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9558e172d4e8533736ba97870c4b2cd63f84b382a3d6eb063da41b91cce17289" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + [[package]] name = "schemars_derive" version = "0.8.22" @@ -2002,7 +2155,7 @@ dependencies = [ [[package]] name = "serde_human_bytes" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/serde_human_bytes?branch=main#0a09794501b6208120528c3b457d5f3a8cb17424" +source = "git+https://github.com/oxidecomputer/serde_human_bytes.git?branch=main#0a09794501b6208120528c3b457d5f3a8cb17424" dependencies = [ "hex", "serde", @@ -2038,6 +2191,38 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c522100790450cf78eeac1507263d0a350d4d5b30df0c8e1fe051a10c22b376e" +dependencies = [ + "base64 0.22.1", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.7.1", + "schemars 0.9.0", + "schemars 1.1.0", + "serde", + "serde_derive", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327ada00f7d64abaac1e55a6911e90cf665aa051b9a561c7006c157f4633135e" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.96", +] + [[package]] name = "sha2" version = "0.10.8" @@ -2238,6 +2423,28 @@ dependencies = [ "syn 2.0.96", ] +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.96", +] + [[package]] name = "subtle" version = "2.6.1" @@ -2619,10 +2826,12 @@ dependencies = [ "camino", "camino-tempfile", "chrono", + "ciborium", "flate2", "futures-util", "hex", "hubtools", + "rats-corim", "rawzip", "semver", "serde", @@ -2649,7 +2858,7 @@ dependencies = [ "hubtools", "proptest", "regex", - "schemars", + "schemars 0.8.22", "serde", "serde_human_bytes", "serde_json", @@ -3093,6 +3302,15 @@ dependencies = [ "zerocopy-derive 0.7.35", ] +[[package]] +name = "zerocopy" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" +dependencies = [ + "zerocopy-derive 0.8.27", +] + [[package]] name = "zerocopy-derive" version = "0.6.6" @@ -3115,6 +3333,17 @@ dependencies = [ "syn 2.0.96", ] +[[package]] +name = "zerocopy-derive" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] + [[package]] name = "zerofrom" version = "0.1.4" diff --git a/Cargo.toml b/Cargo.toml index 62f04e8..8967453 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ bytes = "1.10.0" camino = "1.1.9" camino-tempfile = "1.1.1" chrono = { version = "0.4.40", default-features = false, features = ["std"] } +ciborium = "0.2.2" clap = { version = "4.5.31", features = ["cargo", "derive", "wrap_help"] } daft = { version = "0.1.1", features = ["derive"] } expectorate = "1.1.0" @@ -31,12 +32,13 @@ futures-util = "0.3.31" hex = "0.4.3" hubtools = { git = "https://github.com/oxidecomputer/hubtools.git", branch = "main" } proptest = "1.5.0" +rats-corim = { git = "https://github.com/oxidecomputer/rats-corim.git", branch = "main" } rawzip = "0.4.2" regex = "1.11.1" schemars = "0.8.21" semver = { version = "1.0.25", features = ["serde"] } serde = { version = "1.0.218", features = ["derive"] } -serde_human_bytes = { git = "https://github.com/oxidecomputer/serde_human_bytes", branch = "main" } +serde_human_bytes = { git = "https://github.com/oxidecomputer/serde_human_bytes.git", branch = "main" } serde_json = "1.0.139" sha2 = "0.10.8" slog = "2.7.0" diff --git a/artifact/src/hubris.rs b/artifact/src/hubris.rs index 255f373..8c98d01 100644 --- a/artifact/src/hubris.rs +++ b/artifact/src/hubris.rs @@ -9,6 +9,7 @@ use hubtools::CabooseError; use crate::KnownArtifactTags; use crate::RotSlot; +use crate::Sign; impl KnownArtifactTags { pub fn from_rot_caboose( @@ -17,7 +18,7 @@ impl KnownArtifactTags { ) -> Result { Ok(KnownArtifactTags::Rot { board: read_board(caboose)?, - sign: read_sign(caboose)?, + sign: Sign(read_sign(caboose)?), slot, }) } @@ -27,7 +28,7 @@ impl KnownArtifactTags { ) -> Result { Ok(KnownArtifactTags::RotBootloader { board: read_board(caboose)?, - sign: read_sign(caboose)?, + sign: Sign(read_sign(caboose)?), }) } @@ -58,6 +59,10 @@ fn read_sign(caboose: &Caboose) -> Result, ReadCabooseError> { } } +pub fn read_name(caboose: &Caboose) -> Result<&str, ReadCabooseError> { + utf8(caboose.name()?, "NAME") +} + pub fn read_version(caboose: &Caboose) -> Result<&str, ReadCabooseError> { utf8(caboose.version()?, "VERS") } diff --git a/artifact/src/installinator.rs b/artifact/src/installinator.rs index 19ad7f0..2ed77c2 100644 --- a/artifact/src/installinator.rs +++ b/artifact/src/installinator.rs @@ -28,13 +28,14 @@ pub struct InstallinatorDocument { pub struct InstallinatorArtifact { /// A file name; not necessarily the target name. pub name: String, + #[serde(flatten)] pub kind: InstallinatorArtifactKind, pub sha256: ArtifactHash, } /// The artifact kind for an installinator artifact. #[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(rename_all = "snake_case")] +#[serde(tag = "kind", rename_all = "kebab-case")] pub enum InstallinatorArtifactKind { /// A measurement corpus. MeasurementCorpus, diff --git a/artifact/src/lib.rs b/artifact/src/lib.rs index fe2f4d1..1a97cf7 100644 --- a/artifact/src/lib.rs +++ b/artifact/src/lib.rs @@ -6,11 +6,13 @@ mod artifact; mod hash; pub mod hubris; mod installinator; +mod sign; mod tags; mod version; pub use artifact::*; pub use hash::*; pub use installinator::*; +pub use sign::*; pub use tags::*; pub use version::*; diff --git a/artifact/src/sign.rs b/artifact/src/sign.rs new file mode 100644 index 0000000..644f7c8 --- /dev/null +++ b/artifact/src/sign.rs @@ -0,0 +1,122 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::HashMap; +use std::fmt::Debug; +use std::fmt::Display; +use std::sync::LazyLock; + +use serde::Deserialize; +use serde::Serialize; + +const CA_LIST: [(&str, &str); 9] = [ + ( + "f62eb434dd27302521d12958a7c3c18c69f6e0239361654f0a09e5cc554e0fab", + "production-cosmo-rot", + ), + ( + "5796ee3433f840519c3bcde73e19ee82ccb6af3857eddaabb928b8d9726d93c0", + "production-gimlet-rot", + ), + ( + "31942f8d53dc908c5cb338bdcecb204785fa87834e8b18f706fc972a42886c8b", + "production-psc-rot", + ), + ( + "5c69a42ee1f1e6cd5f356d14f81d46f8dbee783bb28777334226c689f169c0eb", + "production-sidecar-rot", + ), + ( + "855a51518d13dd696a8102a5b2436bb2e8c39a6c0102909fce0a32adc6b76fb4", + "staging-cosmo-rot", + ), + ( + "11594bb5548a757e918e6fe056e2ad9e084297c9555417a025d8788eacf55daf", + "staging-gimlet-rot", + ), + ( + "f592d8f109b81881221eed5af6438abad9b5df8c220b9129c03763e7e10b22c7", + "staging-psc-rot", + ), + ( + "1432cc4cfe5688c51b55546fe37837c753cfbc89e8c3c6aabcf977fdf0c41e27", + "staging-sidecar-rot", + ), + ( + "84332ef8279df87fbb759dc3866cbc50cd246fbb5a64705a7e60ba86bf01c27d", + "test-bart", + ), +]; +static CA_MAP: LazyLock> = + LazyLock::new(|| HashMap::from(CA_LIST)); + +#[derive( + Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, +)] +#[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] +#[serde(transparent)] +pub struct Sign(pub Option); + +impl Sign { + pub const UNSIGNED: Sign = Sign(None); + + pub fn new(sign: Option) -> Self { + Self(sign) + } + + pub fn is_signed(&self) -> bool { + self.0.is_some() + } + + pub fn is_unsigned(&self) -> bool { + self.0.is_none() + } + + pub fn friendly_ca_name(&self) -> Option<&'static str> { + let inner = self.0.as_deref()?; + CA_MAP.get(inner).copied() + } +} + +impl Debug for Sign { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + Debug::fmt(&self.0, f) + } +} + +impl Display for Sign { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self.0.as_deref() { + Some(inner) => { + let s = CA_MAP.get(inner).copied().unwrap_or(inner); + write!(f, "{s}") + } + None => write!(f, "unsigned"), + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + + use crate::sign::CA_LIST; + use crate::sign::CA_MAP; + + #[test] + fn ensure_consistency() { + // no duplicate keys + assert_eq!(CA_MAP.len(), CA_LIST.len()); + // no duplicate values + let values = CA_MAP.values().collect::>(); + assert_eq!(values.len(), CA_LIST.len()); + // CA_LIST is ordered by friendly name + for window in CA_LIST.windows(2) { + let [(_, v1), (_, v2)] = window else { + panic!("slice::windows is broken") + }; + assert!(v1 < v2, "{v1} incorrectly placed before {v2} in CA_LIST"); + } + } +} diff --git a/artifact/src/tags.rs b/artifact/src/tags.rs index 06f6d51..7da3631 100644 --- a/artifact/src/tags.rs +++ b/artifact/src/tags.rs @@ -9,6 +9,7 @@ use serde::Serialize; use serde::de::value::MapDeserializer; use crate::InstallinatorArtifactKind; +use crate::Sign; /// Sets of artifact tags known to the control plane. #[derive( @@ -41,8 +42,8 @@ pub enum KnownArtifactTags { /// For unsigned images this will not be present; this will generally /// never occur in release repos but can be useful on hardware that has /// not fully made it through manufacturing yet. - #[serde(skip_serializing_if = "Option::is_none")] - sign: Option, + #[serde(skip_serializing_if = "Sign::is_unsigned")] + sign: Sign, /// ROT images are compiled for two different locations in flash; this /// identifies which slot this image belongs to. slot: RotSlot, @@ -58,8 +59,8 @@ pub enum KnownArtifactTags { /// For unsigned images this will not be present; this will generally /// never occur in release repos but can be useful on hardware that has /// not fully made it through manufacturing yet. - #[serde(skip_serializing_if = "Option::is_none")] - sign: Option, + #[serde(skip_serializing_if = "Sign::is_unsigned")] + sign: Sign, }, /// Hubris archive for a Service Processor image. @@ -182,6 +183,7 @@ display_serialize!(OsBoard); Serialize, )] #[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] +#[serde(rename_all = "kebab-case")] pub enum RotSlot { A, B, diff --git a/bin/src/assemble.rs b/bin/src/assemble.rs deleted file mode 100644 index ae96889..0000000 --- a/bin/src/assemble.rs +++ /dev/null @@ -1,105 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use anyhow::Context; -use anyhow::Result; -use camino::Utf8Path; -use camino::Utf8PathBuf; -use chrono::Utc; -use clap::Parser; -use semver::Version; -use tufaceous::edit::RepositoryEditor; -use tufaceous_artifact::ArtifactVersion; -use tufaceous_artifact::KnownArtifactTags; -use tufaceous_artifact::OsVariant; -use tufaceous_artifact::RotSlot; - -use crate::sign::SignOptions; - -#[derive(Debug, Parser)] -pub struct Args { - #[clap(long)] - version: Version, - #[clap(long)] - no_installinator_document: bool, - #[clap(long)] - measurement_corpus: Vec, - #[clap(long)] - host_os_dir: Option, - #[clap(long)] - recovery_os_dir: Option, - #[clap(long)] - rot_slot_a: Vec, - #[clap(long)] - rot_slot_b: Vec, - #[clap(long)] - rot_bootloader: Vec, - #[clap(long)] - sp: Vec, - #[clap(long)] - zone: Vec, - - #[clap(flatten)] - sign_options: SignOptions, - output: Utf8PathBuf, -} - -impl Args { - pub async fn run(self) -> Result<()> { - let version = ArtifactVersion::new(self.version.to_string())?; - let mut editor = RepositoryEditor::new(self.version); - if !self.no_installinator_document { - editor = editor.generate_installinator_document(); - } - for path in self.measurement_corpus { - editor = editor - .add_artifact( - file_name(&path)?, - version.clone(), - KnownArtifactTags::MeasurementCorpus {}, - path, - ) - .await?; - } - if let Some(path) = self.host_os_dir { - editor = editor.add_os_artifacts(OsVariant::Host, path).await?; - } - if let Some(path) = self.recovery_os_dir { - editor = editor.add_os_artifacts(OsVariant::Recovery, path).await?; - } - for (slot, paths) in - [(RotSlot::A, self.rot_slot_a), (RotSlot::B, self.rot_slot_b)] - { - for path in paths { - editor = - editor.add_rot_image(file_name(&path)?, slot, path).await?; - } - } - for path in self.rot_bootloader { - editor = editor - .add_rot_bootloader_image(file_name(&path)?, path) - .await?; - } - for path in self.sp { - editor = editor.add_sp_image(file_name(&path)?, path).await?; - } - for path in self.zone { - editor = editor.add_zone_image(&file_name(&path)?, path).await?; - } - - let unsigned = editor.finish().await?; - self.sign_options - .sign(unsigned) - .await? - .write_zip_file(self.output, Utc::now()) - .await?; - Ok(()) - } -} - -fn file_name(path: &Utf8Path) -> Result { - path.file_name() - .map(str::to_owned) - .with_context(|| format!("{path} has no file name")) -} diff --git a/bin/src/build.rs b/bin/src/build.rs new file mode 100644 index 0000000..9dde6bd --- /dev/null +++ b/bin/src/build.rs @@ -0,0 +1,42 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use anyhow::Result; +use camino::Utf8PathBuf; +use chrono::Utc; +use clap::Parser; +use semver::Version; +use tufaceous::edit::RepositoryEditor; + +use crate::sign::SignOptions; + +#[derive(Debug, Parser)] +pub struct Args { + artifacts: Vec, + #[clap(long)] + no_installinator_document: bool, + #[clap(short = 'o', long)] + output: Utf8PathBuf, + #[clap(flatten)] + sign_options: SignOptions, + #[clap(short = 'V', long)] + version: Version, +} + +impl Args { + pub async fn run(self) -> Result<()> { + let mut editor = RepositoryEditor::new(self.version) + .generate_installinator_document(!self.no_installinator_document); + for path in self.artifacts { + editor = editor.guess_artifact(path).await?; + } + let unsigned = editor.finish().await?; + self.sign_options + .sign(unsigned) + .await? + .write_zip_file(self.output, Utc::now()) + .await?; + Ok(()) + } +} diff --git a/bin/src/main.rs b/bin/src/main.rs index 78a81d1..121c741 100644 --- a/bin/src/main.rs +++ b/bin/src/main.rs @@ -2,41 +2,41 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -mod assemble; +mod build; mod sign; +use std::sync::LazyLock; + use anyhow::Result; use clap::Parser; use slog::Drain; use slog::Logger; -#[derive(Debug, Parser)] -enum Command { - Assemble(assemble::Args), -} - -#[tokio::main] -async fn main() -> Result<()> { - let stderr_drain = stderr_env_drain("RUST_LOG"); - let drain = slog_async::Async::new(stderr_drain).build().fuse(); - let _log = Logger::root(drain, slog::o!()); - - let args = Command::parse(); - match args { - Command::Assemble(args) => args.run().await, - } -} - -fn stderr_env_drain(env_var: &str) -> impl Drain { +static _LOG: LazyLock = LazyLock::new(|| { let stderr_decorator = slog_term::TermDecorator::new().build(); let stderr_drain = slog_term::FullFormat::new(stderr_decorator).build().fuse(); let mut builder = slog_envlogger::LogBuilder::new(stderr_drain); - if let Ok(s) = std::env::var(env_var) { + if let Ok(s) = std::env::var("RUST_LOG") { builder = builder.parse(&s); } else { // Log at the info level by default. builder = builder.filter(None, slog::FilterLevel::Info); } - builder.build() + let stderr_drain = builder.build(); + let drain = slog_async::Async::new(stderr_drain).build().fuse(); + Logger::root(drain, slog::o!()) +}); + +#[derive(Debug, Parser)] +enum Command { + Build(build::Args), +} + +#[tokio::main] +async fn main() -> Result<()> { + let args = Command::parse(); + match args { + Command::Build(args) => args.run().await, + } } diff --git a/lib/Cargo.toml b/lib/Cargo.toml index c16340e..0132413 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -12,10 +12,12 @@ bytes.workspace = true camino.workspace = true camino-tempfile.workspace = true chrono.workspace = true +ciborium.workspace = true flate2.workspace = true futures-util.workspace = true hex.workspace = true hubtools.workspace = true +rats-corim.workspace = true rawzip.workspace = true semver.workspace = true serde.workspace = true diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index 75652c8..21db729 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -5,15 +5,18 @@ use std::collections::BTreeMap; use std::collections::HashMap; use std::collections::HashSet; +use std::io::Read; use camino::Utf8Path; use camino::Utf8PathBuf; +use flate2::read::GzDecoder; use futures_util::FutureExt; use futures_util::TryFutureExt; -use futures_util::TryStreamExt; use hubtools::Caboose; +use rats_corim::Corim; use semver::Version; use tokio::fs::File; +use tokio::io::AsyncReadExt; use tokio::task::JoinSet; use tufaceous_artifact::ArtifactVersion; use tufaceous_artifact::InstallinatorArtifact; @@ -22,7 +25,10 @@ use tufaceous_artifact::KnownArtifactTags; use tufaceous_artifact::OsBoard; use tufaceous_artifact::OsVariant; use tufaceous_artifact::RotSlot; +use tufaceous_artifact::Sign; use tufaceous_artifact::hubris::ReadCabooseError; +use tufaceous_brand_metadata::LayerInfo; +use tufaceous_brand_metadata::Metadata; use crate::Repository; use crate::edit::UnsignedRepository; @@ -43,48 +49,430 @@ const MIB: usize = 1024 * KIB; #[derive(Debug)] pub struct RepositoryEditor<'a> { system_version: Version, + generate_installinator_document: bool, targets: HashMap>>, artifacts: HashMap>, metadata: BTreeMap, - generate_installinator_document: bool, } impl<'a> RepositoryEditor<'a> { + /// Create an empty repository editor. pub fn new(system_version: Version) -> Self { Self { system_version, + generate_installinator_document: true, targets: HashMap::new(), artifacts: HashMap::new(), metadata: BTreeMap::new(), - generate_installinator_document: false, } } + /// Change the system version of the repository. pub fn system_version(self, system_version: Version) -> Self { Self { system_version, ..self } } - pub async fn add_artifact( + /// Sets whether an Installinator document should be generated based on the + /// artifacts in the repository. + /// + /// Defaults to `true`. The document is generated during + /// [`RepositoryEditor::finish`]. + pub fn generate_installinator_document( + self, + generate_installinator_document: bool, + ) -> Self { + Self { generate_installinator_document, ..self } + } + + /// Add a measurement corpus to the repository. + pub async fn measurement_corpus( + self, + path: Utf8PathBuf, + ) -> Result { + self.measurement_corpus_inner(FileSource::open(path).await?, None).await + } + + async fn measurement_corpus_inner( + mut self, + mut source: FileSource, + corim: Option, + ) -> Result { + let Corim { id, .. } = match corim { + Some(corim) => corim, + None => { + let vec = source.read_to_end().await?; + ciborium::from_reader(vec.as_slice()).map_err(|s| { + ErrorKind::Corim { source: s, path: source.path.clone() } + })? + } + }; + let sha256 = source.sha256().await?; + let target_name = + format!("measurements/{id}-{}.cbor", hex::encode(sha256)); + let version = ArtifactVersion::new(self.system_version.to_string())?; + let tags = KnownArtifactTags::MeasurementCorpus {}; + self.insert_artifact(target_name, version, tags, source); + Ok(self) + } + + fn guess_measurement_corpus( + mut file_start: &[u8], + ) -> (bool, Option) { + if !matches!(file_start[0], 0xa0..=0xbf /* CBOR map */) { + return (false, None); + } + match ciborium::from_reader::(&mut file_start) { + Ok(corim) => (true, Some(corim)), + Err(ciborium::de::Error::Io(err)) + if err.kind() == std::io::ErrorKind::UnexpectedEof => + { + // This was plausibly a CoRIM manifest until we hit the end of + // the buffer, indicating a very high likelihood that if we read + // the entire thing it'd still be a CoRIM manifest. + (true, None) + } + Err(_) => (false, None), + } + } + + /// Add an OS image to the repository. + /// + /// `output_dir` is a path to the output directory for `helios-build image` + /// (the `-o` argument). This directory contains `cosmo.rom`, `gimlet.rom`, + /// `zfs.img`, and `os.tar.gz`. Metadata stored in `os.tar.gz` is copied + /// into the repository. + pub async fn os_image_dir( mut self, + variant: OsVariant, + output_dir: &Utf8Path, + ) -> Result { + let base = Utf8PathBuf::from(format!("os-{variant}")); + let version = ArtifactVersion::new(self.system_version.to_string())?; + + self.insert_artifact( + base.join("image/cosmo.rom").into(), + version.clone(), + KnownArtifactTags::OsPhase1 { variant, board: OsBoard::Cosmo }, + FileSource::open(output_dir.join("cosmo.rom")).await?, + ); + self.insert_artifact( + base.join("image/gimlet.rom").into(), + version.clone(), + KnownArtifactTags::OsPhase1 { variant, board: OsBoard::Gimlet }, + FileSource::open(output_dir.join("gimlet.rom")).await?, + ); + self.insert_artifact( + base.join("image/zfs.img").into(), + version, + KnownArtifactTags::OsPhase2 { variant }, + FileSource::open(output_dir.join("zfs.img")).await?, + ); + for path in ["cpio.z", "unix.z"] { + let source = FileSource::open(output_dir.join(path)).await?; + self.targets + .entry(base.join("image").join(path).into()) + .or_default() + .push(TargetSource::File(source)); + } + + let tarball_path = output_dir.join("os.tar.gz"); + let metadata_sources = tokio::task::spawn_blocking(move || { + let mut sources = Vec::new(); + let file = + std::fs::File::open(&tarball_path).map_err(|source| { + ErrorKind::OpenFile { source, path: tarball_path.clone() } + })?; + let mut archive = tar::Archive::new(GzDecoder::new(file)); + for entry in archive.entries().map_err(|source| { + ErrorKind::ReadFile { source, path: tarball_path.clone() } + })? { + let mut entry = entry.map_err(|source| { + ErrorKind::ReadFile { source, path: tarball_path.clone() } + })?; + if entry.header().entry_type() != tar::EntryType::Regular { + continue; + } + let path = entry + .path() + .and_then(|path| { + Utf8PathBuf::try_from(path.into_owned()) + .map_err(|error| error.into_io_error()) + }) + .map_err(|source| ErrorKind::ReadFile { + source, + path: tarball_path.clone(), + })?; + if path == "image/zfs.img" { + break; + } + let mut vec = Vec::new(); + entry.read_to_end(&mut vec).map_err(|source| { + ErrorKind::ReadFile { source, path: tarball_path.clone() } + })?; + sources.push((path, vec)); + } + Ok::<_, Error>(sources) + }) + .await??; + for (path, source) in metadata_sources { + self.targets + .entry(base.join(path).into()) + .or_default() + .push(TargetSource::Bytes(BytesSource(source.into()))); + } + + Ok(self) + } + + async fn guess_os_image(path: &Utf8Path) -> Option { + if !path.join("os.tar.gz").exists() { + return None; + } + let mut file = File::open(path.join("zfs.img")).await.ok()?; + // Read the header block from the image and guess whether it's a + // recovery image based on the image name. + let mut buf = [0; 4096]; + file.read_exact(&mut buf).await.ok()?; + if !buf.starts_with(&0x1DEB0075_u32.to_le_bytes()) { + return None; + } + // see https://github.com/oxidecomputer/boot-image-tools/blob/main/src/diskimage.rs + let image_name = &buf[200..328]; + Some(if image_name.starts_with(b"recovery") { + OsVariant::Recovery + } else { + OsVariant::Host + }) + } + + pub async fn rot_archive( + self, + slot: RotSlot, + path: Utf8PathBuf, + ) -> Result { + self.hubris_archive(path, |caboose| { + KnownArtifactTags::from_rot_caboose(caboose, slot) + }) + .await + } + + pub async fn rot_bootloader_archive( + self, + path: Utf8PathBuf, + ) -> Result { + self.hubris_archive( + path, + KnownArtifactTags::from_rot_bootloader_caboose, + ) + .await + } + + pub async fn sp_archive(self, path: Utf8PathBuf) -> Result { + self.hubris_archive(path, KnownArtifactTags::from_sp_caboose).await + } + + async fn hubris_archive( + self, + path: Utf8PathBuf, + tag_fn: F, + ) -> Result + where + F: FnOnce(&Caboose) -> Result, + { + let mut source = FileSource::open(path.clone()).await?; + let caboose = source.read_hubris_caboose().await?; + let data = CabooseData::new(&caboose, tag_fn, &path)?; + Ok(self.hubris_archive_inner(source, data)) + } + + fn hubris_archive_inner( + mut self, + source: FileSource, + CabooseData { tags, name, version }: CabooseData, + ) -> Self { + let target_name = match &tags { + KnownArtifactTags::Rot { board, sign, slot } => { + format!("rot/{board}-{sign}-{version}-slot-{slot}.zip") + } + KnownArtifactTags::RotBootloader { board, sign } => { + format!("rot-bootloader/{board}-{sign}-{version}.zip") + } + KnownArtifactTags::Sp { board } => { + let target_name = format!("sp/{name}-{version}.zip"); + if board.as_str() != name { + // This is likely a lab image. These should not be included + // in the list of artifacts as a control plane should never + // run them, but these should be present in the repository. + // TODO: determine if this is the correct decision + self.targets + .entry(target_name) + .or_default() + .push(TargetSource::File(source)); + return self; + } + target_name + } + _ => unreachable!(), + }; + self.insert_artifact(target_name, version, tags, source); + self + } + + async fn guess_hubris_archive( + file_start: &[u8], + path: &Utf8Path, + ) -> Option<(FileSource, CabooseData)> { + if !file_start.starts_with(b"PK\x03\x04") { + return None; + } + let mut source = FileSource::open(path.to_owned()).await.ok()?; + let archive = source.read_hubris_archive().await.ok()?; + let caboose = archive.read_caboose().ok()?; + // HACK: We are reading the `image-name` file in the archive, which + // appears to be "a" or "b" if it's an ROT image, "default" if it's + // an SP image, and nonexistent if it's an ROT bootloader image. This + // seems fragile. Ideally this can be in the caboose someday (see + // sprot-release#74). + let data = match archive.image_name().as_deref() { + Ok("a") => CabooseData::new( + &caboose, + |caboose| { + KnownArtifactTags::from_rot_caboose(caboose, RotSlot::A) + }, + path, + ), + Ok("b") => CabooseData::new( + &caboose, + |caboose| { + KnownArtifactTags::from_rot_caboose(caboose, RotSlot::B) + }, + path, + ), + Ok("default") => CabooseData::new( + &caboose, + KnownArtifactTags::from_sp_caboose, + path, + ), + Err(hubtools::Error::MissingFile(_, _)) => CabooseData::new( + &caboose, + KnownArtifactTags::from_rot_bootloader_caboose, + path, + ), + _ => return None, + }; + Some((source, data.ok()?)) + } + + pub async fn zone_image(self, path: Utf8PathBuf) -> Result { + let cloned_path = path.clone(); + let (file, layer_info) = tokio::task::spawn_blocking(move || { + let file = std::fs::File::open(&path).map_err(|source| { + ErrorKind::ReadFile { source, path: path.clone() } + })?; + let mut archive = tar::Archive::new(GzDecoder::new(file)); + let layer_info = Metadata::read_from_tar(&mut archive) + .and_then(|metadata| metadata.layer_info().cloned()) + .map_err(|source| ErrorKind::ReadZoneOxideJson { + source, + path: path.clone(), + })?; + Ok::<_, Error>((archive.into_inner().into_inner(), layer_info)) + }) + .await??; + let source = FileSource::from_file(file.into(), cloned_path); + Ok(self.zone_image_inner(source, layer_info)) + } + + fn zone_image_inner( + mut self, + source: FileSource, + LayerInfo { pkg, version }: LayerInfo, + ) -> Self { + let target_name = format!("zones/{pkg}.tar.gz"); + let tags = KnownArtifactTags::Zone { name: pkg }; + self.insert_artifact(target_name, version, tags, source); + self + } + + fn guess_zone_image(file_start: &[u8]) -> Option { + // `oxide.json` is the first file of a zone image and is relatively + // small, so it should be contained entirely within the first 4K of the + // compressed tarball. + let mut archive = tar::Archive::new(GzDecoder::new(file_start)); + Metadata::read_from_tar(&mut archive).ok()?.layer_info().ok().cloned() + } + + /// Attempt to guess the artifact kind at `path` and add it to the + /// repository. + /// + /// This should only be used as a convenience method to human users. + /// Automation should not be making any guesses. + pub async fn guess_artifact( + self, + path: Utf8PathBuf, + ) -> Result { + if let Some(variant) = Self::guess_os_image(&path).await { + return self.os_image_dir(variant, &path).await; + } + + let mut file = File::open(&path).await.map_err(|source| { + ErrorKind::OpenFile { source, path: path.clone() } + })?; + let mut buf = [0; 4096]; + let n = file.read(&mut buf).await.map_err(|source| { + ErrorKind::ReadFile { source, path: path.clone() } + })?; + if n == 0 { + // we're not going to try to guess an empty file + return Err(ErrorKind::GuessArtifact { path }.into()); + } + let buf = &buf[..n]; + let source = FileSource::from_file(file, path.clone()); + + let (likely_corim, corim) = Self::guess_measurement_corpus(buf); + if likely_corim { + return self.measurement_corpus_inner(source, corim).await; + } + + if let Some((source, data)) = + Self::guess_hubris_archive(buf, &path).await + { + return Ok(self.hubris_archive_inner(source, data)); + } + + if let Some(layer_info) = Self::guess_zone_image(buf) { + return Ok(self.zone_image_inner(source, layer_info)); + } + + Err(ErrorKind::GuessArtifact { path }.into()) + } + + fn insert_artifact( + &mut self, target_name: String, version: ArtifactVersion, tags: KnownArtifactTags, - path: impl Into, - ) -> Result { + source: FileSource, + ) { self.targets .entry(target_name.clone()) .or_default() - .push(TargetSource::File(FileSource::open(path).await?)); + .push(TargetSource::File(source)); self.artifacts.entry(target_name.clone()).or_default().insert( ArtifactSchema { target_name, version, tags: tags.to_tags() }, ); - Ok(self) } - pub async fn add_extra_target( + /// Add a non-artifact target to the repository. + /// + /// This target will be part of the signed set of files in the repository, + /// but will not be copied onto sleds for use by the control plane. This is + /// intended for ancillary files that are useful to systems other than the + /// control plane. + pub async fn extra_target( mut self, target_name: String, - path: impl Into, + path: Utf8PathBuf, ) -> Result { self.targets .entry(target_name.clone()) @@ -93,7 +481,7 @@ impl<'a> RepositoryEditor<'a> { Ok(self) } - pub fn add_fake_artifact( + pub fn fake_artifact( mut self, target_name: String, version: ArtifactVersion, @@ -113,12 +501,11 @@ impl<'a> RepositoryEditor<'a> { pub fn fake(system_version: Version) -> Result { let version = ArtifactVersion::new(system_version.to_string())?; - let mut editor = - Self::new(system_version).generate_installinator_document(); + let mut editor = Self::new(system_version); for hash in ["123abc", "def456"] { - editor = editor.add_fake_artifact( - format!("measurement-corpus-{hash}.corim"), + editor = editor.fake_artifact( + format!("measurements/corim-fake-{version}-{hash}.cbor"), version.clone(), KnownArtifactTags::MeasurementCorpus {}, 4 * KIB, @@ -126,49 +513,49 @@ impl<'a> RepositoryEditor<'a> { } for variant in [OsVariant::Host, OsVariant::Recovery] { for board in [OsBoard::Gimlet, OsBoard::Cosmo] { - editor = editor.add_fake_artifact( - format!("{variant}-os/{board}.rom"), + editor = editor.fake_artifact( + format!("os-{variant}/{board}.rom"), version.clone(), KnownArtifactTags::OsPhase1 { variant, board }, MIB, ); } - editor = editor.add_fake_artifact( - format!("{variant}-os/zfs.img"), + editor = editor.fake_artifact( + format!("os-{variant}/zfs.img"), version.clone(), KnownArtifactTags::OsPhase2 { variant }, 4 * MIB, ); } for slot in [RotSlot::A, RotSlot::B] { - editor = editor.add_fake_artifact( - format!("rot-fake-slot{slot}.zip"), + editor = editor.fake_artifact( + format!("rot/fake-unsigned-{version}-slot-{slot}.zip"), version.clone(), KnownArtifactTags::Rot { board: "fake".into(), - sign: None, + sign: Sign::UNSIGNED, slot, }, 256 * KIB, ); } - editor = editor.add_fake_artifact( - "rot-bootloader-fake.zip".into(), + editor = editor.fake_artifact( + format!("rot-bootloader/fake-unsigned-{version}.zip"), version.clone(), KnownArtifactTags::RotBootloader { board: "fake".into(), - sign: None, + sign: Sign::UNSIGNED, }, 64 * KIB, ); - editor = editor.add_fake_artifact( - "sp-fake.zip".into(), + editor = editor.fake_artifact( + format!("sp/fake-{version}.zip"), version.clone(), KnownArtifactTags::Sp { board: "fake".into() }, MIB, ); for name in ["zone1", "zone2"] { - editor = editor.add_fake_artifact( + editor = editor.fake_artifact( format!("zones/{name}.tar.gz"), version.clone(), KnownArtifactTags::Zone { name: name.into() }, @@ -179,6 +566,20 @@ impl<'a> RepositoryEditor<'a> { Ok(editor) } + pub async fn metadata( + mut self, + key: String, + value: serde_json::Value, + ) -> Self { + self.metadata.insert(key, value); + self + } + + pub async fn remove_metadata(mut self, key: &str) -> Self { + self.metadata.remove(key); + self + } + pub fn from_repo(repo: &'a mut Repository) -> Result { Self::new(repo.system_version().clone()).import_repo(repo) } @@ -217,10 +618,6 @@ impl<'a> RepositoryEditor<'a> { Ok(self) } - pub fn generate_installinator_document(self) -> Self { - Self { generate_installinator_document: true, ..self } - } - pub async fn finish(self) -> Result, Error> { // Un-nest `self.artifacts`, returning an error if we have multiple // artifact definitions for a single target name. @@ -302,11 +699,13 @@ impl<'a> RepositoryEditor<'a> { if let Some(kind) = KnownArtifactTags::from_tags(&artifact.tags) .ok() .and_then(|tags| tags.to_installinator()) + && let Some(file_name) = + Utf8Path::new(&artifact.target_name).file_name() && let Some(target) = targets.0.get(&artifact.target_name) && let Ok(sha256) = target.sha256.as_slice().try_into() { document.artifacts.push(InstallinatorArtifact { - name: artifact.target_name.clone(), + name: file_name.to_owned(), kind, sha256, }); @@ -329,165 +728,44 @@ impl<'a> RepositoryEditor<'a> { targets.insert(ArtifactsSchema::TARGET_NAME.to_owned(), target)?; Ok(UnsignedRepository::from_targets(targets.0)) } +} - pub async fn add_os_artifacts( - mut self, - variant: OsVariant, - build_output_path: impl Into, - ) -> Result { - let base = Utf8Path::new(match variant { - OsVariant::Host => "host-os", - OsVariant::Recovery => "recovery-os", - }); - let version = ArtifactVersion::new(self.system_version.to_string())?; - let mut read_dir = crate::util::read_dir(build_output_path).await?; - while let Some(entry) = read_dir.try_next().await? { - let target_name = base.join(entry.file_name()); - let tags = match entry.file_name() { - "cosmo.rom" => KnownArtifactTags::OsPhase1 { - variant, - board: OsBoard::Cosmo, - }, - "gimlet.rom" => KnownArtifactTags::OsPhase1 { - variant, - board: OsBoard::Gimlet, - }, - "zfs.img" => KnownArtifactTags::OsPhase2 { variant }, - - _ => { - self = self - .add_extra_target(target_name.into(), entry.into_path()) - .await?; - continue; - } - }; - self = self - .add_artifact( - target_name.into(), - version.clone(), - tags, - entry.into_path(), - ) - .await?; - } - Ok(self) - } +#[derive(Debug)] +struct CabooseData { + tags: KnownArtifactTags, + name: String, + version: ArtifactVersion, +} - async fn add_hubris_image( - mut self, - target_name: String, - path: impl Into, - f: F, +impl CabooseData { + fn new( + caboose: &Caboose, + tag_fn: F, + path: &Utf8Path, ) -> Result where F: FnOnce(&Caboose) -> Result, { - let path = path.into(); - let mut source = FileSource::open(path.clone()).await?; - let caboose = source.read_hubris_caboose().await?; - self.targets - .entry(target_name.clone()) - .or_default() - .push(TargetSource::File(source)); - let (tags, version) = f(&caboose) - .and_then(|tags| { - Ok((tags, tufaceous_artifact::hubris::read_version(&caboose)?)) - }) - .map_err(|source| ErrorKind::ReadCaboose { source, path })?; - let version = ArtifactVersion::new(version)?; - self.artifacts.entry(target_name.clone()).or_default().insert( - ArtifactSchema { target_name, version, tags: tags.to_tags() }, - ); - Ok(self) - } - - pub async fn add_rot_image( - self, - target_name: String, - slot: RotSlot, - path: impl Into, - ) -> Result { - self.add_hubris_image(target_name, path, |caboose| { - KnownArtifactTags::from_rot_caboose(caboose, slot) - }) - .await - } - - pub async fn add_rot_bootloader_image( - self, - target_name: String, - path: impl Into, - ) -> Result { - self.add_hubris_image( - target_name, - path, - KnownArtifactTags::from_rot_bootloader_caboose, - ) - .await - } - - pub async fn add_sp_image( - self, - target_name: String, - path: impl Into, - ) -> Result { - self.add_hubris_image( - target_name, - path, - KnownArtifactTags::from_sp_caboose, - ) - .await - } - - pub async fn add_zone_image( - mut self, - file_name: &str, - path: impl Into, - ) -> Result { - let path = path.into(); - let file = File::open(&path) - .await - .map_err(|source| ErrorKind::OpenFile { + let tags = tag_fn(caboose).map_err(|source| { + ErrorKind::ReadCaboose { source, path: path.to_owned() } + })?; + let name = tufaceous_artifact::hubris::read_name(caboose).map_err( + |source| ErrorKind::ReadCaboose { source, path: path.to_owned() }, + )?; + let version = tufaceous_artifact::hubris::read_version(caboose) + .map_err(|source| ErrorKind::ReadCaboose { source, - path: path.clone(), - })? - .into_std() - .await; - let (file, layer_info) = - crate::util::read_zone_layer_info(file, path.clone()).await?; - let source = FileSource::from_file(file.into(), path); - let target_name = format!("zones/{file_name}"); - self.targets - .entry(target_name.clone()) - .or_default() - .push(TargetSource::File(source)); - self.artifacts.entry(target_name.clone()).or_default().insert( - ArtifactSchema { - target_name, - version: layer_info.version, - tags: KnownArtifactTags::Zone { name: layer_info.pkg } - .to_tags(), - }, - ); - Ok(self) - } - - pub async fn insert_metadata( - mut self, - key: String, - value: serde_json::Value, - ) -> Self { - self.metadata.insert(key, value); - self - } - - pub async fn remove_metadata(mut self, key: &str) -> Self { - self.metadata.remove(key); - self + path: path.to_owned(), + })?; + Ok(Self { + tags, + name: name.to_owned(), + version: ArtifactVersion::new(version)?, + }) } } -#[derive(Default)] +#[derive(Debug, Default)] struct TargetMap<'a>(BTreeMap>); impl<'a> TargetMap<'a> { diff --git a/lib/src/edit/sign.rs b/lib/src/edit/sign.rs index bbbedc5..e6d0ffe 100644 --- a/lib/src/edit/sign.rs +++ b/lib/src/edit/sign.rs @@ -178,8 +178,9 @@ impl<'a> UnsignedRepository<'a> { while let Some(entry) = read_dir.try_next().await? { // This is opening a file within the tempdir that is about to be // deleted. This is expected to be fine. - let source = - TargetSource::File(FileSource::open(entry.path()).await?); + let source = TargetSource::File( + FileSource::open(entry.path().to_owned()).await?, + ); sources.insert( (FilePrefix::Metadata, entry.file_name().into()), source, @@ -273,6 +274,17 @@ fn deflate_heuristic(buf: &[u8]) -> Compression { } else if buf.starts_with(b"\x1f\x8b") { // gzip, e.g. illumos zone tarball Compression::none() + } else if buf.starts_with(b"\x78") + && let [x, y] = &buf[..2] + && u16::from_be_bytes([*x, *y]) % 31 == 0 + { + // zlib + if y & 0xc0 == 0 { + // compression level 0 + Compression::best() + } else { + Compression::none() + } } else if buf.starts_with(b"PK\x03\x04") { // ZIP archive, e.g. hubris archive. not necessarily compressed based on // this heuristic alone but in our case it's very likely. diff --git a/lib/src/edit/source.rs b/lib/src/edit/source.rs index 0d95f84..bdcf7f4 100644 --- a/lib/src/edit/source.rs +++ b/lib/src/edit/source.rs @@ -90,15 +90,12 @@ impl BytesSource { #[derive(Debug)] pub(crate) struct FileSource { file: File, - path: Utf8PathBuf, + pub(crate) path: Utf8PathBuf, length_sha256: Option<(u64, ArtifactHash)>, } impl FileSource { - pub(crate) async fn open( - path: impl Into, - ) -> Result { - let path = path.into(); + pub(crate) async fn open(path: Utf8PathBuf) -> Result { let file = File::open(&path).await.map_err(|source| { ErrorKind::OpenFile { source, path: path.clone() } })?; @@ -109,7 +106,21 @@ impl FileSource { Self { file, path, length_sha256: None } } - async fn read( + pub(crate) async fn into_target( + mut self, + ) -> Result, Error> { + let (length, sha256) = match self.length_sha256 { + Some(inner) => inner, + None => self.read_impl(None).await?, + }; + Ok(Target { + length, + sha256: sha256.0.to_vec(), + source: TargetSource::File(self), + }) + } + + async fn read_impl( &mut self, mut vec: Option<&mut Vec>, ) -> Result<(u64, ArtifactHash), Error> { @@ -131,31 +142,38 @@ impl FileSource { Ok(*self.length_sha256.insert((length, sha256))) } - pub(crate) async fn into_target( - mut self, - ) -> Result, Error> { - let (length, sha256) = match self.length_sha256 { + pub(crate) async fn sha256(&mut self) -> Result { + let (_, sha256) = match self.length_sha256 { Some(inner) => inner, - None => self.read(None).await?, + None => self.read_impl(None).await?, }; - Ok(Target { - length, - sha256: sha256.0.to_vec(), - source: TargetSource::File(self), - }) + Ok(sha256) } - pub(crate) async fn read_hubris_caboose( + pub(crate) async fn read_to_end(&mut self) -> Result, Error> { + let mut vec = Vec::new(); + self.read_impl(Some(&mut vec)).await?; + Ok(vec) + } + + pub(crate) async fn read_hubris_archive( &mut self, - ) -> Result { - let mut image = Vec::new(); - self.read(Some(&mut image)).await?; - RawHubrisArchive::from_vec(image) - .and_then(|image| image.read_caboose()) - .map_err(|source| { + ) -> Result { + RawHubrisArchive::from_vec(self.read_to_end().await?).map_err( + |source| { ErrorKind::ReadHubrisArchive { source, path: self.path.clone() } .into() - }) + }, + ) + } + + pub(crate) async fn read_hubris_caboose( + &mut self, + ) -> Result { + self.read_hubris_archive().await?.read_caboose().map_err(|source| { + ErrorKind::ReadHubrisArchive { source, path: self.path.clone() } + .into() + }) } pub(crate) fn stream( diff --git a/lib/src/error.rs b/lib/src/error.rs index 6de161b..28d4446 100644 --- a/lib/src/error.rs +++ b/lib/src/error.rs @@ -132,6 +132,10 @@ pub enum ErrorKind { #[error(transparent)] ArtifactVersion(#[from] tufaceous_artifact::ArtifactVersionError), + #[error("failed to read {path} as CoRIM")] + Corim { source: ciborium::de::Error, path: Utf8PathBuf }, + #[error("failed to guess what kind of artifact {path} is")] + GuessArtifact { path: Utf8PathBuf }, #[error("target name collision on {target_name}")] TargetNameCollision { target_name: String }, #[error("failed to serialize artifacts document")] diff --git a/lib/src/loader.rs b/lib/src/loader.rs index 832ce2b..af4fe9d 100644 --- a/lib/src/loader.rs +++ b/lib/src/loader.rs @@ -72,7 +72,7 @@ impl RepositoryLoader { } /// Add a trusted root role to the trust store. - pub fn trust_root(mut self, trust_root: impl AsRef) -> Self { + pub fn trust_root(mut self, trust_root: impl AsRef<[u8]>) -> Self { self.trust_roots.push(trust_root.as_ref().into()); self } @@ -127,7 +127,7 @@ impl RepositoryLoader { /// Load a Tufaceous-generated ZIP archive from a file. pub async fn load_zip_file( self, - archive_path: impl Into, + archive_path: Utf8PathBuf, log: &Logger, ) -> Result { let transport = ZipTransport::from_file(archive_path, log).await?; diff --git a/lib/src/util.rs b/lib/src/util.rs index d3b5ab1..6d946ed 100644 --- a/lib/src/util.rs +++ b/lib/src/util.rs @@ -32,10 +32,7 @@ pub(crate) async fn read_zone_layer_info( .await? } -pub(crate) async fn read_dir( - path: impl Into, -) -> Result { - let path = path.into(); +pub(crate) async fn read_dir(path: Utf8PathBuf) -> Result { match tokio::fs::read_dir(&path).await { Ok(inner) => Ok(ReadDir { inner, path }), Err(source) => Err(ErrorKind::ReadDir { source, path }.into()), @@ -81,10 +78,6 @@ impl DirEntry { &self.path } - pub(crate) fn into_path(self) -> Utf8PathBuf { - self.path - } - pub(crate) fn file_name(&self) -> &str { self.path .file_name() diff --git a/lib/src/zip_transport.rs b/lib/src/zip_transport.rs index 52d8c0a..ac6fe22 100644 --- a/lib/src/zip_transport.rs +++ b/lib/src/zip_transport.rs @@ -96,10 +96,9 @@ impl + Debug + Send + Sync + 'static> ZipTransport> { impl ZipTransport { pub async fn from_file( - archive_path: impl Into, + archive_path: Utf8PathBuf, log: &Logger, ) -> Result { - let archive_path = archive_path.into(); let log = log.clone(); tokio::task::spawn_blocking(move || { let archive_path = archive_path; diff --git a/lib/src/zip_writer.rs b/lib/src/zip_writer.rs index 636030e..999ff64 100644 --- a/lib/src/zip_writer.rs +++ b/lib/src/zip_writer.rs @@ -8,6 +8,7 @@ use atomicwrites::AtomicFile; use atomicwrites::OverwriteBehavior; use bytes::Bytes; use camino::Utf8Path; +use camino::Utf8PathBuf; use flate2::Compression; use flate2::write::DeflateEncoder; use rawzip::CompressionMethod; @@ -31,9 +32,8 @@ impl ZipWriter { } impl ZipWriter<()> { - pub(crate) fn create(path: impl AsRef) -> Self { - let file = - AtomicFile::new(path.as_ref(), OverwriteBehavior::AllowOverwrite); + pub(crate) fn create(path: &Utf8Path) -> Self { + let file = AtomicFile::new(path, OverwriteBehavior::AllowOverwrite); let (file_tx, file_rx) = mpsc::channel(1); let task = tokio::task::spawn_blocking(move || { file.write(|file| write_task(file, file_rx).map(|_| ())).map_err( @@ -50,7 +50,7 @@ impl ZipWriter<()> { impl ZipWriter { pub(crate) fn new_file( &mut self, - name: impl Into, + name: Utf8PathBuf, ) -> ZipFileBuilder<'_, W> { let (bytes_tx, bytes_rx) = mpsc::channel(1); ZipFileBuilder { diff --git a/lib/tests/tests.rs b/lib/tests/tests.rs index 8d846c4..a971ee3 100644 --- a/lib/tests/tests.rs +++ b/lib/tests/tests.rs @@ -41,6 +41,7 @@ async fn it_works() -> Result<(), Error> { async fn no_artifacts() -> Result<(), Error> { let log = slog::Logger::root(slog::Discard, slog::o!()); let zip = RepositoryEditor::new(VERSION) + .generate_installinator_document(false) .finish() .await? .generate_root() @@ -61,7 +62,8 @@ async fn no_artifacts() -> Result<(), Error> { async fn empty_artifact() -> Result<(), Error> { let log = slog::Logger::root(slog::Discard, slog::o!()); let zip = RepositoryEditor::new(VERSION) - .add_fake_artifact( + .generate_installinator_document(false) + .fake_artifact( "empty.img".to_owned(), "1.0.0".parse().unwrap(), tufaceous_artifact::KnownArtifactTags::InstallinatorDocument {}, From 09fd2126e58025f0f5a925070762b30e029fd3ef Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Fri, 19 Dec 2025 09:14:20 -0800 Subject: [PATCH 004/137] todo -> todone --- lib/src/edit/editor.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index 21db729..f57b0bb 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -300,10 +300,12 @@ impl<'a> RepositoryEditor<'a> { KnownArtifactTags::Sp { board } => { let target_name = format!("sp/{name}-{version}.zip"); if board.as_str() != name { - // This is likely a lab image. These should not be included - // in the list of artifacts as a control plane should never - // run them, but these should be present in the repository. - // TODO: determine if this is the correct decision + // This is likely a lab image. As of writing these are + // stored in the TUF repo for manufacturing but are + // explicitly ignored by the control plane, as they can + // never be used in an actual rack. The current thinking is + // that they will eventually no longer need to be in the TUF + // repo. Add these as an extra target, not an artifact. self.targets .entry(target_name) .or_default() From 2e4f4840a8259b8396e6c612f088cd36c623dec2 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Fri, 19 Dec 2025 10:50:34 -0800 Subject: [PATCH 005/137] add edit/ls/cat commands --- Cargo.lock | 1 + bin/Cargo.toml | 1 + bin/src/edit.rs | 72 +++++++++++++++++++++++++++++++++++++++++ bin/src/list_targets.rs | 32 ++++++++++++++++++ bin/src/main.rs | 13 +++++++- bin/src/show_target.rs | 35 ++++++++++++++++++++ lib/src/edit/editor.rs | 13 +++++--- 7 files changed, 161 insertions(+), 6 deletions(-) create mode 100644 bin/src/edit.rs create mode 100644 bin/src/list_targets.rs create mode 100644 bin/src/show_target.rs diff --git a/Cargo.lock b/Cargo.lock index a83a3cf..64d5847 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2875,6 +2875,7 @@ dependencies = [ "chrono", "clap", "fs-err", + "futures-util", "semver", "slog", "slog-async", diff --git a/bin/Cargo.toml b/bin/Cargo.toml index 6dba6f8..73cdbee 100644 --- a/bin/Cargo.toml +++ b/bin/Cargo.toml @@ -15,6 +15,7 @@ camino.workspace = true chrono.workspace = true clap.workspace = true fs-err = { workspace = true, features = ["tokio"] } +futures-util.workspace = true semver.workspace = true slog.workspace = true slog-async.workspace = true diff --git a/bin/src/edit.rs b/bin/src/edit.rs new file mode 100644 index 0000000..40a679d --- /dev/null +++ b/bin/src/edit.rs @@ -0,0 +1,72 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use anyhow::Result; +use camino::Utf8PathBuf; +use chrono::Utc; +use clap::Parser; +use semver::Version; +use tough::ExpirationEnforcement; +use tufaceous::RepositoryLoader; +use tufaceous::TrustStoreBehavior; +use tufaceous::edit::RepositoryEditor; +use tufaceous_artifact::KnownArtifactTags; + +use crate::sign::SignOptions; + +#[derive(Debug, Parser)] +pub struct Args { + #[arg(short = 'a', long, num_args(1..))] + add_artifacts: Vec, + #[clap(long)] + no_installinator_document: bool, + #[clap(short = 'o', long)] + output: Option, + #[arg(short = 'd', long, num_args(1..))] + remove_targets: Vec, + repo: Utf8PathBuf, + #[clap(flatten)] + sign_options: SignOptions, + #[clap(short = 'V', long)] + version: Option, +} + +impl Args { + pub async fn run(self) -> Result<()> { + let repo = RepositoryLoader::new() + .expiration_enforcement(ExpirationEnforcement::Unsafe) + .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .load_zip_file(self.repo.clone(), &crate::LOG) + .await?; + let mut editor = RepositoryEditor::from_repo(&repo)? + .generate_installinator_document(!self.no_installinator_document); + + // always remove an existing installinator document + for artifact in repo + .artifacts() + .get_all(KnownArtifactTags::InstallinatorDocument {}) + { + editor = editor.remove_target(&artifact.target_name); + } + + if let Some(version) = self.version { + editor = editor.system_version(version); + } + for target_name in &self.remove_targets { + editor = editor.remove_target(target_name); + } + for path in self.add_artifacts { + editor = editor.guess_artifact(path).await?; + } + + let unsigned = editor.finish().await?; + let output = self.output.unwrap_or(self.repo); + self.sign_options + .sign(unsigned) + .await? + .write_zip_file(output, Utc::now()) + .await?; + Ok(()) + } +} diff --git a/bin/src/list_targets.rs b/bin/src/list_targets.rs new file mode 100644 index 0000000..c64a13c --- /dev/null +++ b/bin/src/list_targets.rs @@ -0,0 +1,32 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::BTreeSet; + +use anyhow::Result; +use camino::Utf8PathBuf; +use clap::Parser; +use tough::ExpirationEnforcement; +use tufaceous::RepositoryLoader; +use tufaceous::TrustStoreBehavior; + +#[derive(Debug, Parser)] +pub struct Args { + repo: Utf8PathBuf, +} + +impl Args { + pub async fn run(self) -> Result<()> { + let repo = RepositoryLoader::new() + .expiration_enforcement(ExpirationEnforcement::Unsafe) + .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .load_zip_file(self.repo.clone(), &crate::LOG) + .await?; + let target_names = repo.targets().keys().collect::>(); + for target_name in target_names { + println!("{}", target_name.raw()); + } + Ok(()) + } +} diff --git a/bin/src/main.rs b/bin/src/main.rs index 121c741..603cc15 100644 --- a/bin/src/main.rs +++ b/bin/src/main.rs @@ -3,6 +3,9 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. mod build; +mod edit; +mod list_targets; +mod show_target; mod sign; use std::sync::LazyLock; @@ -12,7 +15,7 @@ use clap::Parser; use slog::Drain; use slog::Logger; -static _LOG: LazyLock = LazyLock::new(|| { +static LOG: LazyLock = LazyLock::new(|| { let stderr_decorator = slog_term::TermDecorator::new().build(); let stderr_drain = slog_term::FullFormat::new(stderr_decorator).build().fuse(); @@ -31,6 +34,11 @@ static _LOG: LazyLock = LazyLock::new(|| { #[derive(Debug, Parser)] enum Command { Build(build::Args), + Edit(edit::Args), + #[clap(alias = "ls")] + ListTargets(list_targets::Args), + #[clap(aliases = ["cat", "show"])] + ShowTarget(show_target::Args), } #[tokio::main] @@ -38,5 +46,8 @@ async fn main() -> Result<()> { let args = Command::parse(); match args { Command::Build(args) => args.run().await, + Command::Edit(args) => args.run().await, + Command::ListTargets(args) => args.run().await, + Command::ShowTarget(args) => args.run().await, } } diff --git a/bin/src/show_target.rs b/bin/src/show_target.rs new file mode 100644 index 0000000..9d974d7 --- /dev/null +++ b/bin/src/show_target.rs @@ -0,0 +1,35 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::io::Write; + +use anyhow::Result; +use camino::Utf8PathBuf; +use clap::Parser; +use futures_util::TryStreamExt; +use tough::ExpirationEnforcement; +use tufaceous::RepositoryLoader; +use tufaceous::TrustStoreBehavior; + +#[derive(Debug, Parser)] +pub struct Args { + repo: Utf8PathBuf, + target_name: String, +} + +impl Args { + pub async fn run(self) -> Result<()> { + let repo = RepositoryLoader::new() + .expiration_enforcement(ExpirationEnforcement::Unsafe) + .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .load_zip_file(self.repo.clone(), &crate::LOG) + .await?; + let mut stream = repo.read_target(&self.target_name).await?; + let mut stdout = std::io::stdout().lock(); + while let Some(bytes) = stream.try_next().await? { + stdout.write_all(&bytes)?; + } + Ok(()) + } +} diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index f57b0bb..93cc4c0 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -483,6 +483,12 @@ impl<'a> RepositoryEditor<'a> { Ok(self) } + pub fn remove_target(mut self, target_name: &str) -> Self { + self.targets.remove(target_name); + self.artifacts.remove(target_name); + self + } + pub fn fake_artifact( mut self, target_name: String, @@ -582,14 +588,11 @@ impl<'a> RepositoryEditor<'a> { self } - pub fn from_repo(repo: &'a mut Repository) -> Result { + pub fn from_repo(repo: &'a Repository) -> Result { Self::new(repo.system_version().clone()).import_repo(repo) } - pub fn import_repo( - mut self, - repo: &'a mut Repository, - ) -> Result { + pub fn import_repo(mut self, repo: &'a Repository) -> Result { if repo.is_v1() { return Err(ErrorKind::ImportV1Repo.into()); } From e5f270ead5c9207691d555088acdfb610749ad4b Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Fri, 19 Dec 2025 15:34:05 -0800 Subject: [PATCH 006/137] match tokio-util version with omicron --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 64d5847..be950a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2737,9 +2737,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.17" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594" +checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" dependencies = [ "bytes", "futures-core", diff --git a/Cargo.toml b/Cargo.toml index 8967453..778b2ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,7 +49,7 @@ tar = "0.4.44" test-strategy = "0.4.0" thiserror = "2.0.12" tokio = "1.47.0" -tokio-util = "0.7.17" +tokio-util = "0.7.15" tough = "0.21.0" tufaceous.path = "lib" tufaceous-artifact.path = "artifact" From b6a79ba5dc9fb640e14659ee0beb04e6c6e89b02 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Fri, 19 Dec 2025 20:56:11 -0800 Subject: [PATCH 007/137] impl Eq for Artifacts --- artifact/src/artifact.rs | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/artifact/src/artifact.rs b/artifact/src/artifact.rs index 03024bc..bf58ae4 100644 --- a/artifact/src/artifact.rs +++ b/artifact/src/artifact.rs @@ -4,6 +4,7 @@ use std::collections::BTreeMap; use std::collections::HashMap; +use std::collections::HashSet; use std::collections::hash_map::IntoValues; use std::collections::hash_map::Values; use std::iter::Flatten; @@ -12,7 +13,7 @@ use crate::ArtifactHash; use crate::ArtifactVersion; use crate::KnownArtifactTags; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Artifact { pub target_name: String, pub version: ArtifactVersion, @@ -27,9 +28,9 @@ impl Artifact { } } -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct Artifacts { - inner: HashMap, Vec>, + inner: HashMap, HashSet>, } impl Artifacts { @@ -38,7 +39,7 @@ impl Artifacts { } pub fn len(&self) -> usize { - self.inner.values().map(Vec::len).sum() + self.inner.values().map(HashSet::len).sum() } pub fn is_empty(&self) -> bool { @@ -47,16 +48,25 @@ impl Artifacts { } pub fn insert(&mut self, artifact: Artifact) { - self.inner.entry(artifact.known_tags()).or_default().push(artifact); + self.inner.entry(artifact.known_tags()).or_default().insert(artifact); } pub fn get(&self, tags: KnownArtifactTags) -> Result<&Artifact, GetError> { let vec = self.inner.get(&Some(tags)).ok_or(GetError::NotFound)?; - if vec.len() == 1 { Ok(&vec[0]) } else { Err(GetError::TooMany) } + if vec.len() == 1 + && let Some(artifact) = vec.iter().next() + { + Ok(artifact) + } else { + Err(GetError::TooMany) + } } - pub fn get_all(&self, tags: KnownArtifactTags) -> &[Artifact] { - self.inner.get(&Some(tags)).map(Vec::as_slice).unwrap_or_default() + pub fn get_all( + &self, + tags: KnownArtifactTags, + ) -> impl Iterator { + self.inner.get(&Some(tags)).map(HashSet::iter).unwrap_or_default() } pub fn iter(&self) -> Iter<'_> { @@ -100,7 +110,7 @@ impl<'a> IntoIterator for &'a Artifacts { #[derive(Debug)] pub struct IntoIter { - inner: Flatten, Vec>>, + inner: Flatten, HashSet>>, } impl Iterator for IntoIter { @@ -113,7 +123,7 @@ impl Iterator for IntoIter { #[derive(Debug, Clone)] pub struct Iter<'a> { - inner: Flatten, Vec>>, + inner: Flatten, HashSet>>, } impl<'a> Iterator for Iter<'a> { From 2d998adfc601d187b2d9ef62476b434e3cdaf4da Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 8 Jan 2026 23:07:42 -0800 Subject: [PATCH 008/137] wip changes while working on omicron integration --- artifact/src/artifact.rs | 66 +++++++++++++++++++++++------ artifact/src/installinator.rs | 80 +++++++++++++++++++++++++++++++---- artifact/src/tags.rs | 6 ++- lib/src/edit/editor.rs | 2 +- lib/src/error.rs | 2 +- lib/src/repo.rs | 4 +- lib/src/repo/v1.rs | 33 +++++++-------- 7 files changed, 149 insertions(+), 44 deletions(-) diff --git a/artifact/src/artifact.rs b/artifact/src/artifact.rs index bf58ae4..65a357f 100644 --- a/artifact/src/artifact.rs +++ b/artifact/src/artifact.rs @@ -3,22 +3,26 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use std::collections::BTreeMap; -use std::collections::HashMap; -use std::collections::HashSet; -use std::collections::hash_map::IntoValues; -use std::collections::hash_map::Values; +use std::collections::BTreeSet; +use std::collections::btree_map::IntoValues; +use std::collections::btree_map::Values; use std::iter::Flatten; +use serde::Deserialize; +use serde::Serialize; + use crate::ArtifactHash; use crate::ArtifactVersion; use crate::KnownArtifactTags; -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, +)] pub struct Artifact { pub target_name: String, pub version: ArtifactVersion, pub tags: BTreeMap, - pub sha256: ArtifactHash, + pub hash: ArtifactHash, pub length: u64, } @@ -30,7 +34,7 @@ impl Artifact { #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct Artifacts { - inner: HashMap, HashSet>, + inner: BTreeMap, BTreeSet>, } impl Artifacts { @@ -39,7 +43,7 @@ impl Artifacts { } pub fn len(&self) -> usize { - self.inner.values().map(HashSet::len).sum() + self.inner.values().map(BTreeSet::len).sum() } pub fn is_empty(&self) -> bool { @@ -54,7 +58,7 @@ impl Artifacts { pub fn get(&self, tags: KnownArtifactTags) -> Result<&Artifact, GetError> { let vec = self.inner.get(&Some(tags)).ok_or(GetError::NotFound)?; if vec.len() == 1 - && let Some(artifact) = vec.iter().next() + && let Some(artifact) = vec.first() { Ok(artifact) } else { @@ -66,7 +70,7 @@ impl Artifacts { &self, tags: KnownArtifactTags, ) -> impl Iterator { - self.inner.get(&Some(tags)).map(HashSet::iter).unwrap_or_default() + self.inner.get(&Some(tags)).map(BTreeSet::iter).unwrap_or_default() } pub fn iter(&self) -> Iter<'_> { @@ -108,9 +112,47 @@ impl<'a> IntoIterator for &'a Artifacts { } } +impl<'de> Deserialize<'de> for Artifacts { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct Visitor; + + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = Artifacts; + + fn expecting( + &self, + f: &mut std::fmt::Formatter, + ) -> std::fmt::Result { + write!(f, "a sequence") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + std::iter::from_fn(|| seq.next_element().transpose()).collect() + } + } + + deserializer.deserialize_seq(Visitor) + } +} + +impl Serialize for Artifacts { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.collect_seq(self.iter()) + } +} + #[derive(Debug)] pub struct IntoIter { - inner: Flatten, HashSet>>, + inner: Flatten, BTreeSet>>, } impl Iterator for IntoIter { @@ -123,7 +165,7 @@ impl Iterator for IntoIter { #[derive(Debug, Clone)] pub struct Iter<'a> { - inner: Flatten, HashSet>>, + inner: Flatten, BTreeSet>>, } impl<'a> Iterator for Iter<'a> { diff --git a/artifact/src/installinator.rs b/artifact/src/installinator.rs index 2ed77c2..f0ab5b4 100644 --- a/artifact/src/installinator.rs +++ b/artifact/src/installinator.rs @@ -2,38 +2,48 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use std::borrow::Cow; + use serde::Deserialize; use serde::Serialize; use crate::ArtifactHash; -/// Artifact-specific information used by installinator. +/// Artifact-specific information used by Installinator. /// /// This document is treated as an opaque blob by Wicketd and Nexus, since /// we'd like previous versions of those services to be able to process newer /// versions of this document. /// /// There are no backwards compatibility constraints for this document. The -/// version of installinator that processes this document is the same as the +/// version of Installinator that processes this document is the same as the /// version of tufaceous that creates it. #[derive(Debug, Clone, Deserialize, Serialize)] pub struct InstallinatorDocument { pub artifacts: Vec, } -/// Describes an artifact available to installinator. -/// -/// The fields here match [`Artifact`](crate::Artifact). +/// Describes an artifact available to Installinator. #[derive(Debug, Clone, Deserialize, Serialize)] pub struct InstallinatorArtifact { - /// A file name; not necessarily the target name. - pub name: String, + /// A file name without directory separators; not necessarily the target + /// name. + pub file_name: String, #[serde(flatten)] pub kind: InstallinatorArtifactKind, pub sha256: ArtifactHash, } -/// The artifact kind for an installinator artifact. +impl InstallinatorArtifact { + pub fn downgrade(&self) -> InstallinatorArtifactId { + InstallinatorArtifactId { + kind: self.kind.downgrade(), + hash: self.sha256, + } + } +} + +/// The artifact kind for an Installinator artifact. #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(tag = "kind", rename_all = "kebab-case")] pub enum InstallinatorArtifactKind { @@ -42,5 +52,57 @@ pub enum InstallinatorArtifactKind { /// The host phase 2 artifact. HostPhase2, /// A control plane zone artifact. - Zone { name: String }, + Zone { zone_name: String }, +} + +impl InstallinatorArtifactKind { + pub fn downgrade(&self) -> InstallinatorArtifactKindId { + InstallinatorArtifactKindId(match self { + InstallinatorArtifactKind::MeasurementCorpus => { + Cow::Borrowed("measurement_corpus") + } + InstallinatorArtifactKind::HostPhase2 => { + Cow::Borrowed("host_phase2") + } + InstallinatorArtifactKind::Zone { zone_name } => { + Cow::Owned(format!("zone-{zone_name}")) + } + }) + } +} + +/// Identifies an artifact that Installinator wants or has used. +/// +/// Historically this was called `ArtifactHashId` and consists of two +/// strings: `kind` and `hash`. It was developed before the present system +/// of tags representing an artifact kind but remains in use for any +/// Installinator-related interfaces (namely Wicket and MUPdate overrides). +/// +/// This schema is stored to disk and should not change. +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize, +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub struct InstallinatorArtifactId { + pub kind: InstallinatorArtifactKindId, + pub hash: ArtifactHash, +} + +/// Encodes [`InstallinatorArtifactKind`] as a string. +/// +/// Used only in [`InstallinatorArtifactId`]. +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize, +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +#[serde(transparent)] +pub struct InstallinatorArtifactKindId(Cow<'static, str>); + +impl InstallinatorArtifactKindId { + pub const INSTALLINATOR_DOCUMENT: Self = + Self(Cow::Borrowed("installinator_document")); + + pub fn as_str(&self) -> &str { + &self.0 + } } diff --git a/artifact/src/tags.rs b/artifact/src/tags.rs index 7da3631..937a2cd 100644 --- a/artifact/src/tags.rs +++ b/artifact/src/tags.rs @@ -18,7 +18,7 @@ use crate::Sign; #[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] #[serde(tag = "kind", rename_all = "kebab-case")] pub enum KnownArtifactTags { - /// JSON document describing the artifacts installinator is responsible for + /// JSON document describing the artifacts Installinator is responsible for /// writing during mupdate and sled recovery. InstallinatorDocument {}, @@ -110,7 +110,9 @@ impl KnownArtifactTags { Some(InstallinatorArtifactKind::HostPhase2) } KnownArtifactTags::Zone { name } => { - Some(InstallinatorArtifactKind::Zone { name: name.clone() }) + Some(InstallinatorArtifactKind::Zone { + zone_name: name.clone(), + }) } _ => None, } diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index 93cc4c0..e0f879e 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -710,7 +710,7 @@ impl<'a> RepositoryEditor<'a> { && let Ok(sha256) = target.sha256.as_slice().try_into() { document.artifacts.push(InstallinatorArtifact { - name: file_name.to_owned(), + file_name: file_name.to_owned(), kind, sha256, }); diff --git a/lib/src/error.rs b/lib/src/error.rs index 28d4446..196caca 100644 --- a/lib/src/error.rs +++ b/lib/src/error.rs @@ -140,7 +140,7 @@ pub enum ErrorKind { TargetNameCollision { target_name: String }, #[error("failed to serialize artifacts document")] SerializeArtifacts(#[source] serde_json::Error), - #[error("failed to serialize installinator document")] + #[error("failed to serialize Installinator document")] SerializeInstallinator(#[source] serde_json::Error), #[error("no root provided to sign repository")] NoSigningRoot, diff --git a/lib/src/repo.rs b/lib/src/repo.rs index e30a9c6..b3c3b31 100644 --- a/lib/src/repo.rs +++ b/lib/src/repo.rs @@ -69,8 +69,8 @@ impl Repository { let artifacts = artifacts .into_iter() .filter_map(|ArtifactSchema { target_name, version, tags }| { - let (sha256, length) = sha256_length(&repo, log, &target_name)?; - Some(Artifact { target_name, version, tags, sha256, length }) + let (hash, length) = sha256_length(&repo, log, &target_name)?; + Some(Artifact { target_name, version, tags, hash, length }) }) .collect(); Ok(Repository { diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs index 48f1d08..69c738d 100644 --- a/lib/src/repo/v1.rs +++ b/lib/src/repo/v1.rs @@ -79,8 +79,7 @@ pub(crate) async fn from_loaded( continue; } }; - let Some((sha256, length)) = - sha256_length(&repo, log, &artifact.target) + let Some((hash, length)) = sha256_length(&repo, log, &artifact.target) else { continue; }; @@ -109,7 +108,7 @@ pub(crate) async fn from_loaded( target_name: artifact.target, version: artifact.version, tags: tags.to_tags(), - sha256, + hash, length, }); } @@ -137,7 +136,7 @@ pub(crate) async fn from_loaded( target_name: artifact.target, version: artifact.version, tags: tags.to_tags(), - sha256, + hash, length, }); } @@ -149,7 +148,7 @@ pub(crate) async fn from_loaded( let unpacked = CompositeArtifact::unpack(&repo, &target).await?; for (tar_path, inner) in unpacked.entries { - let UnpackedArtifact { file, sha256, length } = inner; + let UnpackedArtifact { file, hash, length } = inner; let slot = match tar_path.as_ref() { "archive-a.zip" => RotSlot::A, "archive-b.zip" => RotSlot::B, @@ -182,13 +181,13 @@ pub(crate) async fn from_loaded( let target_name = format!("{}/{tar_path}", artifact.target); entries.insert( target_name.clone(), - UnpackedArtifact { file, sha256, length }, + UnpackedArtifact { file, hash, length }, ); artifacts.insert(Artifact { target_name, version: artifact.version.clone(), tags: tags.to_tags(), - sha256, + hash, length, }); } @@ -218,7 +217,7 @@ pub(crate) async fn from_loaded( target_name: artifact.target, version: artifact.version, tags: KnownArtifactTags::InstallinatorDocument {}.to_tags(), - sha256, + hash, length, }); } @@ -227,7 +226,7 @@ pub(crate) async fn from_loaded( let unpacked = CompositeArtifact::unpack(&repo, &artifact.target).await?; for (tar_path, inner) in unpacked.entries { - let UnpackedArtifact { file, sha256, length } = inner; + let UnpackedArtifact { file, hash, length } = inner; if tar_path.starts_with("zones/") { let path = tar_path.to_string(); let (file, layer_info) = @@ -241,7 +240,7 @@ pub(crate) async fn from_loaded( format!("{}/{tar_path}", artifact.target); entries.insert( target_name.clone(), - UnpackedArtifact { file, sha256, length }, + UnpackedArtifact { file, hash, length }, ); artifacts.insert(Artifact { target_name, @@ -250,7 +249,7 @@ pub(crate) async fn from_loaded( name: layer_info.pkg, } .to_tags(), - sha256, + hash, length, }); } @@ -262,7 +261,7 @@ pub(crate) async fn from_loaded( target_name: artifact.target, version: artifact.version, tags: KnownArtifactTags::MeasurementCorpus {}.to_tags(), - sha256, + hash, length, }); } @@ -282,7 +281,7 @@ pub(crate) async fn from_loaded( #[derive(Debug, Clone)] pub(crate) struct UnpackedArtifact { file: Arc, - sha256: ArtifactHash, + hash: ArtifactHash, length: u64, } @@ -306,7 +305,7 @@ impl UnpackedArtifact { .await??; let bytes = buf.split().freeze(); if bytes.is_empty() { - let msg = if self.sha256 + let msg = if self.hash != ArtifactHash(hasher.finalize().into()) { "invalid checksum" @@ -387,8 +386,8 @@ impl CompositeArtifact { length += u64::try_from(n).unwrap(); } let file = Arc::new(file.into()); - let sha256 = ArtifactHash(hasher.finalize().into()); - entries.insert(path, UnpackedArtifact { file, sha256, length }); + let hash = ArtifactHash(hasher.finalize().into()); + entries.insert(path, UnpackedArtifact { file, hash, length }); } Ok(Self { entries }) }); @@ -422,7 +421,7 @@ fn unpack_os( target_name: target_name.clone(), version: artifact.version.clone(), tags: tags.to_tags(), - sha256: inner.sha256, + hash: inner.hash, length: inner.length, }); entries.insert(target_name, inner); From 243782f884bd8e069fa91e5e8c50917beef40966 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 12 Jan 2026 14:14:45 -0800 Subject: [PATCH 009/137] load repo from opened zip; flatten io error variants --- bin/src/edit.rs | 2 +- bin/src/list_targets.rs | 2 +- bin/src/show_target.rs | 2 +- lib/src/edit/editor.rs | 90 ++++++++++++++++++---------------------- lib/src/edit/sign.rs | 9 ++-- lib/src/edit/source.rs | 26 +++++------- lib/src/error.rs | 46 ++++++++++++-------- lib/src/loader.rs | 30 ++++++++++---- lib/src/repo/v1.rs | 22 +++++----- lib/src/util.rs | 9 ++-- lib/src/zip_transport.rs | 27 ++++++------ 11 files changed, 142 insertions(+), 123 deletions(-) diff --git a/bin/src/edit.rs b/bin/src/edit.rs index 40a679d..327a050 100644 --- a/bin/src/edit.rs +++ b/bin/src/edit.rs @@ -37,7 +37,7 @@ impl Args { let repo = RepositoryLoader::new() .expiration_enforcement(ExpirationEnforcement::Unsafe) .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) - .load_zip_file(self.repo.clone(), &crate::LOG) + .load_zip_path(self.repo.clone(), &crate::LOG) .await?; let mut editor = RepositoryEditor::from_repo(&repo)? .generate_installinator_document(!self.no_installinator_document); diff --git a/bin/src/list_targets.rs b/bin/src/list_targets.rs index c64a13c..d9421a1 100644 --- a/bin/src/list_targets.rs +++ b/bin/src/list_targets.rs @@ -21,7 +21,7 @@ impl Args { let repo = RepositoryLoader::new() .expiration_enforcement(ExpirationEnforcement::Unsafe) .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) - .load_zip_file(self.repo.clone(), &crate::LOG) + .load_zip_path(self.repo.clone(), &crate::LOG) .await?; let target_names = repo.targets().keys().collect::>(); for target_name in target_names { diff --git a/bin/src/show_target.rs b/bin/src/show_target.rs index 9d974d7..a395777 100644 --- a/bin/src/show_target.rs +++ b/bin/src/show_target.rs @@ -23,7 +23,7 @@ impl Args { let repo = RepositoryLoader::new() .expiration_enforcement(ExpirationEnforcement::Unsafe) .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) - .load_zip_file(self.repo.clone(), &crate::LOG) + .load_zip_path(self.repo.clone(), &crate::LOG) .await?; let mut stream = repo.read_target(&self.target_name).await?; let mut stdout = std::io::stdout().lock(); diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index e0f879e..f58993b 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -40,6 +40,7 @@ use crate::edit::source::Target; use crate::edit::source::TargetSource; use crate::error::Error; use crate::error::ErrorKind; +use crate::error::try_path; use crate::schema::ArtifactSchema; use crate::schema::ArtifactsSchema; @@ -101,9 +102,11 @@ impl<'a> RepositoryEditor<'a> { Some(corim) => corim, None => { let vec = source.read_to_end().await?; - ciborium::from_reader(vec.as_slice()).map_err(|s| { - ErrorKind::Corim { source: s, path: source.path.clone() } - })? + try_path!( + ciborium::from_reader(vec.as_slice()), + Corim, + source.path + ) } }; let sha256 = source.sha256().await?; @@ -178,37 +181,30 @@ impl<'a> RepositoryEditor<'a> { let tarball_path = output_dir.join("os.tar.gz"); let metadata_sources = tokio::task::spawn_blocking(move || { let mut sources = Vec::new(); - let file = - std::fs::File::open(&tarball_path).map_err(|source| { - ErrorKind::OpenFile { source, path: tarball_path.clone() } - })?; + let file = try_path!( + std::fs::File::open(&tarball_path), + OpenFile, + tarball_path + ); let mut archive = tar::Archive::new(GzDecoder::new(file)); - for entry in archive.entries().map_err(|source| { - ErrorKind::ReadFile { source, path: tarball_path.clone() } - })? { - let mut entry = entry.map_err(|source| { - ErrorKind::ReadFile { source, path: tarball_path.clone() } - })?; + for entry in try_path!(archive.entries(), ReadFile, tarball_path) { + let mut entry = try_path!(entry, ReadFile, tarball_path); if entry.header().entry_type() != tar::EntryType::Regular { continue; } - let path = entry - .path() - .and_then(|path| { + let path = try_path!( + entry.path().and_then(|path| { Utf8PathBuf::try_from(path.into_owned()) .map_err(|error| error.into_io_error()) - }) - .map_err(|source| ErrorKind::ReadFile { - source, - path: tarball_path.clone(), - })?; + }), + ReadFile, + tarball_path + ); if path == "image/zfs.img" { break; } let mut vec = Vec::new(); - entry.read_to_end(&mut vec).map_err(|source| { - ErrorKind::ReadFile { source, path: tarball_path.clone() } - })?; + try_path!(entry.read_to_end(&mut vec), ReadFile, tarball_path); sources.push((path, vec)); } Ok::<_, Error>(sources) @@ -368,16 +364,14 @@ impl<'a> RepositoryEditor<'a> { pub async fn zone_image(self, path: Utf8PathBuf) -> Result { let cloned_path = path.clone(); let (file, layer_info) = tokio::task::spawn_blocking(move || { - let file = std::fs::File::open(&path).map_err(|source| { - ErrorKind::ReadFile { source, path: path.clone() } - })?; + let file = try_path!(std::fs::File::open(&path), ReadFile, path); let mut archive = tar::Archive::new(GzDecoder::new(file)); - let layer_info = Metadata::read_from_tar(&mut archive) - .and_then(|metadata| metadata.layer_info().cloned()) - .map_err(|source| ErrorKind::ReadZoneOxideJson { - source, - path: path.clone(), - })?; + let layer_info = try_path!( + Metadata::read_from_tar(&mut archive) + .and_then(|metadata| metadata.layer_info().cloned()), + ReadZoneOxideJson, + path + ); Ok::<_, Error>((archive.into_inner().into_inner(), layer_info)) }) .await??; @@ -417,13 +411,9 @@ impl<'a> RepositoryEditor<'a> { return self.os_image_dir(variant, &path).await; } - let mut file = File::open(&path).await.map_err(|source| { - ErrorKind::OpenFile { source, path: path.clone() } - })?; + let mut file = try_path!(File::open(&path).await, OpenFile, path); let mut buf = [0; 4096]; - let n = file.read(&mut buf).await.map_err(|source| { - ErrorKind::ReadFile { source, path: path.clone() } - })?; + let n = try_path!(file.read(&mut buf).await, ReadFile, path); if n == 0 { // we're not going to try to guess an empty file return Err(ErrorKind::GuessArtifact { path }.into()); @@ -751,17 +741,17 @@ impl CabooseData { where F: FnOnce(&Caboose) -> Result, { - let tags = tag_fn(caboose).map_err(|source| { - ErrorKind::ReadCaboose { source, path: path.to_owned() } - })?; - let name = tufaceous_artifact::hubris::read_name(caboose).map_err( - |source| ErrorKind::ReadCaboose { source, path: path.to_owned() }, - )?; - let version = tufaceous_artifact::hubris::read_version(caboose) - .map_err(|source| ErrorKind::ReadCaboose { - source, - path: path.to_owned(), - })?; + let tags = try_path!(tag_fn(caboose), ReadCaboose, path); + let name = try_path!( + tufaceous_artifact::hubris::read_name(caboose), + ReadCaboose, + path + ); + let version = try_path!( + tufaceous_artifact::hubris::read_version(caboose), + ReadCaboose, + path + ); Ok(Self { tags, name: name.to_owned(), diff --git a/lib/src/edit/sign.rs b/lib/src/edit/sign.rs index e6d0ffe..c3a1e88 100644 --- a/lib/src/edit/sign.rs +++ b/lib/src/edit/sign.rs @@ -25,6 +25,7 @@ use crate::edit::source::Target; use crate::edit::source::TargetSource; use crate::error::Error; use crate::error::ErrorKind; +use crate::error::try_path; use crate::zip_writer::ZipWriter; pub(crate) const DEFAULT_VALIDITY: Duration = @@ -134,9 +135,11 @@ impl<'a> UnsignedRepository<'a> { // root role, which is mildly silly; we need to write the root out to a // temporary directory so it can read it back in again. let root_path = tempdir.path().join("root.json"); - tokio::fs::write(&root_path, &root).await.map_err(|source| { - ErrorKind::WriteFile { source, path: root_path.clone() } - })?; + try_path!( + tokio::fs::write(&root_path, &root).await, + WriteFile, + root_path + ); let mut editor = tough::editor::RepositoryEditor::new(&root_path).await?; editor diff --git a/lib/src/edit/source.rs b/lib/src/edit/source.rs index bdcf7f4..8132407 100644 --- a/lib/src/edit/source.rs +++ b/lib/src/edit/source.rs @@ -24,6 +24,7 @@ use tufaceous_artifact::ArtifactHash; use crate::Repository; use crate::error::Error; use crate::error::ErrorKind; +use crate::error::try_path; #[derive(Debug)] pub(crate) struct Target<'a> { @@ -96,9 +97,7 @@ pub(crate) struct FileSource { impl FileSource { pub(crate) async fn open(path: Utf8PathBuf) -> Result { - let file = File::open(&path).await.map_err(|source| { - ErrorKind::OpenFile { source, path: path.clone() } - })?; + let file = try_path!(File::open(&path).await, OpenFile, path); Ok(Self::from_file(file, path)) } @@ -180,18 +179,15 @@ impl FileSource { &mut self, ) -> impl Stream> { stream::once(async { - if let Err(source) = self.file.rewind().await { - return Err(Error::from(ErrorKind::SeekFile { - source, - path: self.path.clone(), - })); - } - Ok(ReaderStream::new(&mut self.file) - .map_err(|source| ErrorKind::ReadFile { - source, - path: self.path.clone(), - }) - .err_into::()) + try_path!(self.file.rewind().await, SeekFile, self.path.clone()); + Ok::<_, Error>( + ReaderStream::new(&mut self.file) + .map_err(|source| ErrorKind::ReadFile { + source, + path: Some(self.path.clone()), + }) + .err_into::(), + ) }) .try_flatten() } diff --git a/lib/src/error.rs b/lib/src/error.rs index 196caca..b6d7172 100644 --- a/lib/src/error.rs +++ b/lib/src/error.rs @@ -84,25 +84,20 @@ pub enum ErrorKind { archive_path: Option, }, - #[error("failed to open file {path}")] - OpenFile { source: std::io::Error, path: Utf8PathBuf }, - #[error("failed to read from file {path}")] - ReadFile { source: std::io::Error, path: Utf8PathBuf }, - #[error("failed to write to file {path}")] - WriteFile { source: std::io::Error, path: Utf8PathBuf }, - #[error("failed to seek in file {path}")] - SeekFile { source: std::io::Error, path: Utf8PathBuf }, - #[error("failed to read directory {path}")] - ReadDir { source: std::io::Error, path: Utf8PathBuf }, - - #[error("failed to create temporary file")] - CreateTempFile(#[source] std::io::Error), - #[error("failed to read from temporary file")] - ReadTempFile(#[source] std::io::Error), - #[error("failed to write to temporary file")] - WriteTempFile(#[source] std::io::Error), #[error("failed to create temporary directory")] CreateTempDir(#[source] std::io::Error), + #[error("failed to create temporary file")] + CreateTempFile(#[source] std::io::Error), + #[error("failed to open file{path}", path = SpacePath(path))] + OpenFile { source: std::io::Error, path: Option }, + #[error("failed to read directory{path}", path = SpacePath(path))] + ReadDir { source: std::io::Error, path: Option }, + #[error("failed to read from file{path}", path = SpacePath(path))] + ReadFile { source: std::io::Error, path: Option }, + #[error("failed to seek in file{path}", path = SpacePath(path))] + SeekFile { source: std::io::Error, path: Option }, + #[error("failed to write to file{path}", path = SpacePath(path))] + WriteFile { source: std::io::Error, path: Option }, #[error("failed to read hubris archive {path}")] ReadHubrisArchive { source: hubtools::Error, path: Utf8PathBuf }, @@ -190,6 +185,21 @@ impl Debug for DebugByteString<'_> { } } +macro_rules! try_path { + ($result:expr, $kind:ident, $path:expr) => { + match $result { + Ok(value) => value, + Err(source) => { + return Err( + ErrorKind::$kind { source, path: $path.into() }.into() + ) + } + } + }; +} + +pub(crate) use try_path; + #[cfg(test)] mod tests { use std::error::Error as _; @@ -202,7 +212,7 @@ mod tests { fn error_display_chain_doesnt_repeat() { let err = Error::from(ErrorKind::OpenFile { source: std::io::Error::from(std::io::ErrorKind::NotFound), - path: "/nowhere/in/particular".into(), + path: Some("/nowhere/in/particular".into()), }); let mut chain = err.to_string(); let mut source = err.source(); diff --git a/lib/src/loader.rs b/lib/src/loader.rs index af4fe9d..4c3d2d3 100644 --- a/lib/src/loader.rs +++ b/lib/src/loader.rs @@ -7,6 +7,8 @@ use std::fmt::Debug; use camino::Utf8PathBuf; use futures_util::TryStreamExt; use slog::Logger; +pub use tough::ExpirationEnforcement; +pub use tough::Limits; use url::Url; use crate::Repository; @@ -31,8 +33,8 @@ pub enum TrustStoreBehavior { #[derive(Debug, Clone, Default)] pub struct RepositoryLoader { - expiration_enforcement: tough::ExpirationEnforcement, - limits: tough::Limits, + expiration_enforcement: ExpirationEnforcement, + limits: Limits, metadata_base_url: Option, targets_base_url: Option, trust_roots: Vec>, @@ -51,13 +53,13 @@ impl RepositoryLoader { /// Set whether metadata expiration times are enforced. pub fn expiration_enforcement( self, - expiration_enforcement: tough::ExpirationEnforcement, + expiration_enforcement: ExpirationEnforcement, ) -> Self { Self { expiration_enforcement, ..self } } /// Set limits used while fetching repository metadata. - pub fn limits(self, limits: tough::Limits) -> Self { + pub fn limits(self, limits: Limits) -> Self { Self { limits, ..self } } @@ -124,13 +126,27 @@ impl RepositoryLoader { self.zip_base_urls().load(transport, log).await } - /// Load a Tufaceous-generated ZIP archive from a file. - pub async fn load_zip_file( + /// Load a Tufaceous-generated ZIP archive from a file path. + pub async fn load_zip_path( self, archive_path: Utf8PathBuf, log: &Logger, ) -> Result { - let transport = ZipTransport::from_file(archive_path, log).await?; + let transport = ZipTransport::from_path(archive_path, log).await?; + self.zip_base_urls().load(transport, log).await + } + + /// Load a Tufaceous-generated ZIP archive from an opened file. + /// + /// `archive_path` is used in errors, if available. + pub async fn load_zip_file( + self, + file: std::fs::File, + archive_path: Option, + log: &Logger, + ) -> Result { + let transport = + ZipTransport::from_file(file, archive_path, log).await?; self.zip_base_urls().load(transport, log).await } diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs index 69c738d..4478ab2 100644 --- a/lib/src/repo/v1.rs +++ b/lib/src/repo/v1.rs @@ -160,10 +160,9 @@ pub(crate) async fn from_loaded( .map(Vec::with_capacity) .unwrap_or_default(); let image = tokio::task::spawn_blocking(move || { - reader - .read_to_end(&mut vec) - .map(|_| vec) - .map_err(ErrorKind::ReadTempFile) + reader.read_to_end(&mut vec).map(|_| vec).map_err( + |source| ErrorKind::ReadFile { source, path: None }, + ) }) .await??; let caboose = RawHubrisArchive::from_vec(image) @@ -296,9 +295,9 @@ impl UnpackedArtifact { buf.resize(buf.capacity(), 0); let file = self.file.clone(); let mut buf = tokio::task::spawn_blocking(move || { - let n = file - .read_at(&mut buf, bytes_read) - .map_err(ErrorKind::ReadTempFile)?; + let n = file.read_at(&mut buf, bytes_read).map_err( + |source| ErrorKind::ReadFile { source, path: None }, + )?; buf.truncate(n); Ok::<_, Error>(buf) }) @@ -318,7 +317,9 @@ impl UnpackedArtifact { std::io::ErrorKind::InvalidData, msg, ); - return Err(ErrorKind::ReadTempFile(source).into()); + return Err( + ErrorKind::ReadFile { source, path: None }.into() + ); } hasher.update(&bytes); bytes_read += u64::try_from(bytes.len()).unwrap(); @@ -380,8 +381,9 @@ impl CompositeArtifact { } })? && n > 0 { - file.write_all(&buf[..n]) - .map_err(ErrorKind::WriteTempFile)?; + file.write_all(&buf[..n]).map_err(|source| { + ErrorKind::WriteFile { source, path: None } + })?; hasher.update(&buf[..n]); length += u64::try_from(n).unwrap(); } diff --git a/lib/src/util.rs b/lib/src/util.rs index 6d946ed..a955e36 100644 --- a/lib/src/util.rs +++ b/lib/src/util.rs @@ -17,6 +17,7 @@ use tufaceous_brand_metadata::Metadata; use crate::error::Error; use crate::error::ErrorKind; +use crate::error::try_path; pub(crate) async fn read_zone_layer_info( reader: R, @@ -33,10 +34,8 @@ pub(crate) async fn read_zone_layer_info( } pub(crate) async fn read_dir(path: Utf8PathBuf) -> Result { - match tokio::fs::read_dir(&path).await { - Ok(inner) => Ok(ReadDir { inner, path }), - Err(source) => Err(ErrorKind::ReadDir { source, path }.into()), - } + let inner = try_path!(tokio::fs::read_dir(&path).await, ReadDir, path); + Ok(ReadDir { inner, path }) } pub(crate) struct ReadDir { @@ -55,7 +54,7 @@ impl Stream for ReadDir { result .and_then(|option| option.map(DirEntry::new).transpose()) .map_err(|source| { - ErrorKind::ReadDir { source, path: self.path.clone() } + ErrorKind::ReadDir { source, path: Some(self.path.clone()) } .into() }) .transpose() diff --git a/lib/src/zip_transport.rs b/lib/src/zip_transport.rs index ac6fe22..09fc8f0 100644 --- a/lib/src/zip_transport.rs +++ b/lib/src/zip_transport.rs @@ -38,6 +38,7 @@ use url::Url; use crate::error::Error; use crate::error::ErrorKind; +use crate::error::try_path; const MAX_SYMLINK_TARGET_LEN: usize = 1024; const MAX_SYMLINK_TRAVERSAL: usize = 8; @@ -95,25 +96,27 @@ impl + Debug + Send + Sync + 'static> ZipTransport> { } impl ZipTransport { - pub async fn from_file( + pub async fn from_path( archive_path: Utf8PathBuf, log: &Logger, + ) -> Result { + let file = try_path!( + tokio::fs::File::open(&archive_path).await, + OpenFile, + archive_path + ); + Self::from_file(file.into_std().await, Some(archive_path), log).await + } + + pub async fn from_file( + file: File, + archive_path: Option, + log: &Logger, ) -> Result { let log = log.clone(); tokio::task::spawn_blocking(move || { let archive_path = archive_path; - let file = match File::open(&archive_path) { - Ok(file) => file, - Err(source) => { - return Err(ErrorKind::OpenFile { - source, - path: archive_path, - } - .into()); - } - }; let mut buffer = vec![0; rawzip::RECOMMENDED_BUFFER_SIZE]; - let archive_path = Some(archive_path); let archive = ZipArchive::from_file(file, &mut buffer) .upgrade(archive_path.as_ref())?; Self::from_impl_blocking(archive, archive_path, Some(buffer), &log) From 10f3578d615fd5b41e9db6aa1df6bb81a4089703 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 13 Jan 2026 09:55:26 -0800 Subject: [PATCH 010/137] fix incorrect conversion into TransportErrorKind::FileNotFound --- lib/src/zip_transport.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/src/zip_transport.rs b/lib/src/zip_transport.rs index 09fc8f0..388861a 100644 --- a/lib/src/zip_transport.rs +++ b/lib/src/zip_transport.rs @@ -413,19 +413,17 @@ pub enum ZipTransportError { impl ZipTransportError { fn upgrade(self, url: Url) -> TransportError { let kind = match &self { + ZipTransportError::FileNotFound => TransportErrorKind::FileNotFound, + ZipTransportError::Io(_) | ZipTransportError::Join(_) - | ZipTransportError::Zip(_) => TransportErrorKind::Other, - - ZipTransportError::UrlJoin { .. } + | ZipTransportError::UrlJoin { .. } + | ZipTransportError::Zip(_) | ZipTransportError::CompressionMethod(_) | ZipTransportError::Duplicate - | ZipTransportError::FileNotFound | ZipTransportError::IsADirectory | ZipTransportError::SymlinkTraversalLimit - | ZipTransportError::SymlinkUtf8(_) => { - TransportErrorKind::FileNotFound - } + | ZipTransportError::SymlinkUtf8(_) => TransportErrorKind::Other, }; TransportError::new_with_cause(kind, url, self) } From fdd859eb5930782efe119e0a16cfeea260119edc Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 13 Jan 2026 13:26:28 -0800 Subject: [PATCH 011/137] ability to retrieve archive path/sha256 from loaded repo --- lib/src/error.rs | 4 ++ lib/src/loader.rs | 113 ++++++++++++++++++++++++++++++++++++--- lib/src/repo.rs | 24 +++++++++ lib/src/repo/v1.rs | 2 + lib/src/zip_transport.rs | 13 ----- lib/tests/tests.rs | 50 +++++++++++++++++ 6 files changed, 187 insertions(+), 19 deletions(-) diff --git a/lib/src/error.rs b/lib/src/error.rs index b6d7172..5220b3a 100644 --- a/lib/src/error.rs +++ b/lib/src/error.rs @@ -98,6 +98,10 @@ pub enum ErrorKind { SeekFile { source: std::io::Error, path: Option }, #[error("failed to write to file{path}", path = SpacePath(path))] WriteFile { source: std::io::Error, path: Option }, + #[error("failed to read stream")] + ReadStream(#[source] Box), + #[error("stream exceeded length limit ({limit})")] + StreamLimit { limit: u64 }, #[error("failed to read hubris archive {path}")] ReadHubrisArchive { source: hubtools::Error, path: Utf8PathBuf }, diff --git a/lib/src/loader.rs b/lib/src/loader.rs index 4c3d2d3..3deeb8e 100644 --- a/lib/src/loader.rs +++ b/lib/src/loader.rs @@ -4,9 +4,15 @@ use std::fmt::Debug; +use bytes::Bytes; use camino::Utf8PathBuf; +use futures_util::Stream; use futures_util::TryStreamExt; +use futures_util::pin_mut; +use sha2::Digest; +use sha2::Sha256; use slog::Logger; +use tokio::io::AsyncWriteExt; pub use tough::ExpirationEnforcement; pub use tough::Limits; use url::Url; @@ -15,6 +21,7 @@ use crate::Repository; use crate::ZipTransport; use crate::error::Error; use crate::error::ErrorKind; +use crate::error::try_path; /// Controls how repository trust is established. /// @@ -33,6 +40,7 @@ pub enum TrustStoreBehavior { #[derive(Debug, Clone, Default)] pub struct RepositoryLoader { + compute_archive_sha256: bool, expiration_enforcement: ExpirationEnforcement, limits: Limits, metadata_base_url: Option, @@ -50,6 +58,15 @@ impl RepositoryLoader { Self::default() } + /// Set whether to compute and store the SHA256 digest of the archive, if an + /// archive is used to load the repository. + /// + /// Only affects [`Self::load_zip_buffer`], [`Self::load_zip_path`], + /// [`Self::load_zip_file`], and [`Self::load_zip_stream`]. + pub fn compute_archive_sha256(self, compute_archive_sha256: bool) -> Self { + Self { compute_archive_sha256, ..self } + } + /// Set whether metadata expiration times are enforced. pub fn expiration_enforcement( self, @@ -122,8 +139,19 @@ impl RepositoryLoader { where T: AsRef<[u8]> + Debug + Send + Sync + 'static, { + let (data, sha256) = if self.compute_archive_sha256 { + tokio::task::spawn_blocking(move || { + let digest = Sha256::digest(&data).into(); + (data, Some(digest)) + }) + .await? + } else { + (data, None) + }; let transport = ZipTransport::from_slice(data, log)?; - self.zip_base_urls().load(transport, log).await + let mut repo = self.zip_base_urls().load(transport, log).await?; + repo.archive_sha256 = sha256; + Ok(repo) } /// Load a Tufaceous-generated ZIP archive from a file path. @@ -132,8 +160,12 @@ impl RepositoryLoader { archive_path: Utf8PathBuf, log: &Logger, ) -> Result { - let transport = ZipTransport::from_path(archive_path, log).await?; - self.zip_base_urls().load(transport, log).await + let file = try_path!( + tokio::fs::File::open(&archive_path).await, + OpenFile, + archive_path + ); + self.load_zip_file(file.into_std().await, Some(archive_path), log).await } /// Load a Tufaceous-generated ZIP archive from an opened file. @@ -141,13 +173,80 @@ impl RepositoryLoader { /// `archive_path` is used in errors, if available. pub async fn load_zip_file( self, - file: std::fs::File, + mut file: std::fs::File, archive_path: Option, log: &Logger, ) -> Result { + let (file, sha256) = if self.compute_archive_sha256 { + let path = archive_path.clone(); + tokio::task::spawn_blocking(move || { + let mut hasher = Sha256::new(); + try_path!( + std::io::copy(&mut file, &mut hasher), + ReadFile, + path + ); + Ok::<_, Error>((file, Some(hasher.finalize().into()))) + }) + .await?? + } else { + (file, None) + }; + let transport = - ZipTransport::from_file(file, archive_path, log).await?; - self.zip_base_urls().load(transport, log).await + ZipTransport::from_file(file, archive_path.clone(), log).await?; + let mut repo = self.zip_base_urls().load(transport, log).await?; + repo.archive_path = archive_path; + repo.archive_sha256 = sha256; + Ok(repo) + } + + /// Load a Tufaceous-generated ZIP archive from a stream. + /// + /// This writes the contents of the stream to a temporary file. You can + /// optionally provide a maximum number of bytes that will be written. If + /// this is not provided, you should ensure something else is limiting the + /// size of this stream (such as an HTTP body length limit). + pub async fn load_zip_stream( + self, + stream: impl Stream>, + limit: Option, + log: &Logger, + ) -> Result + where + E: std::error::Error + Send + Sync + 'static, + { + pin_mut!(stream); + let file = + camino_tempfile::tempfile().map_err(ErrorKind::CreateTempFile)?; + let mut file = tokio::fs::File::from(file); + let mut hasher = self.compute_archive_sha256.then(Sha256::new); + let mut bytes_read = limit.map(|_| 0u64); + while let Some(chunk) = stream + .try_next() + .await + .map_err(|source| ErrorKind::ReadStream(Box::new(source)))? + { + if let Some(limit) = limit + && let Some(bytes_read) = bytes_read.as_mut() + { + *bytes_read = u64::try_from(chunk.len()) + .ok() + .and_then(|len| bytes_read.checked_add(len)) + .ok_or(ErrorKind::StreamLimit { limit })?; + if *bytes_read > limit { + return Err(ErrorKind::StreamLimit { limit }.into()); + } + } + if let Some(hasher) = hasher.as_mut() { + hasher.update(&chunk); + } + try_path!(file.write_all(&chunk).await, WriteFile, None); + } + let mut repo = + self.load_zip_file(file.into_std().await, None, log).await?; + repo.archive_sha256 = hasher.map(|hasher| hasher.finalize().into()); + Ok(repo) } /// Load a repository from the configured metadata and targets base URLs @@ -208,6 +307,8 @@ impl RepositoryLoader { repo, log, trust_root, + None, + None, self.v1_compatibility, ) .await; diff --git a/lib/src/repo.rs b/lib/src/repo.rs index b3c3b31..830a16a 100644 --- a/lib/src/repo.rs +++ b/lib/src/repo.rs @@ -9,6 +9,8 @@ use std::collections::HashMap; use std::pin::Pin; use bytes::Bytes; +use camino::Utf8Path; +use camino::Utf8PathBuf; use futures_util::Stream; use futures_util::TryStreamExt; use semver::Version; @@ -34,6 +36,8 @@ pub struct Repository { inner: tough::Repository, system_version: Version, trust_root: Vec, + pub(crate) archive_path: Option, + pub(crate) archive_sha256: Option<[u8; 32]>, artifacts: Artifacts, metadata: BTreeMap, v1_unpacked: Option, @@ -48,6 +52,8 @@ impl Repository { repo: tough::Repository, log: &Logger, trust_root: Vec, + archive_path: Option, + archive_sha256: Option<[u8; 32]>, v1_compatibility: bool, ) -> Result { let Some(ArtifactsSchema { system_version, artifacts, metadata }) = @@ -76,6 +82,8 @@ impl Repository { Ok(Repository { inner: repo, trust_root, + archive_path, + archive_sha256, system_version, artifacts, metadata, @@ -91,6 +99,22 @@ impl Repository { &self.trust_root } + /// Returns the path to the loaded archive, if there is one. + /// + /// Set when [`RepositoryLoader::load_zip_file`] or + /// [`RepositoryLoader::load_zip_path`] are used. + pub fn archive_path(&self) -> Option<&Utf8Path> { + self.archive_path.as_deref() + } + + /// Returns the SHA256 digest of the loaded archive, if there is one. + /// + /// Set when [`RepositoryLoader::compute_archive_sha256`] is set to `true` + /// and one of the archive loading methods is used. + pub fn archive_sha256(&self) -> Option<&[u8; 32]> { + self.archive_sha256.as_ref() + } + pub fn targets(&self) -> &HashMap { &self.inner.targets().signed.targets } diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs index 4478ab2..ef4341a 100644 --- a/lib/src/repo/v1.rs +++ b/lib/src/repo/v1.rs @@ -271,6 +271,8 @@ pub(crate) async fn from_loaded( inner: repo, system_version, trust_root, + archive_path: None, + archive_sha256: None, artifacts: Artifacts::new(artifacts), metadata: BTreeMap::new(), v1_unpacked: Some(Unpacked { entries }), diff --git a/lib/src/zip_transport.rs b/lib/src/zip_transport.rs index 388861a..b194ddb 100644 --- a/lib/src/zip_transport.rs +++ b/lib/src/zip_transport.rs @@ -38,7 +38,6 @@ use url::Url; use crate::error::Error; use crate::error::ErrorKind; -use crate::error::try_path; const MAX_SYMLINK_TARGET_LEN: usize = 1024; const MAX_SYMLINK_TRAVERSAL: usize = 8; @@ -96,18 +95,6 @@ impl + Debug + Send + Sync + 'static> ZipTransport> { } impl ZipTransport { - pub async fn from_path( - archive_path: Utf8PathBuf, - log: &Logger, - ) -> Result { - let file = try_path!( - tokio::fs::File::open(&archive_path).await, - OpenFile, - archive_path - ); - Self::from_file(file.into_std().await, Some(archive_path), log).await - } - pub async fn from_file( file: File, archive_path: Option, diff --git a/lib/tests/tests.rs b/lib/tests/tests.rs index a971ee3..e30563b 100644 --- a/lib/tests/tests.rs +++ b/lib/tests/tests.rs @@ -2,8 +2,14 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use std::convert::Infallible; +use std::io::Write; + +use bytes::BufMut; +use bytes::BytesMut; use chrono::Utc; use futures_util::TryStreamExt; +use futures_util::stream; use semver::Version; use tufaceous::RepositoryLoader; use tufaceous::TrustStoreBehavior; @@ -91,3 +97,47 @@ async fn empty_artifact() -> Result<(), Error> { assert!(data.is_empty()); Ok(()) } + +#[tokio::test] +async fn compute_archive_sha256() -> Result<(), Error> { + let log = slog::Logger::root(slog::Discard, slog::o!()); + let zip = RepositoryEditor::fake(VERSION)? + .finish() + .await? + .generate_root() + .sign() + .await? + .write_zip(BytesMut::new().writer(), Utc::now()) + .await? + .into_inner() + .freeze(); + for should_compute in [false, true] { + let repo = RepositoryLoader::new() + .compute_archive_sha256(should_compute) + .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .load_zip_buffer(zip.clone(), &log) + .await?; + assert_eq!(repo.archive_sha256().is_some(), should_compute); + + let mut file = camino_tempfile::tempfile().unwrap(); + file.write_all(&zip).unwrap(); + let repo = RepositoryLoader::new() + .compute_archive_sha256(should_compute) + .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .load_zip_file(file, None, &log) + .await?; + assert_eq!(repo.archive_sha256().is_some(), should_compute); + + let repo = RepositoryLoader::new() + .compute_archive_sha256(should_compute) + .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .load_zip_stream( + stream::once(async { Ok::<_, Infallible>(zip.clone()) }), + None, + &log, + ) + .await?; + assert_eq!(repo.archive_sha256().is_some(), should_compute); + } + Ok(()) +} From cae592ac8b2ae60bf06c092aec65461a50918de3 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Wed, 14 Jan 2026 12:19:48 -0800 Subject: [PATCH 012/137] impl From for Sign --- artifact/src/sign.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/artifact/src/sign.rs b/artifact/src/sign.rs index 644f7c8..34726b5 100644 --- a/artifact/src/sign.rs +++ b/artifact/src/sign.rs @@ -97,6 +97,15 @@ impl Display for Sign { } } +impl From for Sign +where + Option: From, +{ + fn from(sign: T) -> Self { + Self(sign.into()) + } +} + #[cfg(test)] mod tests { use std::collections::HashSet; From 734c37601e01df20f478bf78d27826d10908b6f5 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 15 Jan 2026 14:15:21 -0800 Subject: [PATCH 013/137] refactor RepositoryEditor::fake --- bin/src/build.rs | 9 +- lib/src/edit/editor.rs | 511 ++++++++++++++++++++++++++++------------- lib/src/edit/sign.rs | 5 +- lib/src/edit/source.rs | 36 ++- lib/src/error.rs | 2 + lib/src/util.rs | 4 + 6 files changed, 396 insertions(+), 171 deletions(-) diff --git a/bin/src/build.rs b/bin/src/build.rs index 9dde6bd..73a804f 100644 --- a/bin/src/build.rs +++ b/bin/src/build.rs @@ -15,6 +15,8 @@ use crate::sign::SignOptions; pub struct Args { artifacts: Vec, #[clap(long)] + fake: bool, + #[clap(long)] no_installinator_document: bool, #[clap(short = 'o', long)] output: Utf8PathBuf, @@ -26,7 +28,12 @@ pub struct Args { impl Args { pub async fn run(self) -> Result<()> { - let mut editor = RepositoryEditor::new(self.version) + let mut editor = if self.fake { + RepositoryEditor::fake(self.version)? + } else { + RepositoryEditor::new(self.version) + }; + editor = editor .generate_installinator_document(!self.no_installinator_document); for path in self.artifacts { editor = editor.guess_artifact(path).await?; diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index f58993b..cc09271 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -5,20 +5,29 @@ use std::collections::BTreeMap; use std::collections::HashMap; use std::collections::HashSet; -use std::io::Read; +use std::path::Path; +use bytes::BufMut; +use bytes::BytesMut; use camino::Utf8Path; use camino::Utf8PathBuf; use flate2::read::GzDecoder; use futures_util::FutureExt; use futures_util::TryFutureExt; +use futures_util::TryStreamExt; use hubtools::Caboose; +use hubtools::CabooseBuilder; +use hubtools::HubrisArchiveBuilder; use rats_corim::Corim; +use rats_corim::CorimBuilder; use semver::Version; +use sha2::Digest; +use sha2::Sha256; use tokio::fs::File; use tokio::io::AsyncReadExt; use tokio::task::JoinSet; use tufaceous_artifact::ArtifactVersion; +use tufaceous_artifact::ArtifactVersionError; use tufaceous_artifact::InstallinatorArtifact; use tufaceous_artifact::InstallinatorDocument; use tufaceous_artifact::KnownArtifactTags; @@ -47,9 +56,14 @@ use crate::schema::ArtifactsSchema; const KIB: usize = 1024; const MIB: usize = 1024 * KIB; +const COSMO_PHASE_1_PATH: &str = "cosmo.rom"; +const GIMLET_PHASE_1_PATH: &str = "gimlet.rom"; +const PHASE_2_PATH: &str = "zfs.img"; + #[derive(Debug)] pub struct RepositoryEditor<'a> { system_version: Version, + artifact_version: Result, generate_installinator_document: bool, targets: HashMap>>, artifacts: HashMap>, @@ -60,6 +74,7 @@ impl<'a> RepositoryEditor<'a> { /// Create an empty repository editor. pub fn new(system_version: Version) -> Self { Self { + artifact_version: ArtifactVersion::new(system_version.to_string()), system_version, generate_installinator_document: true, targets: HashMap::new(), @@ -90,29 +105,58 @@ impl<'a> RepositoryEditor<'a> { self, path: Utf8PathBuf, ) -> Result { - self.measurement_corpus_inner(FileSource::open(path).await?, None).await + let source = FileSource::open(path).await?; + self.measurement_corpus_file(source, None).await } - async fn measurement_corpus_inner( - mut self, + async fn measurement_corpus_file( + self, mut source: FileSource, corim: Option, ) -> Result { let Corim { id, .. } = match corim { Some(corim) => corim, None => { - let vec = source.read_to_end().await?; + let v = source.read_to_end().await?; try_path!( - ciborium::from_reader(vec.as_slice()), + ciborium::from_reader(v.as_slice()), Corim, - source.path + &source.path ) } }; let sha256 = source.sha256().await?; + self.measurement_corpus_inner(source.into(), &id, &sha256.0) + } + + pub fn fake_measurement_corpus(self, hashes: usize) -> Result { + let mut builder = CorimBuilder::new(); + builder.vendor("fake-vendor".to_string()); + builder.id("fake-measurement-id".to_string()); + builder.tag_id("fake-tag-id".to_string()); + for i in 0..hashes { + builder.add_hash(format!("layer{i}"), 10, vec![0; 32]); + } + let corim = builder.build().unwrap(); + + let mut writer = BytesMut::new().writer(); + ciborium::into_writer(&corim, &mut writer).unwrap(); + let bytes = writer.into_inner().freeze(); + let sha256 = Sha256::digest(&bytes); + let source = BytesSource(bytes).into(); + + self.measurement_corpus_inner(source, &corim.id, &sha256) + } + + fn measurement_corpus_inner( + mut self, + source: TargetSource<'a>, + corim_id: &str, + sha256: &[u8], + ) -> Result { let target_name = - format!("measurements/{id}-{}.cbor", hex::encode(sha256)); - let version = ArtifactVersion::new(self.system_version.to_string())?; + format!("measurements/{corim_id}-{}.cbor", hex::encode(sha256)); + let version = self.artifact_version.clone()?; let tags = KnownArtifactTags::MeasurementCorpus {}; self.insert_artifact(target_name, version, tags, source); Ok(self) @@ -142,88 +186,149 @@ impl<'a> RepositoryEditor<'a> { /// /// `output_dir` is a path to the output directory for `helios-build image` /// (the `-o` argument). This directory contains `cosmo.rom`, `gimlet.rom`, - /// `zfs.img`, and `os.tar.gz`. Metadata stored in `os.tar.gz` is copied - /// into the repository. + /// and `zfs.img`. pub async fn os_image_dir( - mut self, + self, variant: OsVariant, output_dir: &Utf8Path, ) -> Result { + let cosmo_phase_1 = + FileSource::open(output_dir.join(COSMO_PHASE_1_PATH)).await?.into(); + let gimlet_phase_1 = + FileSource::open(output_dir.join(GIMLET_PHASE_1_PATH)) + .await? + .into(); + let phase_2 = + FileSource::open(output_dir.join(PHASE_2_PATH)).await?.into(); + + // If `os.tar.gz` is present, read the `image/*.txt` files from it + // as extra sources. Stop once we get to `zfs.img`, which comes after + // the additional metadata. (If a GzDecoder was seekable we could + // conceivably skip past any files we don't want to read, but alas.) + // https://github.com/oxidecomputer/helios/blob/f145457f6ccb13a139b8d93408b9b4de5db57bd6/tools/helios-build/src/main.rs#L1881-L1885 + let tarball_path = output_dir.join("os.tar.gz"); + let mut extra_targets = match tokio::fs::File::open(&tarball_path).await + { + Ok(file) => { + let file = file.into_std().await; + tokio::task::spawn_blocking(move || { + read_os_tarball_metadata_blocking(file, tarball_path) + }) + .await?? + } + Err(source) if source.kind() == std::io::ErrorKind::NotFound => { + // We might be reading from an unpacked Tufaceous archive; look + // for any `*.txt` files and include them. + let mut extra_targets = HashMap::new(); + let mut read_dir = + crate::util::read_dir(output_dir.into()).await?; + while let Some(entry) = read_dir.try_next().await? { + if entry.path().extension() != Some("txt") { + continue; + } + let file_name = Utf8Path::new(entry.file_name()).to_owned(); + let source = FileSource::open(entry.into_path()).await?; + extra_targets.insert(file_name, source.into()); + } + extra_targets + } + Err(source) => { + return Err(ErrorKind::OpenFile { + source, + path: Some(tarball_path), + } + .into()); + } + }; + for path in ["unix.z", "cpio.z"] { + let source = FileSource::open(output_dir.join(path)).await?; + extra_targets.insert(path.into(), source.into()); + } + + self.os_image_inner(OsImageSources { + variant, + cosmo_phase_1, + gimlet_phase_1, + phase_2, + extra_targets, + }) + } + + pub fn fake_os_image(self, variant: OsVariant) -> Result { + let version = &self.system_version; + let cosmo_phase_1 = FakeSource::new( + format!("cosmo host phase 1 image version {version}\n"), + MIB, + ) + .into(); + let gimlet_phase_1 = FakeSource::new( + format!("gimlet host phase 1 image version {version}\n"), + MIB, + ) + .into(); + let phase_2 = FakeSource::new( + format!("host phase 2 image version {version}\n"), + 4 * MIB, + ) + .into(); + + let mut extra_targets: HashMap = HashMap::new(); + extra_targets.insert( + "unix.z".into(), + FakeSource::new(format!("unix.z version {version}\n"), 64 * KIB) + .into(), + ); + extra_targets.insert( + "cpio.z".into(), + FakeSource::new(format!("cpio.z version {version}\n"), 256 * KIB) + .into(), + ); + + self.os_image_inner(OsImageSources { + variant, + cosmo_phase_1, + gimlet_phase_1, + phase_2, + extra_targets, + }) + } + + fn os_image_inner( + mut self, + sources: OsImageSources<'a>, + ) -> Result { + let variant = sources.variant; let base = Utf8PathBuf::from(format!("os-{variant}")); - let version = ArtifactVersion::new(self.system_version.to_string())?; + let version = self.artifact_version.clone()?; self.insert_artifact( - base.join("image/cosmo.rom").into(), + base.join("image").join(COSMO_PHASE_1_PATH).into(), version.clone(), KnownArtifactTags::OsPhase1 { variant, board: OsBoard::Cosmo }, - FileSource::open(output_dir.join("cosmo.rom")).await?, + sources.cosmo_phase_1, ); self.insert_artifact( - base.join("image/gimlet.rom").into(), + base.join("image").join(GIMLET_PHASE_1_PATH).into(), version.clone(), KnownArtifactTags::OsPhase1 { variant, board: OsBoard::Gimlet }, - FileSource::open(output_dir.join("gimlet.rom")).await?, + sources.gimlet_phase_1, ); self.insert_artifact( - base.join("image/zfs.img").into(), + base.join("image").join(PHASE_2_PATH).into(), version, KnownArtifactTags::OsPhase2 { variant }, - FileSource::open(output_dir.join("zfs.img")).await?, + sources.phase_2, ); - for path in ["cpio.z", "unix.z"] { - let source = FileSource::open(output_dir.join(path)).await?; + for (file_name, source) in sources.extra_targets { self.targets - .entry(base.join("image").join(path).into()) + .entry(base.join(file_name).into()) .or_default() - .push(TargetSource::File(source)); + .push(source); } - - let tarball_path = output_dir.join("os.tar.gz"); - let metadata_sources = tokio::task::spawn_blocking(move || { - let mut sources = Vec::new(); - let file = try_path!( - std::fs::File::open(&tarball_path), - OpenFile, - tarball_path - ); - let mut archive = tar::Archive::new(GzDecoder::new(file)); - for entry in try_path!(archive.entries(), ReadFile, tarball_path) { - let mut entry = try_path!(entry, ReadFile, tarball_path); - if entry.header().entry_type() != tar::EntryType::Regular { - continue; - } - let path = try_path!( - entry.path().and_then(|path| { - Utf8PathBuf::try_from(path.into_owned()) - .map_err(|error| error.into_io_error()) - }), - ReadFile, - tarball_path - ); - if path == "image/zfs.img" { - break; - } - let mut vec = Vec::new(); - try_path!(entry.read_to_end(&mut vec), ReadFile, tarball_path); - sources.push((path, vec)); - } - Ok::<_, Error>(sources) - }) - .await??; - for (path, source) in metadata_sources { - self.targets - .entry(base.join(path).into()) - .or_default() - .push(TargetSource::Bytes(BytesSource(source.into()))); - } - Ok(self) } async fn guess_os_image(path: &Utf8Path) -> Option { - if !path.join("os.tar.gz").exists() { - return None; - } let mut file = File::open(path.join("zfs.img")).await.ok()?; // Read the header block from the image and guess whether it's a // recovery image based on the image name. @@ -278,12 +383,51 @@ impl<'a> RepositoryEditor<'a> { let mut source = FileSource::open(path.clone()).await?; let caboose = source.read_hubris_caboose().await?; let data = CabooseData::new(&caboose, tag_fn, &path)?; - Ok(self.hubris_archive_inner(source, data)) + Ok(self.hubris_archive_inner(source.into(), data)) + } + + pub fn fake_rot_archive( + self, + board: String, + sign: Sign, + slot: RotSlot, + ) -> Result { + let data = CabooseData { + name: board.clone(), + version: self.artifact_version.clone()?, + tags: KnownArtifactTags::Rot { board, sign, slot }, + }; + let source = data.generate_fake_archive()?; + Ok(self.hubris_archive_inner(source.into(), data)) + } + + pub fn fake_rot_bootloader_archive( + self, + board: String, + sign: Sign, + ) -> Result { + let data = CabooseData { + name: board.clone(), + version: self.artifact_version.clone()?, + tags: KnownArtifactTags::RotBootloader { board, sign }, + }; + let source = data.generate_fake_archive()?; + Ok(self.hubris_archive_inner(source.into(), data)) + } + + pub fn fake_sp_archive(self, board: String) -> Result { + let data = CabooseData { + name: board.clone(), + version: self.artifact_version.clone()?, + tags: KnownArtifactTags::Sp { board }, + }; + let source = data.generate_fake_archive()?; + Ok(self.hubris_archive_inner(source.into(), data)) } fn hubris_archive_inner( mut self, - source: FileSource, + source: TargetSource<'a>, CabooseData { tags, name, version }: CabooseData, ) -> Self { let target_name = match &tags { @@ -302,10 +446,7 @@ impl<'a> RepositoryEditor<'a> { // never be used in an actual rack. The current thinking is // that they will eventually no longer need to be in the TUF // repo. Add these as an extra target, not an artifact. - self.targets - .entry(target_name) - .or_default() - .push(TargetSource::File(source)); + self.targets.entry(target_name).or_default().push(source); return self; } target_name @@ -318,12 +459,11 @@ impl<'a> RepositoryEditor<'a> { async fn guess_hubris_archive( file_start: &[u8], - path: &Utf8Path, - ) -> Option<(FileSource, CabooseData)> { + source: &mut FileSource, + ) -> Option { if !file_start.starts_with(b"PK\x03\x04") { return None; } - let mut source = FileSource::open(path.to_owned()).await.ok()?; let archive = source.read_hubris_archive().await.ok()?; let caboose = archive.read_caboose().ok()?; // HACK: We are reading the `image-name` file in the archive, which @@ -331,34 +471,37 @@ impl<'a> RepositoryEditor<'a> { // an SP image, and nonexistent if it's an ROT bootloader image. This // seems fragile. Ideally this can be in the caboose someday (see // sprot-release#74). - let data = match archive.image_name().as_deref() { + match archive.image_name().as_deref() { Ok("a") => CabooseData::new( &caboose, |caboose| { KnownArtifactTags::from_rot_caboose(caboose, RotSlot::A) }, - path, - ), + &source.path, + ) + .ok(), Ok("b") => CabooseData::new( &caboose, |caboose| { KnownArtifactTags::from_rot_caboose(caboose, RotSlot::B) }, - path, - ), + &source.path, + ) + .ok(), Ok("default") => CabooseData::new( &caboose, KnownArtifactTags::from_sp_caboose, - path, - ), + &source.path, + ) + .ok(), Err(hubtools::Error::MissingFile(_, _)) => CabooseData::new( &caboose, KnownArtifactTags::from_rot_bootloader_caboose, - path, - ), - _ => return None, - }; - Some((source, data.ok()?)) + &source.path, + ) + .ok(), + _ => None, + } } pub async fn zone_image(self, path: Utf8PathBuf) -> Result { @@ -376,12 +519,20 @@ impl<'a> RepositoryEditor<'a> { }) .await??; let source = FileSource::from_file(file.into(), cloned_path); - Ok(self.zone_image_inner(source, layer_info)) + Ok(self.zone_image_inner(source.into(), layer_info)) + } + + pub fn fake_zone_image(self, name: String) -> Result { + let version = self.artifact_version.clone()?; + let source = + FakeSource::new(format!("zone {name} version {version}\n"), MIB); + Ok(self + .zone_image_inner(source.into(), LayerInfo { pkg: name, version })) } fn zone_image_inner( mut self, - source: FileSource, + source: TargetSource<'a>, LayerInfo { pkg, version }: LayerInfo, ) -> Self { let target_name = format!("zones/{pkg}.tar.gz"); @@ -419,21 +570,19 @@ impl<'a> RepositoryEditor<'a> { return Err(ErrorKind::GuessArtifact { path }.into()); } let buf = &buf[..n]; - let source = FileSource::from_file(file, path.clone()); + let mut source = FileSource::from_file(file, path.clone()); let (likely_corim, corim) = Self::guess_measurement_corpus(buf); if likely_corim { - return self.measurement_corpus_inner(source, corim).await; + return self.measurement_corpus_file(source, corim).await; } - if let Some((source, data)) = - Self::guess_hubris_archive(buf, &path).await - { - return Ok(self.hubris_archive_inner(source, data)); + if let Some(data) = Self::guess_hubris_archive(buf, &mut source).await { + return Ok(self.hubris_archive_inner(source.into(), data)); } if let Some(layer_info) = Self::guess_zone_image(buf) { - return Ok(self.zone_image_inner(source, layer_info)); + return Ok(self.zone_image_inner(source.into(), layer_info)); } Err(ErrorKind::GuessArtifact { path }.into()) @@ -444,12 +593,9 @@ impl<'a> RepositoryEditor<'a> { target_name: String, version: ArtifactVersion, tags: KnownArtifactTags, - source: FileSource, + source: TargetSource<'a>, ) { - self.targets - .entry(target_name.clone()) - .or_default() - .push(TargetSource::File(source)); + self.targets.entry(target_name.clone()).or_default().push(source); self.artifacts.entry(target_name.clone()).or_default().insert( ArtifactSchema { target_name, version, tags: tags.to_tags() }, ); @@ -469,7 +615,7 @@ impl<'a> RepositoryEditor<'a> { self.targets .entry(target_name.clone()) .or_default() - .push(TargetSource::File(FileSource::open(path).await?)); + .push(FileSource::open(path).await?.into()); Ok(self) } @@ -479,6 +625,8 @@ impl<'a> RepositoryEditor<'a> { self } + /// Manually adds a fake artifact. Don't use this if building a real + /// repository; use one of the other `fake_*` methods instead. pub fn fake_artifact( mut self, target_name: String, @@ -498,67 +646,28 @@ impl<'a> RepositoryEditor<'a> { } pub fn fake(system_version: Version) -> Result { - let version = ArtifactVersion::new(system_version.to_string())?; let mut editor = Self::new(system_version); - for hash in ["123abc", "def456"] { - editor = editor.fake_artifact( - format!("measurements/corim-fake-{version}-{hash}.cbor"), - version.clone(), - KnownArtifactTags::MeasurementCorpus {}, - 4 * KIB, - ); + for hashes in [4, 16] { + editor = editor.fake_measurement_corpus(hashes)?; } for variant in [OsVariant::Host, OsVariant::Recovery] { - for board in [OsBoard::Gimlet, OsBoard::Cosmo] { - editor = editor.fake_artifact( - format!("os-{variant}/{board}.rom"), - version.clone(), - KnownArtifactTags::OsPhase1 { variant, board }, - MIB, - ); - } - editor = editor.fake_artifact( - format!("os-{variant}/zfs.img"), - version.clone(), - KnownArtifactTags::OsPhase2 { variant }, - 4 * MIB, - ); + editor = editor.fake_os_image(variant)?; } for slot in [RotSlot::A, RotSlot::B] { - editor = editor.fake_artifact( - format!("rot/fake-unsigned-{version}-slot-{slot}.zip"), - version.clone(), - KnownArtifactTags::Rot { - board: "fake".into(), - sign: Sign::UNSIGNED, - slot, - }, - 256 * KIB, - ); + editor = editor.fake_rot_archive( + "fake-rot".into(), + Sign::UNSIGNED, + slot, + )?; + } + editor = editor + .fake_rot_bootloader_archive("fake-rot".into(), Sign::UNSIGNED)?; + for board in ["fake-gimlet", "fake-cosmo", "fake-sidecar", "fake-psc"] { + editor = editor.fake_sp_archive(board.into())?; } - editor = editor.fake_artifact( - format!("rot-bootloader/fake-unsigned-{version}.zip"), - version.clone(), - KnownArtifactTags::RotBootloader { - board: "fake".into(), - sign: Sign::UNSIGNED, - }, - 64 * KIB, - ); - editor = editor.fake_artifact( - format!("sp/fake-{version}.zip"), - version.clone(), - KnownArtifactTags::Sp { board: "fake".into() }, - MIB, - ); for name in ["zone1", "zone2"] { - editor = editor.fake_artifact( - format!("zones/{name}.tar.gz"), - version.clone(), - KnownArtifactTags::Zone { name: name.into() }, - MIB, - ); + editor = editor.fake_zone_image(name.into())?; } Ok(editor) @@ -592,12 +701,13 @@ impl<'a> RepositoryEditor<'a> { continue; } self.targets.entry(target_name.raw().to_owned()).or_default().push( - TargetSource::Repository(RepositorySource { + RepositorySource { repo, target_name: target_name.raw().to_owned(), length: target.length, sha256: target.hashes.sha256.to_vec(), - }), + } + .into(), ); } for artifact in repo.artifacts() { @@ -671,8 +781,7 @@ impl<'a> RepositoryEditor<'a> { } if self.generate_installinator_document { - let version = - ArtifactVersion::new(self.system_version.to_string())?; + let version = self.artifact_version.clone()?; let target_name = format!("installinator_document-{version}.json"); let artifact = ArtifactSchema { target_name: target_name.clone(), @@ -725,6 +834,60 @@ impl<'a> RepositoryEditor<'a> { } } +#[derive(Debug)] +struct OsImageSources<'a> { + variant: OsVariant, + cosmo_phase_1: TargetSource<'a>, + gimlet_phase_1: TargetSource<'a>, + phase_2: TargetSource<'a>, + extra_targets: HashMap>, +} + +fn read_os_tarball_metadata_blocking( + file: std::fs::File, + tarball_path: Utf8PathBuf, +) -> Result>, Error> { + let mut metadata = HashMap::new(); + let mut archive = tar::Archive::new(GzDecoder::new(file)); + for entry in try_path!(archive.entries(), ReadFile, tarball_path) { + let mut entry = try_path!(entry, ReadFile, tarball_path); + if entry.header().entry_type() != tar::EntryType::Regular { + continue; + } + let path = try_path!(entry.path(), ReadFile, tarball_path); + let Some(parent) = path.parent() else { continue }; + if parent != "image" { + continue; + }; + let Some(extension) = path.extension() else { continue }; + if extension != "txt" { + continue; + } + let file_name = Path::new( + path.file_name() + .expect("a path with an extension must have a file name"), + ); + if file_name == "zfs.img" { + break; + } + let file_name = try_path!( + Utf8PathBuf::try_from(file_name.to_owned()) + .map_err(|error| error.into_io_error()), + ReadFile, + tarball_path + ); + let mut writer = BytesMut::new().writer(); + try_path!( + std::io::copy(&mut entry, &mut writer), + ReadFile, + tarball_path + ); + let source = BytesSource(writer.into_inner().freeze()); + metadata.insert(file_name.to_owned(), source.into()); + } + Ok(metadata) +} + #[derive(Debug)] struct CabooseData { tags: KnownArtifactTags, @@ -758,6 +921,36 @@ impl CabooseData { version: ArtifactVersion::new(version)?, }) } + + fn generate_fake_archive(&self) -> Result { + let (kind, board, sign) = match &self.tags { + KnownArtifactTags::Rot { board, sign, .. } => { + ("rot", board, sign.0.as_ref()) + } + KnownArtifactTags::RotBootloader { board, sign } => { + ("rot-bootloader", board, sign.0.as_ref()) + } + KnownArtifactTags::Sp { board } => ("sp", board, None), + _ => panic!("generate_fake_archive called with non-hubris tags"), + }; + + let mut builder = CabooseBuilder::default() + .git_commit(format!("this-is-a-fake-{kind}")) + .board(board) + .name(board) + .version(self.version.to_string()); + if let Some(sign) = sign { + builder = builder.sign(sign); + } + let caboose = builder.build(); + + let mut builder = HubrisArchiveBuilder::with_fake_image(); + builder.write_caboose(caboose.as_slice()).unwrap(); + let vec = builder + .build_to_vec() + .map_err(ErrorKind::GenerateFakeHubrisArchive)?; + Ok(BytesSource(vec.into())) + } } #[derive(Debug, Default)] diff --git a/lib/src/edit/sign.rs b/lib/src/edit/sign.rs index c3a1e88..e47cdff 100644 --- a/lib/src/edit/sign.rs +++ b/lib/src/edit/sign.rs @@ -181,9 +181,8 @@ impl<'a> UnsignedRepository<'a> { while let Some(entry) = read_dir.try_next().await? { // This is opening a file within the tempdir that is about to be // deleted. This is expected to be fine. - let source = TargetSource::File( - FileSource::open(entry.path().to_owned()).await?, - ); + let source = + FileSource::open(entry.path().to_owned()).await?.into(); sources.insert( (FilePrefix::Metadata, entry.file_name().into()), source, diff --git a/lib/src/edit/source.rs b/lib/src/edit/source.rs index 8132407..3bf6db5 100644 --- a/lib/src/edit/source.rs +++ b/lib/src/edit/source.rs @@ -67,6 +67,30 @@ impl TargetSource<'_> { } } +impl From for TargetSource<'_> { + fn from(source: BytesSource) -> Self { + TargetSource::Bytes(source) + } +} + +impl From for TargetSource<'_> { + fn from(source: FileSource) -> Self { + TargetSource::File(source) + } +} + +impl<'a> From> for TargetSource<'a> { + fn from(source: RepositorySource<'a>) -> Self { + TargetSource::Repository(source) + } +} + +impl From for TargetSource<'_> { + fn from(source: FakeSource) -> Self { + TargetSource::Fake(source) + } +} + #[derive(Debug, Clone)] pub(crate) struct BytesSource(pub(crate) Bytes); @@ -83,7 +107,7 @@ impl BytesSource { Target { length: self.0.len().try_into().unwrap(), sha256: Sha256::digest(&self.0).to_vec(), - source: TargetSource::Bytes(self), + source: self.into(), } } } @@ -112,11 +136,7 @@ impl FileSource { Some(inner) => inner, None => self.read_impl(None).await?, }; - Ok(Target { - length, - sha256: sha256.0.to_vec(), - source: TargetSource::File(self), - }) + Ok(Target { length, sha256: sha256.0.to_vec(), source: self.into() }) } async fn read_impl( @@ -206,7 +226,7 @@ impl<'a> RepositorySource<'a> { Target { length: self.length, sha256: self.sha256.clone(), - source: TargetSource::Repository(self), + source: self.into(), } } @@ -243,7 +263,7 @@ impl FakeSource { Target { length: self.length.try_into().unwrap(), sha256: self.sha256().await.0.to_vec(), - source: TargetSource::Fake(self), + source: self.into(), } } diff --git a/lib/src/error.rs b/lib/src/error.rs index 5220b3a..b9b8d09 100644 --- a/lib/src/error.rs +++ b/lib/src/error.rs @@ -110,6 +110,8 @@ pub enum ErrorKind { source: tufaceous_artifact::hubris::ReadCabooseError, path: Utf8PathBuf, }, + #[error("failed to generate fake hubris archive")] + GenerateFakeHubrisArchive(#[source] hubtools::Error), #[error("unspecified failure when generating ed25519 key")] Ed25519Generate, diff --git a/lib/src/util.rs b/lib/src/util.rs index a955e36..14c9f42 100644 --- a/lib/src/util.rs +++ b/lib/src/util.rs @@ -73,6 +73,10 @@ impl DirEntry { Ok(Self { path }) } + pub(crate) fn into_path(self) -> Utf8PathBuf { + self.path + } + pub(crate) fn path(&self) -> &Utf8Path { &self.path } From 9153796ef35cf3c51ddb4e9b002af5fdb482caaf Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 15 Jan 2026 14:49:31 -0800 Subject: [PATCH 014/137] add Repository::fake shorthand method --- Cargo.lock | 23 +++++++++++++++++++++++ Cargo.toml | 1 + lib/Cargo.toml | 1 + lib/src/repo.rs | 43 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index be950a4..9e60eb5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2735,6 +2735,28 @@ dependencies = [ "syn 2.0.96", ] +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-test" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6d24790a10a7af737693a3e8f1d03faef7e6ca0cc99aae5066f533766de545" +dependencies = [ + "futures-core", + "tokio", + "tokio-stream", +] + [[package]] name = "tokio-util" version = "0.7.15" @@ -2841,6 +2863,7 @@ dependencies = [ "tar", "thiserror 2.0.12", "tokio", + "tokio-test", "tokio-util", "tough", "tufaceous-artifact", diff --git a/Cargo.toml b/Cargo.toml index 778b2ef..a015585 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,6 +49,7 @@ tar = "0.4.44" test-strategy = "0.4.0" thiserror = "2.0.12" tokio = "1.47.0" +tokio-test = "0.4.5" tokio-util = "0.7.15" tough = "0.21.0" tufaceous.path = "lib" diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 0132413..b275aba 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -35,6 +35,7 @@ url.workspace = true [dev-dependencies] tokio = { workspace = true, features = ["macros"] } +tokio-test.workspace = true [lints] workspace = true diff --git a/lib/src/repo.rs b/lib/src/repo.rs index 830a16a..96d0421 100644 --- a/lib/src/repo.rs +++ b/lib/src/repo.rs @@ -48,6 +48,49 @@ impl Repository { RepositoryLoader::new() } + /// Generate and load a fake repository. + /// + /// This is shorthand for: + /// + /// ```rust + /// # tokio_test::block_on(async { + /// # let system_version = const { semver::Version::new(1, 0, 0) }; + /// # let log = slog::Logger::root(slog::Discard, slog::o!()); + /// # let log = &log; + /// let zip = tufaceous::edit::RepositoryEditor::fake(system_version)? + /// .finish() + /// .await? + /// .generate_root() + /// .sign() + /// .await? + /// .write_zip(Vec::new(), chrono::Utc::now()) + /// .await?; + /// # Ok::<_, tufaceous::error::Error>( + /// tufaceous::RepositoryLoader::new() + /// .trust_store_behavior(tufaceous::TrustStoreBehavior::UnsafeBlindFaith) + /// .load_zip_buffer(zip, log) + /// .await? + /// # ) + /// # }).unwrap(); + /// ``` + pub async fn fake( + system_version: Version, + log: &Logger, + ) -> Result { + let zip = crate::edit::RepositoryEditor::fake(system_version)? + .finish() + .await? + .generate_root() + .sign() + .await? + .write_zip(Vec::new(), chrono::Utc::now()) + .await?; + RepositoryLoader::new() + .trust_store_behavior(crate::TrustStoreBehavior::UnsafeBlindFaith) + .load_zip_buffer(zip, log) + .await + } + pub(crate) async fn from_loaded( repo: tough::Repository, log: &Logger, From a05e82a5c40acb139a6ac392496531989c159e31 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 26 Jan 2026 10:14:35 -0800 Subject: [PATCH 015/137] add `Artifacts::filter_tags` --- artifact/src/artifact.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/artifact/src/artifact.rs b/artifact/src/artifact.rs index 65a357f..8fbe27d 100644 --- a/artifact/src/artifact.rs +++ b/artifact/src/artifact.rs @@ -73,6 +73,18 @@ impl Artifacts { self.inner.get(&Some(tags)).map(BTreeSet::iter).unwrap_or_default() } + pub fn filter_tags( + &self, + mut predicate: impl FnMut(&KnownArtifactTags) -> bool, + ) -> impl Iterator { + self.inner + .iter() + .filter_map(move |(tags, artifacts)| { + predicate(tags.as_ref()?).then_some(artifacts) + }) + .flatten() + } + pub fn iter(&self) -> Iter<'_> { Iter { inner: self.inner.values().flatten() } } From 49fd3fffa4a3448e047aab88c0ad46bdb4a8ce7a Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 26 Jan 2026 13:27:53 -0800 Subject: [PATCH 016/137] clean up unwraps from fake artifact generation --- lib/src/edit/editor.rs | 11 ++++++++--- lib/src/error.rs | 6 ++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index cc09271..cbd714b 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -137,10 +137,13 @@ impl<'a> RepositoryEditor<'a> { for i in 0..hashes { builder.add_hash(format!("layer{i}"), 10, vec![0; 32]); } - let corim = builder.build().unwrap(); + let corim = builder + .build() + .map_err(ErrorKind::GenerateFakeMeasurementCorpus)?; let mut writer = BytesMut::new().writer(); - ciborium::into_writer(&corim, &mut writer).unwrap(); + ciborium::into_writer(&corim, &mut writer) + .map_err(ErrorKind::SerializeFakeMeasurementCorpus)?; let bytes = writer.into_inner().freeze(); let sha256 = Sha256::digest(&bytes); let source = BytesSource(bytes).into(); @@ -945,7 +948,9 @@ impl CabooseData { let caboose = builder.build(); let mut builder = HubrisArchiveBuilder::with_fake_image(); - builder.write_caboose(caboose.as_slice()).unwrap(); + builder + .write_caboose(caboose.as_slice()) + .map_err(ErrorKind::GenerateFakeHubrisArchive)?; let vec = builder .build_to_vec() .map_err(ErrorKind::GenerateFakeHubrisArchive)?; diff --git a/lib/src/error.rs b/lib/src/error.rs index b9b8d09..35c6b0b 100644 --- a/lib/src/error.rs +++ b/lib/src/error.rs @@ -112,6 +112,12 @@ pub enum ErrorKind { }, #[error("failed to generate fake hubris archive")] GenerateFakeHubrisArchive(#[source] hubtools::Error), + #[error("failed to generate fake measurement corpus")] + GenerateFakeMeasurementCorpus(#[source] rats_corim::Error), + #[error("failed to serialize fake measurement corpus")] + SerializeFakeMeasurementCorpus( + #[source] ciborium::ser::Error, + ), #[error("unspecified failure when generating ed25519 key")] Ed25519Generate, From 3fed6a326e337797b410c3548afeeec6009db04b Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 27 Jan 2026 16:15:22 -0800 Subject: [PATCH 017/137] big-ass editor refactor the primary purpose of this change is `ArtifactsExt::fake`, which is guaranteed to return the same set of `Artifacts` that you would get from generating a repository, but without requiring async. this also replaced the struct variants of `KnownArtifactTags` with named structs (RFD 643). --- artifact/src/artifact.rs | 21 + artifact/src/hubris.rs | 45 +- artifact/src/installinator.rs | 25 +- artifact/src/tags.rs | 177 +++++-- brand-metadata/src/lib.rs | 10 + lib/src/edit/editor.rs | 784 +++++------------------------ lib/src/edit/fake.rs | 118 +++++ lib/src/edit/guess.rs | 69 +++ lib/src/edit/hubris_archive.rs | 201 ++++++++ lib/src/edit/input.rs | 213 ++++++++ lib/src/edit/measurement_corpus.rs | 104 ++++ lib/src/edit/mod.rs | 11 + lib/src/edit/os_images.rs | 218 ++++++++ lib/src/edit/source.rs | 149 +++--- lib/src/edit/zone_image.rs | 98 ++++ lib/src/error.rs | 2 + lib/src/lib.rs | 4 + lib/src/repo/v1.rs | 34 +- lib/src/schema.rs | 6 + 19 files changed, 1483 insertions(+), 806 deletions(-) create mode 100644 lib/src/edit/fake.rs create mode 100644 lib/src/edit/guess.rs create mode 100644 lib/src/edit/hubris_archive.rs create mode 100644 lib/src/edit/input.rs create mode 100644 lib/src/edit/measurement_corpus.rs create mode 100644 lib/src/edit/os_images.rs create mode 100644 lib/src/edit/zone_image.rs diff --git a/artifact/src/artifact.rs b/artifact/src/artifact.rs index 8fbe27d..9dab798 100644 --- a/artifact/src/artifact.rs +++ b/artifact/src/artifact.rs @@ -6,6 +6,7 @@ use std::collections::BTreeMap; use std::collections::BTreeSet; use std::collections::btree_map::IntoValues; use std::collections::btree_map::Values; +use std::fmt::Display; use std::iter::Flatten; use serde::Deserialize; @@ -30,6 +31,26 @@ impl Artifact { pub fn known_tags(&self) -> Option { KnownArtifactTags::from_tags(&self.tags).ok() } + + pub fn display_tags(&self) -> DisplayTags<'_> { + DisplayTags(&self.tags) + } +} + +#[derive(Debug, Clone)] +pub struct DisplayTags<'a>(&'a BTreeMap); + +impl Display for DisplayTags<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut comma = ""; + let kind = self.0.get_key_value("kind").into_iter(); + let remainder = self.0.iter().filter(|(k, _)| *k != "kind"); + for (key, value) in kind.chain(remainder) { + write!(f, "{comma}{key}={value}")?; + comma = ","; + } + Ok(()) + } } #[derive(Debug, Clone, Default, PartialEq, Eq)] diff --git a/artifact/src/hubris.rs b/artifact/src/hubris.rs index 8c98d01..7d45da3 100644 --- a/artifact/src/hubris.rs +++ b/artifact/src/hubris.rs @@ -8,34 +8,59 @@ use hubtools::Caboose; use hubtools::CabooseError; use crate::KnownArtifactTags; +use crate::RotBootloaderTags; use crate::RotSlot; +use crate::RotTags; use crate::Sign; +use crate::SpTags; + +impl RotTags { + pub fn from_caboose( + caboose: &Caboose, + slot: RotSlot, + ) -> Result { + Ok(Self { + rot_board: read_board(caboose)?, + rot_sign: Sign(read_sign(caboose)?), + rot_slot: slot, + }) + } +} + +impl RotBootloaderTags { + pub fn from_caboose(caboose: &Caboose) -> Result { + Ok(Self { + rot_board: read_board(caboose)?, + rot_sign: Sign(read_sign(caboose)?), + }) + } +} + +impl SpTags { + pub fn from_caboose(caboose: &Caboose) -> Result { + Ok(Self { sp_board: read_board(caboose)? }) + } +} impl KnownArtifactTags { pub fn from_rot_caboose( caboose: &Caboose, slot: RotSlot, ) -> Result { - Ok(KnownArtifactTags::Rot { - board: read_board(caboose)?, - sign: Sign(read_sign(caboose)?), - slot, - }) + RotTags::from_caboose(caboose, slot).map(KnownArtifactTags::Rot) } pub fn from_rot_bootloader_caboose( caboose: &Caboose, ) -> Result { - Ok(KnownArtifactTags::RotBootloader { - board: read_board(caboose)?, - sign: Sign(read_sign(caboose)?), - }) + RotBootloaderTags::from_caboose(caboose) + .map(KnownArtifactTags::RotBootloader) } pub fn from_sp_caboose( caboose: &Caboose, ) -> Result { - Ok(KnownArtifactTags::Sp { board: read_board(caboose)? }) + SpTags::from_caboose(caboose).map(KnownArtifactTags::Sp) } } diff --git a/artifact/src/installinator.rs b/artifact/src/installinator.rs index f0ab5b4..c201cab 100644 --- a/artifact/src/installinator.rs +++ b/artifact/src/installinator.rs @@ -3,6 +3,7 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use std::borrow::Cow; +use std::collections::BTreeSet; use serde::Deserialize; use serde::Serialize; @@ -18,20 +19,22 @@ use crate::ArtifactHash; /// There are no backwards compatibility constraints for this document. The /// version of Installinator that processes this document is the same as the /// version of tufaceous that creates it. -#[derive(Debug, Clone, Deserialize, Serialize)] +#[derive(Debug, Clone, Default, Deserialize, Serialize)] pub struct InstallinatorDocument { - pub artifacts: Vec, + pub artifacts: BTreeSet, } /// Describes an artifact available to Installinator. -#[derive(Debug, Clone, Deserialize, Serialize)] +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize, +)] pub struct InstallinatorArtifact { - /// A file name without directory separators; not necessarily the target - /// name. - pub file_name: String, #[serde(flatten)] pub kind: InstallinatorArtifactKind, pub sha256: ArtifactHash, + /// A file name without directory separators; not necessarily the target + /// name. + pub file_name: String, } impl InstallinatorArtifact { @@ -44,13 +47,15 @@ impl InstallinatorArtifact { } /// The artifact kind for an Installinator artifact. -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(tag = "kind", rename_all = "kebab-case")] +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize, +)] +#[serde(tag = "kind", rename_all = "snake_case")] pub enum InstallinatorArtifactKind { - /// A measurement corpus. - MeasurementCorpus, /// The host phase 2 artifact. HostPhase2, + /// A measurement corpus. + MeasurementCorpus, /// A control plane zone artifact. Zone { zone_name: String }, } diff --git a/artifact/src/tags.rs b/artifact/src/tags.rs index 937a2cd..bfe8949 100644 --- a/artifact/src/tags.rs +++ b/artifact/src/tags.rs @@ -16,65 +16,33 @@ use crate::Sign; Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, )] #[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] -#[serde(tag = "kind", rename_all = "kebab-case")] +#[serde(tag = "kind", rename_all = "snake_case")] pub enum KnownArtifactTags { /// JSON document describing the artifacts Installinator is responsible for /// writing during mupdate and sled recovery. - InstallinatorDocument {}, + InstallinatorDocument, /// CORIM manifest for remote attestation. - MeasurementCorpus {}, + MeasurementCorpus, /// Phase 1 OS image, written to flash. Differs based on the target board. - OsPhase1 { variant: OsVariant, board: OsBoard }, + OsPhase1(OsPhase1Tags), /// Phase 2 OS image, a ZFS pool with an Oxide-specific header written to /// M.2 storage. Common across all target boards. - OsPhase2 { variant: OsVariant }, + OsPhase2(OsPhase2Tags), /// Hubris archive for a Root of Trust image. - Rot { - /// The `BORD` field in the caboose (such as `oxide-rot-1`). - board: String, - /// The `SIGN` field in the caboose. This is the Root Key Table Hash - /// (RKTH). - /// - /// For unsigned images this will not be present; this will generally - /// never occur in release repos but can be useful on hardware that has - /// not fully made it through manufacturing yet. - #[serde(skip_serializing_if = "Sign::is_unsigned")] - sign: Sign, - /// ROT images are compiled for two different locations in flash; this - /// identifies which slot this image belongs to. - slot: RotSlot, - }, + Rot(RotTags), /// Hubris archive for a Root of Trust bootloader. - RotBootloader { - /// The `BORD` field in the caboose (such as `oxide-rot-1`). - board: String, - /// The `SIGN` field in the caboose. This is the Root Key Table Hash - /// (RKTH). - /// - /// For unsigned images this will not be present; this will generally - /// never occur in release repos but can be useful on hardware that has - /// not fully made it through manufacturing yet. - #[serde(skip_serializing_if = "Sign::is_unsigned")] - sign: Sign, - }, + RotBootloader(RotBootloaderTags), /// Hubris archive for a Service Processor image. - Sp { - /// The `BORD` field in the caboose (such as `gimlet-d` or `cosmo-b`). - board: String, - }, + Sp(SpTags), /// Tarball of a Helios zone image. - Zone { - /// The zone name, as self-identified in the tarball's `oxide.json` - /// file. This may differ from the filename on disk. - name: String, - }, + Zone(ZoneTags), } impl KnownArtifactTags { @@ -103,15 +71,15 @@ impl KnownArtifactTags { pub fn to_installinator(&self) -> Option { match self { - KnownArtifactTags::MeasurementCorpus {} => { + KnownArtifactTags::MeasurementCorpus => { Some(InstallinatorArtifactKind::MeasurementCorpus) } - KnownArtifactTags::OsPhase2 { variant: OsVariant::Host } => { - Some(InstallinatorArtifactKind::HostPhase2) - } - KnownArtifactTags::Zone { name } => { + KnownArtifactTags::OsPhase2(OsPhase2Tags { + os_variant: OsVariant::Host, + }) => Some(InstallinatorArtifactKind::HostPhase2), + KnownArtifactTags::Zone(ZoneTags { zone_name }) => { Some(InstallinatorArtifactKind::Zone { - zone_name: name.clone(), + zone_name: zone_name.clone(), }) } _ => None, @@ -129,6 +97,35 @@ macro_rules! display_serialize { }; } +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, +)] +#[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] +pub struct OsPhase1Tags { + pub os_variant: OsVariant, + pub os_board: OsBoard, +} + +impl From for KnownArtifactTags { + fn from(tags: OsPhase1Tags) -> Self { + KnownArtifactTags::OsPhase1(tags) + } +} + +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, +)] +#[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] +pub struct OsPhase2Tags { + pub os_variant: OsVariant, +} + +impl From for KnownArtifactTags { + fn from(tags: OsPhase2Tags) -> Self { + KnownArtifactTags::OsPhase2(tags) + } +} + /// OS variant artifact tag (host or recovery). #[derive( Debug, @@ -143,7 +140,7 @@ macro_rules! display_serialize { Serialize, )] #[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] -#[serde(rename_all = "kebab-case")] +#[serde(rename_all = "snake_case")] pub enum OsVariant { Host, Recovery, @@ -164,13 +161,39 @@ display_serialize!(OsVariant); Serialize, )] #[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] -#[serde(rename_all = "kebab-case")] +#[serde(rename_all = "snake_case")] pub enum OsBoard { Gimlet, Cosmo, } display_serialize!(OsBoard); +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, +)] +#[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] +pub struct RotTags { + /// The `BORD` field in the caboose (such as `oxide-rot-1`). + pub rot_board: String, + /// The `SIGN` field in the caboose. This is the Root Key Table Hash + /// (RKTH). + /// + /// For unsigned images this will not be present; this will generally + /// never occur in release repos but can be useful on hardware that has + /// not fully made it through manufacturing yet. + #[serde(skip_serializing_if = "Sign::is_unsigned")] + pub rot_sign: Sign, + /// ROT images are compiled for two different locations in flash; this + /// identifies which slot this image belongs to. + pub rot_slot: RotSlot, +} + +impl From for KnownArtifactTags { + fn from(tags: RotTags) -> Self { + KnownArtifactTags::Rot(tags) + } +} + /// ROT slot artifact tag (A or B). #[derive( Debug, @@ -185,13 +208,67 @@ display_serialize!(OsBoard); Serialize, )] #[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] -#[serde(rename_all = "kebab-case")] +#[serde(rename_all = "snake_case")] pub enum RotSlot { A, B, } display_serialize!(RotSlot); +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, +)] +#[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] +pub struct RotBootloaderTags { + /// The `BORD` field in the caboose (such as `oxide-rot-1`). + pub rot_board: String, + /// The `SIGN` field in the caboose. This is the Root Key Table Hash + /// (RKTH). + /// + /// For unsigned images this will not be present; this will generally + /// never occur in release repos but can be useful on hardware that has + /// not fully made it through manufacturing yet. + #[serde(skip_serializing_if = "Sign::is_unsigned")] + pub rot_sign: Sign, +} + +impl From for KnownArtifactTags { + fn from(tags: RotBootloaderTags) -> Self { + KnownArtifactTags::RotBootloader(tags) + } +} + +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, +)] +#[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] +pub struct SpTags { + /// The `BORD` field in the caboose (such as `oxide-rot-1`). + pub sp_board: String, +} + +impl From for KnownArtifactTags { + fn from(tags: SpTags) -> Self { + KnownArtifactTags::Sp(tags) + } +} + +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, +)] +#[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] +pub struct ZoneTags { + /// The zone name, as self-identified in the tarball's `oxide.json` + /// file. This may differ from the filename on disk. + pub zone_name: String, +} + +impl From for KnownArtifactTags { + fn from(tags: ZoneTags) -> Self { + KnownArtifactTags::Zone(tags) + } +} + #[cfg(test)] mod tests { use test_strategy::proptest; diff --git a/brand-metadata/src/lib.rs b/brand-metadata/src/lib.rs index a5ed8f9..c0e283c 100644 --- a/brand-metadata/src/lib.rs +++ b/brand-metadata/src/lib.rs @@ -112,6 +112,16 @@ impl Metadata { } } + pub fn into_layer_info(self) -> Result { + match self.t { + ArchiveType::Layer(info) => Ok(info), + _ => Err(Error::new( + ErrorKind::InvalidData, + "archive is not the \"layer\" type", + )), + } + } + pub fn is_baseline(&self) -> bool { matches!(&self.t, ArchiveType::Baseline) } diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index cbd714b..e83e9a0 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -5,61 +5,38 @@ use std::collections::BTreeMap; use std::collections::HashMap; use std::collections::HashSet; -use std::path::Path; -use bytes::BufMut; -use bytes::BytesMut; use camino::Utf8Path; use camino::Utf8PathBuf; -use flate2::read::GzDecoder; use futures_util::FutureExt; use futures_util::TryFutureExt; -use futures_util::TryStreamExt; -use hubtools::Caboose; -use hubtools::CabooseBuilder; -use hubtools::HubrisArchiveBuilder; -use rats_corim::Corim; -use rats_corim::CorimBuilder; use semver::Version; -use sha2::Digest; -use sha2::Sha256; -use tokio::fs::File; -use tokio::io::AsyncReadExt; use tokio::task::JoinSet; use tufaceous_artifact::ArtifactVersion; use tufaceous_artifact::ArtifactVersionError; use tufaceous_artifact::InstallinatorArtifact; use tufaceous_artifact::InstallinatorDocument; use tufaceous_artifact::KnownArtifactTags; -use tufaceous_artifact::OsBoard; use tufaceous_artifact::OsVariant; +use tufaceous_artifact::RotBootloaderTags; use tufaceous_artifact::RotSlot; -use tufaceous_artifact::Sign; -use tufaceous_artifact::hubris::ReadCabooseError; -use tufaceous_brand_metadata::LayerInfo; -use tufaceous_brand_metadata::Metadata; +use tufaceous_artifact::RotTags; +use tufaceous_artifact::SpTags; use crate::Repository; use crate::edit::UnsignedRepository; +use crate::edit::input::Input; +use crate::edit::input::Output; use crate::edit::source::BytesSource; -use crate::edit::source::FakeSource; use crate::edit::source::FileSource; use crate::edit::source::RepositorySource; use crate::edit::source::Target; use crate::edit::source::TargetSource; use crate::error::Error; use crate::error::ErrorKind; -use crate::error::try_path; use crate::schema::ArtifactSchema; use crate::schema::ArtifactsSchema; -const KIB: usize = 1024; -const MIB: usize = 1024 * KIB; - -const COSMO_PHASE_1_PATH: &str = "cosmo.rom"; -const GIMLET_PHASE_1_PATH: &str = "gimlet.rom"; -const PHASE_2_PATH: &str = "zfs.img"; - #[derive(Debug)] pub struct RepositoryEditor<'a> { system_version: Version, @@ -106,83 +83,21 @@ impl<'a> RepositoryEditor<'a> { path: Utf8PathBuf, ) -> Result { let source = FileSource::open(path).await?; - self.measurement_corpus_file(source, None).await - } - - async fn measurement_corpus_file( - self, - mut source: FileSource, - corim: Option, - ) -> Result { - let Corim { id, .. } = match corim { - Some(corim) => corim, - None => { - let v = source.read_to_end().await?; - try_path!( - ciborium::from_reader(v.as_slice()), - Corim, - &source.path - ) - } - }; - let sha256 = source.sha256().await?; - self.measurement_corpus_inner(source.into(), &id, &sha256.0) + let input = Input::measurement_corpus( + source, + None, + self.artifact_version.clone()?, + ) + .await?; + Ok(self.insert_input(input)) } pub fn fake_measurement_corpus(self, hashes: usize) -> Result { - let mut builder = CorimBuilder::new(); - builder.vendor("fake-vendor".to_string()); - builder.id("fake-measurement-id".to_string()); - builder.tag_id("fake-tag-id".to_string()); - for i in 0..hashes { - builder.add_hash(format!("layer{i}"), 10, vec![0; 32]); - } - let corim = builder - .build() - .map_err(ErrorKind::GenerateFakeMeasurementCorpus)?; - - let mut writer = BytesMut::new().writer(); - ciborium::into_writer(&corim, &mut writer) - .map_err(ErrorKind::SerializeFakeMeasurementCorpus)?; - let bytes = writer.into_inner().freeze(); - let sha256 = Sha256::digest(&bytes); - let source = BytesSource(bytes).into(); - - self.measurement_corpus_inner(source, &corim.id, &sha256) - } - - fn measurement_corpus_inner( - mut self, - source: TargetSource<'a>, - corim_id: &str, - sha256: &[u8], - ) -> Result { - let target_name = - format!("measurements/{corim_id}-{}.cbor", hex::encode(sha256)); - let version = self.artifact_version.clone()?; - let tags = KnownArtifactTags::MeasurementCorpus {}; - self.insert_artifact(target_name, version, tags, source); - Ok(self) - } - - fn guess_measurement_corpus( - mut file_start: &[u8], - ) -> (bool, Option) { - if !matches!(file_start[0], 0xa0..=0xbf /* CBOR map */) { - return (false, None); - } - match ciborium::from_reader::(&mut file_start) { - Ok(corim) => (true, Some(corim)), - Err(ciborium::de::Error::Io(err)) - if err.kind() == std::io::ErrorKind::UnexpectedEof => - { - // This was plausibly a CoRIM manifest until we hit the end of - // the buffer, indicating a very high likelihood that if we read - // the entire thing it'd still be a CoRIM manifest. - (true, None) - } - Err(_) => (false, None), - } + let input = Input::fake_measurement_corpus( + hashes, + self.artifact_version.clone()?, + )?; + Ok(self.insert_input(input)) } /// Add an OS image to the repository. @@ -195,158 +110,20 @@ impl<'a> RepositoryEditor<'a> { variant: OsVariant, output_dir: &Utf8Path, ) -> Result { - let cosmo_phase_1 = - FileSource::open(output_dir.join(COSMO_PHASE_1_PATH)).await?.into(); - let gimlet_phase_1 = - FileSource::open(output_dir.join(GIMLET_PHASE_1_PATH)) - .await? - .into(); - let phase_2 = - FileSource::open(output_dir.join(PHASE_2_PATH)).await?.into(); - - // If `os.tar.gz` is present, read the `image/*.txt` files from it - // as extra sources. Stop once we get to `zfs.img`, which comes after - // the additional metadata. (If a GzDecoder was seekable we could - // conceivably skip past any files we don't want to read, but alas.) - // https://github.com/oxidecomputer/helios/blob/f145457f6ccb13a139b8d93408b9b4de5db57bd6/tools/helios-build/src/main.rs#L1881-L1885 - let tarball_path = output_dir.join("os.tar.gz"); - let mut extra_targets = match tokio::fs::File::open(&tarball_path).await - { - Ok(file) => { - let file = file.into_std().await; - tokio::task::spawn_blocking(move || { - read_os_tarball_metadata_blocking(file, tarball_path) - }) - .await?? - } - Err(source) if source.kind() == std::io::ErrorKind::NotFound => { - // We might be reading from an unpacked Tufaceous archive; look - // for any `*.txt` files and include them. - let mut extra_targets = HashMap::new(); - let mut read_dir = - crate::util::read_dir(output_dir.into()).await?; - while let Some(entry) = read_dir.try_next().await? { - if entry.path().extension() != Some("txt") { - continue; - } - let file_name = Utf8Path::new(entry.file_name()).to_owned(); - let source = FileSource::open(entry.into_path()).await?; - extra_targets.insert(file_name, source.into()); - } - extra_targets - } - Err(source) => { - return Err(ErrorKind::OpenFile { - source, - path: Some(tarball_path), - } - .into()); - } - }; - for path in ["unix.z", "cpio.z"] { - let source = FileSource::open(output_dir.join(path)).await?; - extra_targets.insert(path.into(), source.into()); - } - - self.os_image_inner(OsImageSources { + let input = Input::os_images( variant, - cosmo_phase_1, - gimlet_phase_1, - phase_2, - extra_targets, - }) - } - - pub fn fake_os_image(self, variant: OsVariant) -> Result { - let version = &self.system_version; - let cosmo_phase_1 = FakeSource::new( - format!("cosmo host phase 1 image version {version}\n"), - MIB, - ) - .into(); - let gimlet_phase_1 = FakeSource::new( - format!("gimlet host phase 1 image version {version}\n"), - MIB, + output_dir, + None, + self.artifact_version.clone()?, ) - .into(); - let phase_2 = FakeSource::new( - format!("host phase 2 image version {version}\n"), - 4 * MIB, - ) - .into(); - - let mut extra_targets: HashMap = HashMap::new(); - extra_targets.insert( - "unix.z".into(), - FakeSource::new(format!("unix.z version {version}\n"), 64 * KIB) - .into(), - ); - extra_targets.insert( - "cpio.z".into(), - FakeSource::new(format!("cpio.z version {version}\n"), 256 * KIB) - .into(), - ); - - self.os_image_inner(OsImageSources { - variant, - cosmo_phase_1, - gimlet_phase_1, - phase_2, - extra_targets, - }) - } - - fn os_image_inner( - mut self, - sources: OsImageSources<'a>, - ) -> Result { - let variant = sources.variant; - let base = Utf8PathBuf::from(format!("os-{variant}")); - let version = self.artifact_version.clone()?; - - self.insert_artifact( - base.join("image").join(COSMO_PHASE_1_PATH).into(), - version.clone(), - KnownArtifactTags::OsPhase1 { variant, board: OsBoard::Cosmo }, - sources.cosmo_phase_1, - ); - self.insert_artifact( - base.join("image").join(GIMLET_PHASE_1_PATH).into(), - version.clone(), - KnownArtifactTags::OsPhase1 { variant, board: OsBoard::Gimlet }, - sources.gimlet_phase_1, - ); - self.insert_artifact( - base.join("image").join(PHASE_2_PATH).into(), - version, - KnownArtifactTags::OsPhase2 { variant }, - sources.phase_2, - ); - for (file_name, source) in sources.extra_targets { - self.targets - .entry(base.join(file_name).into()) - .or_default() - .push(source); - } - Ok(self) + .await?; + Ok(self.insert_input(input)) } - async fn guess_os_image(path: &Utf8Path) -> Option { - let mut file = File::open(path.join("zfs.img")).await.ok()?; - // Read the header block from the image and guess whether it's a - // recovery image based on the image name. - let mut buf = [0; 4096]; - file.read_exact(&mut buf).await.ok()?; - if !buf.starts_with(&0x1DEB0075_u32.to_le_bytes()) { - return None; - } - // see https://github.com/oxidecomputer/boot-image-tools/blob/main/src/diskimage.rs - let image_name = &buf[200..328]; - Some(if image_name.starts_with(b"recovery") { - OsVariant::Recovery - } else { - OsVariant::Host - }) + pub fn fake_os_image(self, variant: OsVariant) -> Result { + let input = + Input::fake_os_images(variant, self.artifact_version.clone()?); + Ok(self.insert_input(input)) } pub async fn rot_archive( @@ -354,202 +131,55 @@ impl<'a> RepositoryEditor<'a> { slot: RotSlot, path: Utf8PathBuf, ) -> Result { - self.hubris_archive(path, |caboose| { - KnownArtifactTags::from_rot_caboose(caboose, slot) - }) - .await + let source = FileSource::open(path).await?; + Ok(self.insert_input(Input::rot_archive(source, None, slot).await?)) } pub async fn rot_bootloader_archive( self, path: Utf8PathBuf, ) -> Result { - self.hubris_archive( - path, - KnownArtifactTags::from_rot_bootloader_caboose, - ) - .await + let source = FileSource::open(path).await?; + Ok(self + .insert_input(Input::rot_bootloader_archive(source, None).await?)) } pub async fn sp_archive(self, path: Utf8PathBuf) -> Result { - self.hubris_archive(path, KnownArtifactTags::from_sp_caboose).await + let source = FileSource::open(path).await?; + Ok(self.insert_input(Input::sp_archive(source, None).await?)) } - async fn hubris_archive( - self, - path: Utf8PathBuf, - tag_fn: F, - ) -> Result - where - F: FnOnce(&Caboose) -> Result, - { - let mut source = FileSource::open(path.clone()).await?; - let caboose = source.read_hubris_caboose().await?; - let data = CabooseData::new(&caboose, tag_fn, &path)?; - Ok(self.hubris_archive_inner(source.into(), data)) - } - - pub fn fake_rot_archive( - self, - board: String, - sign: Sign, - slot: RotSlot, - ) -> Result { - let data = CabooseData { - name: board.clone(), - version: self.artifact_version.clone()?, - tags: KnownArtifactTags::Rot { board, sign, slot }, - }; - let source = data.generate_fake_archive()?; - Ok(self.hubris_archive_inner(source.into(), data)) + pub fn fake_rot_archive(self, tags: RotTags) -> Result { + let input = + Input::fake_rot_archive(tags, self.artifact_version.clone()?)?; + Ok(self.insert_input(input)) } pub fn fake_rot_bootloader_archive( self, - board: String, - sign: Sign, + tags: RotBootloaderTags, ) -> Result { - let data = CabooseData { - name: board.clone(), - version: self.artifact_version.clone()?, - tags: KnownArtifactTags::RotBootloader { board, sign }, - }; - let source = data.generate_fake_archive()?; - Ok(self.hubris_archive_inner(source.into(), data)) - } - - pub fn fake_sp_archive(self, board: String) -> Result { - let data = CabooseData { - name: board.clone(), - version: self.artifact_version.clone()?, - tags: KnownArtifactTags::Sp { board }, - }; - let source = data.generate_fake_archive()?; - Ok(self.hubris_archive_inner(source.into(), data)) - } - - fn hubris_archive_inner( - mut self, - source: TargetSource<'a>, - CabooseData { tags, name, version }: CabooseData, - ) -> Self { - let target_name = match &tags { - KnownArtifactTags::Rot { board, sign, slot } => { - format!("rot/{board}-{sign}-{version}-slot-{slot}.zip") - } - KnownArtifactTags::RotBootloader { board, sign } => { - format!("rot-bootloader/{board}-{sign}-{version}.zip") - } - KnownArtifactTags::Sp { board } => { - let target_name = format!("sp/{name}-{version}.zip"); - if board.as_str() != name { - // This is likely a lab image. As of writing these are - // stored in the TUF repo for manufacturing but are - // explicitly ignored by the control plane, as they can - // never be used in an actual rack. The current thinking is - // that they will eventually no longer need to be in the TUF - // repo. Add these as an extra target, not an artifact. - self.targets.entry(target_name).or_default().push(source); - return self; - } - target_name - } - _ => unreachable!(), - }; - self.insert_artifact(target_name, version, tags, source); - self + let input = Input::fake_rot_bootloader_archive( + tags, + self.artifact_version.clone()?, + )?; + Ok(self.insert_input(input)) } - async fn guess_hubris_archive( - file_start: &[u8], - source: &mut FileSource, - ) -> Option { - if !file_start.starts_with(b"PK\x03\x04") { - return None; - } - let archive = source.read_hubris_archive().await.ok()?; - let caboose = archive.read_caboose().ok()?; - // HACK: We are reading the `image-name` file in the archive, which - // appears to be "a" or "b" if it's an ROT image, "default" if it's - // an SP image, and nonexistent if it's an ROT bootloader image. This - // seems fragile. Ideally this can be in the caboose someday (see - // sprot-release#74). - match archive.image_name().as_deref() { - Ok("a") => CabooseData::new( - &caboose, - |caboose| { - KnownArtifactTags::from_rot_caboose(caboose, RotSlot::A) - }, - &source.path, - ) - .ok(), - Ok("b") => CabooseData::new( - &caboose, - |caboose| { - KnownArtifactTags::from_rot_caboose(caboose, RotSlot::B) - }, - &source.path, - ) - .ok(), - Ok("default") => CabooseData::new( - &caboose, - KnownArtifactTags::from_sp_caboose, - &source.path, - ) - .ok(), - Err(hubtools::Error::MissingFile(_, _)) => CabooseData::new( - &caboose, - KnownArtifactTags::from_rot_bootloader_caboose, - &source.path, - ) - .ok(), - _ => None, - } + pub fn fake_sp_archive(self, tags: SpTags) -> Result { + let input = + Input::fake_sp_archive(tags, self.artifact_version.clone()?)?; + Ok(self.insert_input(input)) } pub async fn zone_image(self, path: Utf8PathBuf) -> Result { - let cloned_path = path.clone(); - let (file, layer_info) = tokio::task::spawn_blocking(move || { - let file = try_path!(std::fs::File::open(&path), ReadFile, path); - let mut archive = tar::Archive::new(GzDecoder::new(file)); - let layer_info = try_path!( - Metadata::read_from_tar(&mut archive) - .and_then(|metadata| metadata.layer_info().cloned()), - ReadZoneOxideJson, - path - ); - Ok::<_, Error>((archive.into_inner().into_inner(), layer_info)) - }) - .await??; - let source = FileSource::from_file(file.into(), cloned_path); - Ok(self.zone_image_inner(source.into(), layer_info)) + Ok(self.insert_input(Input::zone_image(path).await?)) } pub fn fake_zone_image(self, name: String) -> Result { - let version = self.artifact_version.clone()?; - let source = - FakeSource::new(format!("zone {name} version {version}\n"), MIB); - Ok(self - .zone_image_inner(source.into(), LayerInfo { pkg: name, version })) - } - - fn zone_image_inner( - mut self, - source: TargetSource<'a>, - LayerInfo { pkg, version }: LayerInfo, - ) -> Self { - let target_name = format!("zones/{pkg}.tar.gz"); - let tags = KnownArtifactTags::Zone { name: pkg }; - self.insert_artifact(target_name, version, tags, source); - self - } - - fn guess_zone_image(file_start: &[u8]) -> Option { - // `oxide.json` is the first file of a zone image and is relatively - // small, so it should be contained entirely within the first 4K of the - // compressed tarball. - let mut archive = tar::Archive::new(GzDecoder::new(file_start)); - Metadata::read_from_tar(&mut archive).ok()?.layer_info().ok().cloned() + let input = + Input::fake_zone_image(name, self.artifact_version.clone()?)?; + Ok(self.insert_input(input)) } /// Attempt to guess the artifact kind at `path` and add it to the @@ -561,47 +191,27 @@ impl<'a> RepositoryEditor<'a> { self, path: Utf8PathBuf, ) -> Result { - if let Some(variant) = Self::guess_os_image(&path).await { - return self.os_image_dir(variant, &path).await; - } - - let mut file = try_path!(File::open(&path).await, OpenFile, path); - let mut buf = [0; 4096]; - let n = try_path!(file.read(&mut buf).await, ReadFile, path); - if n == 0 { - // we're not going to try to guess an empty file - return Err(ErrorKind::GuessArtifact { path }.into()); - } - let buf = &buf[..n]; - let mut source = FileSource::from_file(file, path.clone()); - - let (likely_corim, corim) = Self::guess_measurement_corpus(buf); - if likely_corim { - return self.measurement_corpus_file(source, corim).await; - } - - if let Some(data) = Self::guess_hubris_archive(buf, &mut source).await { - return Ok(self.hubris_archive_inner(source.into(), data)); - } - - if let Some(layer_info) = Self::guess_zone_image(buf) { - return Ok(self.zone_image_inner(source.into(), layer_info)); - } - - Err(ErrorKind::GuessArtifact { path }.into()) + let input = Input::guess(path, self.artifact_version.clone()?).await?; + Ok(self.insert_input(input)) } - fn insert_artifact( - &mut self, - target_name: String, - version: ArtifactVersion, - tags: KnownArtifactTags, - source: TargetSource<'a>, - ) { - self.targets.entry(target_name.clone()).or_default().push(source); - self.artifacts.entry(target_name.clone()).or_default().insert( - ArtifactSchema { target_name, version, tags: tags.to_tags() }, - ); + fn insert_input(mut self, input: Input) -> Self + where + T: Into>, + { + for output in input.outputs() { + if let Some(artifact) = output.to_artifact_schema() { + self.artifacts + .entry(artifact.target_name.clone()) + .or_default() + .insert(artifact); + } + self.targets + .entry(output.target_name) + .or_default() + .push(output.source.into()); + } + self } /// Add a non-artifact target to the repository. @@ -616,7 +226,7 @@ impl<'a> RepositoryEditor<'a> { path: Utf8PathBuf, ) -> Result { self.targets - .entry(target_name.clone()) + .entry(target_name) .or_default() .push(FileSource::open(path).await?.into()); Ok(self) @@ -641,7 +251,7 @@ impl<'a> RepositoryEditor<'a> { self.targets .entry(target_name.clone()) .or_default() - .push(TargetSource::Fake(FakeSource::new(prefix, length))); + .push(BytesSource::fake_padded(prefix, length).into()); self.artifacts.entry(target_name.clone()).or_default().insert( ArtifactSchema { target_name, version, tags: tags.to_tags() }, ); @@ -650,29 +260,10 @@ impl<'a> RepositoryEditor<'a> { pub fn fake(system_version: Version) -> Result { let mut editor = Self::new(system_version); - - for hashes in [4, 16] { - editor = editor.fake_measurement_corpus(hashes)?; + let version = editor.artifact_version.clone()?; + for input in Input::fake(&version)? { + editor = editor.insert_input(input); } - for variant in [OsVariant::Host, OsVariant::Recovery] { - editor = editor.fake_os_image(variant)?; - } - for slot in [RotSlot::A, RotSlot::B] { - editor = editor.fake_rot_archive( - "fake-rot".into(), - Sign::UNSIGNED, - slot, - )?; - } - editor = editor - .fake_rot_bootloader_archive("fake-rot".into(), Sign::UNSIGNED)?; - for board in ["fake-gimlet", "fake-cosmo", "fake-sidecar", "fake-psc"] { - editor = editor.fake_sp_archive(board.into())?; - } - for name in ["zone1", "zone2"] { - editor = editor.fake_zone_image(name.into())?; - } - Ok(editor) } @@ -753,7 +344,10 @@ impl<'a> RepositoryEditor<'a> { let target_name = target_name.clone(); match source { TargetSource::Bytes(source) => { - all_targets.push((target_name, source.into_target())); + let future = source + .into_target() + .map(|target| Ok((target_name, target))); + tasks.spawn(future); } TargetSource::File(source) => { let future = source @@ -764,12 +358,6 @@ impl<'a> RepositoryEditor<'a> { TargetSource::Repository(source) => { all_targets.push((target_name, source.into_target())); } - TargetSource::Fake(source) => { - let future = source - .into_target() - .map(|target| Ok((target_name, target))); - tasks.spawn(future); - } } } } @@ -784,44 +372,29 @@ impl<'a> RepositoryEditor<'a> { } if self.generate_installinator_document { - let version = self.artifact_version.clone()?; - let target_name = format!("installinator_document-{version}.json"); - let artifact = ArtifactSchema { - target_name: target_name.clone(), - version, - tags: KnownArtifactTags::InstallinatorDocument {}.to_tags(), - }; - if let Some(existing) = artifacts.get(&target_name) { - if existing != &artifact { - return Err( - ErrorKind::TargetNameCollision { target_name }.into() - ); - } - } else { - artifacts.insert(target_name.clone(), artifact); - } - - let mut document = InstallinatorDocument { artifacts: Vec::new() }; - for artifact in artifacts.values() { - if let Some(kind) = KnownArtifactTags::from_tags(&artifact.tags) - .ok() - .and_then(|tags| tags.to_installinator()) - && let Some(file_name) = - Utf8Path::new(&artifact.target_name).file_name() - && let Some(target) = targets.0.get(&artifact.target_name) - && let Ok(sha256) = target.sha256.as_slice().try_into() - { - document.artifacts.push(InstallinatorArtifact { - file_name: file_name.to_owned(), - kind, - sha256, - }); + let output = generate_installinator_document( + artifacts.values().filter_map(|artifact| { + let target = targets.0.get(&artifact.target_name)?; + Some((artifact, target.sha256.as_slice())) + }), + self.artifact_version.clone()?, + )?; + if let Some(artifact) = output.to_artifact_schema() { + if let Some(existing) = artifacts.get(&artifact.target_name) { + if existing != &artifact { + return Err(ErrorKind::TargetNameCollision { + target_name: artifact.target_name, + } + .into()); + } + } else { + artifacts.insert(artifact.target_name.clone(), artifact); } } - let target = BytesSource::json(&document) - .map_err(ErrorKind::SerializeInstallinator)? - .into_target(); - targets.insert(target_name, target)?; + targets.insert( + output.target_name, + output.source.into_target().await, + )?; } let document = ArtifactsSchema { @@ -831,133 +404,13 @@ impl<'a> RepositoryEditor<'a> { }; let target = BytesSource::json(&document) .map_err(ErrorKind::SerializeArtifacts)? - .into_target(); + .into_target() + .await; targets.insert(ArtifactsSchema::TARGET_NAME.to_owned(), target)?; Ok(UnsignedRepository::from_targets(targets.0)) } } -#[derive(Debug)] -struct OsImageSources<'a> { - variant: OsVariant, - cosmo_phase_1: TargetSource<'a>, - gimlet_phase_1: TargetSource<'a>, - phase_2: TargetSource<'a>, - extra_targets: HashMap>, -} - -fn read_os_tarball_metadata_blocking( - file: std::fs::File, - tarball_path: Utf8PathBuf, -) -> Result>, Error> { - let mut metadata = HashMap::new(); - let mut archive = tar::Archive::new(GzDecoder::new(file)); - for entry in try_path!(archive.entries(), ReadFile, tarball_path) { - let mut entry = try_path!(entry, ReadFile, tarball_path); - if entry.header().entry_type() != tar::EntryType::Regular { - continue; - } - let path = try_path!(entry.path(), ReadFile, tarball_path); - let Some(parent) = path.parent() else { continue }; - if parent != "image" { - continue; - }; - let Some(extension) = path.extension() else { continue }; - if extension != "txt" { - continue; - } - let file_name = Path::new( - path.file_name() - .expect("a path with an extension must have a file name"), - ); - if file_name == "zfs.img" { - break; - } - let file_name = try_path!( - Utf8PathBuf::try_from(file_name.to_owned()) - .map_err(|error| error.into_io_error()), - ReadFile, - tarball_path - ); - let mut writer = BytesMut::new().writer(); - try_path!( - std::io::copy(&mut entry, &mut writer), - ReadFile, - tarball_path - ); - let source = BytesSource(writer.into_inner().freeze()); - metadata.insert(file_name.to_owned(), source.into()); - } - Ok(metadata) -} - -#[derive(Debug)] -struct CabooseData { - tags: KnownArtifactTags, - name: String, - version: ArtifactVersion, -} - -impl CabooseData { - fn new( - caboose: &Caboose, - tag_fn: F, - path: &Utf8Path, - ) -> Result - where - F: FnOnce(&Caboose) -> Result, - { - let tags = try_path!(tag_fn(caboose), ReadCaboose, path); - let name = try_path!( - tufaceous_artifact::hubris::read_name(caboose), - ReadCaboose, - path - ); - let version = try_path!( - tufaceous_artifact::hubris::read_version(caboose), - ReadCaboose, - path - ); - Ok(Self { - tags, - name: name.to_owned(), - version: ArtifactVersion::new(version)?, - }) - } - - fn generate_fake_archive(&self) -> Result { - let (kind, board, sign) = match &self.tags { - KnownArtifactTags::Rot { board, sign, .. } => { - ("rot", board, sign.0.as_ref()) - } - KnownArtifactTags::RotBootloader { board, sign } => { - ("rot-bootloader", board, sign.0.as_ref()) - } - KnownArtifactTags::Sp { board } => ("sp", board, None), - _ => panic!("generate_fake_archive called with non-hubris tags"), - }; - - let mut builder = CabooseBuilder::default() - .git_commit(format!("this-is-a-fake-{kind}")) - .board(board) - .name(board) - .version(self.version.to_string()); - if let Some(sign) = sign { - builder = builder.sign(sign); - } - let caboose = builder.build(); - - let mut builder = HubrisArchiveBuilder::with_fake_image(); - builder - .write_caboose(caboose.as_slice()) - .map_err(ErrorKind::GenerateFakeHubrisArchive)?; - let vec = builder - .build_to_vec() - .map_err(ErrorKind::GenerateFakeHubrisArchive)?; - Ok(BytesSource(vec.into())) - } -} - #[derive(Debug, Default)] struct TargetMap<'a>(BTreeMap>); @@ -984,3 +437,34 @@ impl<'a> TargetMap<'a> { Ok(()) } } + +pub(crate) fn generate_installinator_document( + artifacts: impl Iterator, impl AsRef<[u8]>)>, + version: ArtifactVersion, +) -> Result, Error> { + let target_name = format!("installinator_document-{version}.json"); + let mut document = InstallinatorDocument::default(); + for (artifact, sha256) in artifacts { + let artifact = artifact.as_ref(); + if let Ok(tags) = KnownArtifactTags::from_tags(&artifact.tags) + && let Some(kind) = tags.to_installinator() + && let Some(file_name) = + Utf8Path::new(&artifact.target_name).file_name() + && let Ok(sha256) = sha256.as_ref().try_into() + { + document.artifacts.insert(InstallinatorArtifact { + file_name: file_name.to_owned(), + kind, + sha256, + }); + } + } + let source = BytesSource::json(&document) + .map_err(ErrorKind::SerializeInstallinator)?; + Ok(Output::new( + target_name, + version, + KnownArtifactTags::InstallinatorDocument, + source, + )) +} diff --git a/lib/src/edit/fake.rs b/lib/src/edit/fake.rs new file mode 100644 index 0000000..ddb4f70 --- /dev/null +++ b/lib/src/edit/fake.rs @@ -0,0 +1,118 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use tufaceous_artifact::ArtifactVersion; +use tufaceous_artifact::Artifacts; +use tufaceous_artifact::OsVariant; +use tufaceous_artifact::RotBootloaderTags; +use tufaceous_artifact::RotSlot; +use tufaceous_artifact::RotTags; +use tufaceous_artifact::Sign; +use tufaceous_artifact::SpTags; + +use crate::edit::generate_installinator_document; +use crate::edit::input::Input; +use crate::edit::source::BytesSource; +use crate::error::Error; +use crate::schema::ArtifactSchema; + +pub trait ArtifactsExt: Sized { + fn fake(version: ArtifactVersion) -> Result; +} + +impl ArtifactsExt for Artifacts { + fn fake(version: ArtifactVersion) -> Result { + let mut artifacts = Artifacts::default(); + for input in Input::fake(&version)? { + for output in input.outputs() { + artifacts.extend(output.into_artifact()); + } + } + artifacts.extend( + generate_installinator_document( + artifacts.iter().map(|artifact| { + ( + ArtifactSchema { + target_name: artifact.target_name.clone(), + version: artifact.version.clone(), + tags: artifact.tags.clone(), + }, + &artifact.hash, + ) + }), + version, + )? + .into_artifact(), + ); + Ok(artifacts) + } +} + +impl Input { + pub(crate) fn fake(version: &ArtifactVersion) -> Result, Error> { + let mut inputs = Vec::new(); + for hashes in [4, 16] { + inputs + .push(Self::fake_measurement_corpus(hashes, version.clone())?); + } + for variant in [OsVariant::Host, OsVariant::Recovery] { + inputs.push(Self::fake_os_images(variant, version.clone())); + } + for slot in [RotSlot::A, RotSlot::B] { + inputs.push(Self::fake_rot_archive( + RotTags { + rot_board: "fake-rot".into(), + rot_sign: Sign::UNSIGNED, + rot_slot: slot, + }, + version.clone(), + )?); + } + inputs.push(Self::fake_rot_bootloader_archive( + RotBootloaderTags { + rot_board: "fake-rot".into(), + rot_sign: Sign::UNSIGNED, + }, + version.clone(), + )?); + for board in ["fake-gimlet", "fake-cosmo", "fake-sidecar", "fake-psc"] { + inputs.push(Self::fake_sp_archive( + SpTags { sp_board: board.into() }, + version.clone(), + )?); + } + for name in ["zone1", "zone2"] { + inputs.push(Self::fake_zone_image(name.into(), version.clone())?); + } + Ok(inputs) + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use semver::Version; + use tufaceous_artifact::ArtifactVersion; + use tufaceous_artifact::Artifacts; + + use crate::Repository; + use crate::edit::ArtifactsExt; + + #[tokio::test] + async fn fake_artifacts_equals_fake_repo() { + let log = slog::Logger::root(slog::Discard, slog::o!()); + let system_version = Version::new(1, 0, 0); + let version = ArtifactVersion::new(system_version.to_string()).unwrap(); + + let artifacts = Artifacts::fake(version).unwrap(); + // sleep 1 second to ensure any embedded timestamps would be different. + // we could ostensibly use `tokio::time::Instant` throughout the code + // base but that wouldn't take into account third-party libraries we use + // to generate fake artifacts. + tokio::time::sleep(Duration::from_secs(1)).await; + let repo = Repository::fake(system_version, &log).await.unwrap(); + assert_eq!(&artifacts, repo.artifacts()); + } +} diff --git a/lib/src/edit/guess.rs b/lib/src/edit/guess.rs new file mode 100644 index 0000000..78a9596 --- /dev/null +++ b/lib/src/edit/guess.rs @@ -0,0 +1,69 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::ops::ControlFlow; + +use bytes::Bytes; +use bytes::BytesMut; +use camino::Utf8PathBuf; +use tokio::fs::File; +use tokio::io::AsyncReadExt; +use tufaceous_artifact::ArtifactVersion; + +use crate::edit::input::Input; +use crate::edit::source::FileSource; +use crate::edit::source::TargetSource; +use crate::error::Error; +use crate::error::ErrorKind; +use crate::error::try_path; + +pub(crate) struct GuessInput { + pub(crate) file_start: Bytes, + pub(crate) source: FileSource, + pub(crate) version: ArtifactVersion, +} + +pub(crate) type GuessResult = + Result>, GuessInput>, Error>; + +impl Input> { + pub(crate) async fn guess( + path: Utf8PathBuf, + version: ArtifactVersion, + ) -> Result { + if let Some(result) = Self::guess_os_images(&path, &version).await { + return result; + } + + let mut file = try_path!(File::open(&path).await, OpenFile, path); + let mut buf = BytesMut::zeroed(4096); + let n = try_path!(file.read(&mut buf).await, ReadFile, path); + if n == 0 { + // we're not going to try to guess an empty file + return Err(ErrorKind::GuessArtifact { path }.into()); + } + buf.truncate(n); + + let guess_input = GuessInput { + file_start: buf.freeze(), + source: FileSource::from_file(file, path.clone()), + version, + }; + let guess_input = + match Self::guess_measurement_corpus(guess_input).await? { + ControlFlow::Break(input) => return Ok(input), + ControlFlow::Continue(guess_input) => guess_input, + }; + let guess_input = match Self::guess_hubris_archive(guess_input).await? { + ControlFlow::Break(input) => return Ok(input), + ControlFlow::Continue(guess_input) => guess_input, + }; + match Self::guess_zone_image(guess_input).await? { + ControlFlow::Break(input) => Ok(input), + ControlFlow::Continue(_) => { + Err(ErrorKind::GuessArtifact { path }.into()) + } + } + } +} diff --git a/lib/src/edit/hubris_archive.rs b/lib/src/edit/hubris_archive.rs new file mode 100644 index 0000000..b93601c --- /dev/null +++ b/lib/src/edit/hubris_archive.rs @@ -0,0 +1,201 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::ops::ControlFlow; + +use hubtools::Caboose; +use hubtools::CabooseBuilder; +use hubtools::HubrisArchiveBuilder; +use tufaceous_artifact::ArtifactVersion; +use tufaceous_artifact::RotBootloaderTags; +use tufaceous_artifact::RotSlot; +use tufaceous_artifact::RotTags; +use tufaceous_artifact::Sign; +use tufaceous_artifact::SpTags; +use tufaceous_artifact::hubris::read_name; +use tufaceous_artifact::hubris::read_version; + +use crate::edit::guess::GuessInput; +use crate::edit::guess::GuessResult; +use crate::edit::input::Input; +use crate::edit::source::BytesSource; +use crate::edit::source::FileSource; +use crate::edit::source::TargetSource; +use crate::error::Error; +use crate::error::ErrorKind; +use crate::error::try_path; + +impl Input> { + pub(crate) async fn rot_archive( + mut source: FileSource, + caboose: Option, + slot: RotSlot, + ) -> Result { + let caboose = match caboose { + Some(caboose) => caboose, + None => source.read_hubris_caboose().await?, + }; + let tags = try_path!( + RotTags::from_caboose(&caboose, slot), + ReadCaboose, + source.path + ); + let version = ArtifactVersion::new(try_path!( + read_version(&caboose), + ReadCaboose, + source.path + ))?; + Ok(Self::Rot { source: source.into(), tags, version }) + } + + pub(crate) async fn rot_bootloader_archive( + mut source: FileSource, + caboose: Option, + ) -> Result { + let caboose = match caboose { + Some(caboose) => caboose, + None => source.read_hubris_caboose().await?, + }; + let tags = try_path!( + RotBootloaderTags::from_caboose(&caboose), + ReadCaboose, + source.path + ); + let version = ArtifactVersion::new(try_path!( + read_version(&caboose), + ReadCaboose, + source.path + ))?; + Ok(Self::RotBootloader { source: source.into(), tags, version }) + } + + pub(crate) async fn sp_archive( + mut source: FileSource, + caboose: Option, + ) -> Result { + let caboose = match caboose { + Some(caboose) => caboose, + None => source.read_hubris_caboose().await?, + }; + let tags = + try_path!(SpTags::from_caboose(&caboose), ReadCaboose, source.path); + let name = + try_path!(read_name(&caboose), ReadCaboose, source.path).to_owned(); + let version = ArtifactVersion::new(try_path!( + read_version(&caboose), + ReadCaboose, + source.path + ))?; + Ok(Self::Sp { source: source.into(), tags, name, version }) + } + + pub(crate) async fn guess_hubris_archive( + mut input: GuessInput, + ) -> GuessResult { + if !input.file_start.starts_with(b"PK\x03\x04") { + return Ok(ControlFlow::Continue(input)); + } + let Ok(archive) = input.source.read_hubris_archive().await else { + return Ok(ControlFlow::Continue(input)); + }; + let Ok(caboose) = archive.read_caboose() else { + return Ok(ControlFlow::Continue(input)); + }; + // HACK: We are reading the `image-name` file in the archive, which + // appears to be "a" or "b" if it's an ROT image, "default" if it's + // an SP image, and nonexistent if it's an ROT bootloader image. This + // seems fragile. Ideally this can be in the caboose someday (see + // sprot-release#74). + match archive.image_name().as_deref() { + Ok("a") => { + Self::rot_archive(input.source, Some(caboose), RotSlot::A).await + } + Ok("b") => { + Self::rot_archive(input.source, Some(caboose), RotSlot::B).await + } + Ok("default") => { + Self::sp_archive(input.source, Some(caboose)).await + } + Err(hubtools::Error::MissingFile(_, _)) => { + Self::rot_bootloader_archive(input.source, Some(caboose)).await + } + _ => return Ok(ControlFlow::Continue(input)), + } + .map(ControlFlow::Break) + } +} + +impl Input { + pub(crate) fn fake_rot_archive( + tags: RotTags, + version: ArtifactVersion, + ) -> Result { + let data = CabooseData { + board: &tags.rot_board, + sign: &tags.rot_sign, + commit: "this-is-a-fake-rot", + version: &version, + }; + let source = data.generate_fake_archive()?; + Ok(Input::Rot { source, tags, version }) + } + + pub(crate) fn fake_rot_bootloader_archive( + tags: RotBootloaderTags, + version: ArtifactVersion, + ) -> Result { + let data = CabooseData { + board: &tags.rot_board, + sign: &tags.rot_sign, + commit: "this-is-a-fake-rot-bootloader", + version: &version, + }; + let source = data.generate_fake_archive()?; + Ok(Input::RotBootloader { source, tags, version }) + } + + pub(crate) fn fake_sp_archive( + tags: SpTags, + version: ArtifactVersion, + ) -> Result { + let data = CabooseData { + board: &tags.sp_board, + sign: &Sign::UNSIGNED, + commit: "this-is-a-fake-sp", + version: &version, + }; + let source = data.generate_fake_archive()?; + Ok(Input::Sp { source, name: tags.sp_board.clone(), tags, version }) + } +} + +struct CabooseData<'a> { + board: &'a str, + sign: &'a Sign, + commit: &'static str, + version: &'a ArtifactVersion, +} + +impl CabooseData<'_> { + fn generate_fake_archive(self) -> Result { + let mut builder = CabooseBuilder::default() + .board(self.board) + .name(self.board) + .git_commit(self.commit) + .version(self.version.to_string()); + if let Some(sign) = self.sign.0.as_ref() { + builder = builder.sign(sign); + } + let caboose = builder.build(); + + let mut builder = HubrisArchiveBuilder::with_fake_image(); + builder + .write_caboose(caboose.as_slice()) + .map_err(ErrorKind::GenerateFakeHubrisArchive)?; + let vec = builder + .build_to_vec() + .map_err(ErrorKind::GenerateFakeHubrisArchive)?; + Ok(BytesSource::new(vec)) + } +} diff --git a/lib/src/edit/input.rs b/lib/src/edit/input.rs new file mode 100644 index 0000000..38d0377 --- /dev/null +++ b/lib/src/edit/input.rs @@ -0,0 +1,213 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::BTreeMap; + +use camino::Utf8PathBuf; +use sha2::Digest; +use sha2::Sha256; +use tufaceous_artifact::Artifact; +use tufaceous_artifact::ArtifactHash; +use tufaceous_artifact::ArtifactVersion; +use tufaceous_artifact::KnownArtifactTags; +use tufaceous_artifact::OsBoard; +use tufaceous_artifact::OsPhase1Tags; +use tufaceous_artifact::OsPhase2Tags; +use tufaceous_artifact::OsVariant; +use tufaceous_artifact::RotBootloaderTags; +use tufaceous_artifact::RotTags; +use tufaceous_artifact::SpTags; +use tufaceous_artifact::ZoneTags; + +use crate::COSMO_PHASE_1_PATH; +use crate::GIMLET_PHASE_1_PATH; +use crate::PHASE_2_PATH; +use crate::edit::source::BytesSource; +use crate::schema::ArtifactSchema; + +#[derive(Debug)] +pub(crate) enum Input { + MeasurementCorpus { + source: Source, + corim_id: String, + sha256: ArtifactHash, + version: ArtifactVersion, + }, + OsImages { + cosmo_phase_1: Source, + gimlet_phase_1: Source, + phase_2: Source, + extra_targets: BTreeMap, + os_variant: OsVariant, + version: ArtifactVersion, + }, + Rot { + source: Source, + tags: RotTags, + version: ArtifactVersion, + }, + RotBootloader { + source: Source, + tags: RotBootloaderTags, + version: ArtifactVersion, + }, + Sp { + source: Source, + tags: SpTags, + name: String, + version: ArtifactVersion, + }, + Zone { + source: Source, + tags: ZoneTags, + version: ArtifactVersion, + }, +} + +impl Input { + pub(crate) fn outputs(self) -> Vec> { + match self { + Input::MeasurementCorpus { source, corim_id, sha256, version } => { + let target_name = format!( + "measurements/{corim_id}-{}.cbor", + hex::encode(sha256) + ); + let tags = KnownArtifactTags::MeasurementCorpus {}; + vec![Output::new(target_name, version, tags, source)] + } + Input::OsImages { + cosmo_phase_1, + gimlet_phase_1, + phase_2, + extra_targets, + os_variant, + version, + } => { + let base = Utf8PathBuf::from(format!("os-{os_variant}")); + let mut vec = Vec::new(); + vec.push(Output::new( + base.join("image").join(COSMO_PHASE_1_PATH).into(), + version.clone(), + OsPhase1Tags { os_variant, os_board: OsBoard::Cosmo } + .into(), + cosmo_phase_1, + )); + vec.push(Output::new( + base.join("image").join(GIMLET_PHASE_1_PATH).into(), + version.clone(), + OsPhase1Tags { os_variant, os_board: OsBoard::Gimlet } + .into(), + gimlet_phase_1, + )); + vec.push(Output::new( + base.join("image").join(PHASE_2_PATH).into(), + version, + OsPhase2Tags { os_variant }.into(), + phase_2, + )); + for (file_name, source) in extra_targets { + vec.push(Output::extra( + base.join(file_name).into(), + source, + )); + } + vec + } + Input::Rot { source, tags, version } => { + let target_name = format!( + "rot/{board}-{sign}-{version}-slot-{slot}.zip", + board = tags.rot_board, + sign = tags.rot_sign, + slot = tags.rot_slot + ); + vec![Output::new(target_name, version, tags.into(), source)] + } + Input::RotBootloader { source, tags, version } => { + let target_name = format!( + "rot-bootloader/{board}-{sign}-{version}.zip", + board = tags.rot_board, + sign = tags.rot_sign + ); + vec![Output::new(target_name, version, tags.into(), source)] + } + Input::Sp { source, tags, name, version } => { + let target_name = format!("sp/{name}-{version}.zip"); + if tags.sp_board.as_str() == name { + vec![Output::new(target_name, version, tags.into(), source)] + } else { + // This is likely a lab image. As of writing these are + // stored in the TUF repo for manufacturing but are + // explicitly ignored by the control plane, as they can + // never be used in an actual rack. The current thinking is + // that they will eventually no longer need to be in the TUF + // repo. Add these as an extra target, not an artifact. + vec![Output::extra(target_name, source)] + } + } + Input::Zone { source, tags, version } => { + let target_name = + format!("zones/{name}.tar.gz", name = tags.zone_name); + vec![Output::new(target_name, version, tags.into(), source)] + } + } + } +} + +#[derive(Debug)] +pub(crate) struct Output { + pub(crate) target_name: String, + pub(crate) source: Source, + artifact_data: Option, +} + +#[derive(Debug)] +pub(crate) struct ArtifactData { + tags: KnownArtifactTags, + version: ArtifactVersion, +} + +impl Output { + pub(crate) fn new( + target_name: String, + version: ArtifactVersion, + tags: KnownArtifactTags, + source: Source, + ) -> Self { + Output { + target_name, + source, + artifact_data: Some(ArtifactData { tags, version }), + } + } + + fn extra(target_name: String, source: Source) -> Self { + Output { target_name, source, artifact_data: None } + } + + pub(crate) fn to_artifact_schema(&self) -> Option { + let data = self.artifact_data.as_ref()?; + Some(ArtifactSchema { + target_name: self.target_name.clone(), + version: data.version.clone(), + tags: data.tags.to_tags(), + }) + } +} + +impl Output { + pub(crate) fn into_artifact(self) -> Option { + let data = self.artifact_data?; + let mut hasher = Sha256::new(); + for bytes in self.source.iter_bytes() { + hasher.update(bytes); + } + Some(Artifact { + target_name: self.target_name, + version: data.version, + tags: data.tags.to_tags(), + hash: ArtifactHash(hasher.finalize().into()), + length: self.source.length().try_into().unwrap(), + }) + } +} diff --git a/lib/src/edit/measurement_corpus.rs b/lib/src/edit/measurement_corpus.rs new file mode 100644 index 0000000..615a5eb --- /dev/null +++ b/lib/src/edit/measurement_corpus.rs @@ -0,0 +1,104 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::ops::ControlFlow; + +use bytes::BufMut; +use bytes::BytesMut; +use rats_corim::Corim; +use rats_corim::CorimBuilder; +use sha2::Digest; +use sha2::Sha256; +use tufaceous_artifact::ArtifactHash; +use tufaceous_artifact::ArtifactVersion; + +use crate::edit::guess::GuessInput; +use crate::edit::guess::GuessResult; +use crate::edit::input::Input; +use crate::edit::source::BytesSource; +use crate::edit::source::FileSource; +use crate::edit::source::TargetSource; +use crate::error::Error; +use crate::error::ErrorKind; +use crate::error::try_path; + +impl Input> { + pub(crate) async fn measurement_corpus( + mut source: FileSource, + corim: Option, + version: ArtifactVersion, + ) -> Result { + let Corim { id, .. } = match corim { + Some(corim) => corim, + None => { + let v = source.read_to_end().await?; + try_path!( + ciborium::from_reader(v.as_slice()), + Corim, + &source.path + ) + } + }; + let sha256 = source.sha256().await?; + Ok(Self::MeasurementCorpus { + source: source.into(), + corim_id: id, + sha256, + version, + }) + } + + pub(crate) async fn guess_measurement_corpus( + input: GuessInput, + ) -> GuessResult { + if !matches!(input.file_start[0], 0xa0..=0xbf /* CBOR map */) { + return Ok(ControlFlow::Continue(input)); + } + let corim = + match ciborium::from_reader::(&mut &*input.file_start) { + Ok(corim) => Some(corim), + Err(ciborium::de::Error::Io(err)) + if err.kind() == std::io::ErrorKind::UnexpectedEof => + { + // This was plausibly a CoRIM manifest until we hit the end of + // the buffer, indicating a very high likelihood that if we read + // the entire thing it'd still be a CoRIM manifest. + None + } + Err(_) => return Ok(ControlFlow::Continue(input)), + }; + Self::measurement_corpus(input.source, corim, input.version) + .await + .map(ControlFlow::Break) + } +} + +impl Input { + pub(crate) fn fake_measurement_corpus( + hashes: usize, + version: ArtifactVersion, + ) -> Result { + let mut builder = CorimBuilder::new(); + builder.vendor("fake-vendor".to_string()); + builder.id("fake-measurement-id".to_string()); + builder.tag_id("fake-tag-id".to_string()); + for i in 0..hashes { + builder.add_hash(format!("layer{i}"), 10, vec![0; 32]); + } + let corim = builder + .build() + .map_err(ErrorKind::GenerateFakeMeasurementCorpus)?; + + let mut writer = BytesMut::new().writer(); + ciborium::into_writer(&corim, &mut writer) + .map_err(ErrorKind::SerializeFakeMeasurementCorpus)?; + let bytes = writer.into_inner().freeze(); + Ok(Input::MeasurementCorpus { + corim_id: corim.id, + sha256: ArtifactHash(Sha256::digest(&bytes).into()), + source: BytesSource::new(bytes), + version, + }) + } +} diff --git a/lib/src/edit/mod.rs b/lib/src/edit/mod.rs index 839a057..90029d4 100644 --- a/lib/src/edit/mod.rs +++ b/lib/src/edit/mod.rs @@ -3,12 +3,23 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. mod editor; +mod fake; +mod guess; +mod hubris_archive; +mod input; mod key; +mod measurement_corpus; +mod os_images; mod root; mod sign; mod source; +mod zone_image; pub use editor::*; +pub use fake::*; pub use key::*; pub use root::*; pub use sign::*; + +const KIB: usize = 1024; +const MIB: usize = 1024 * KIB; diff --git a/lib/src/edit/os_images.rs b/lib/src/edit/os_images.rs new file mode 100644 index 0000000..182d957 --- /dev/null +++ b/lib/src/edit/os_images.rs @@ -0,0 +1,218 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::BTreeMap; +use std::path::Path; + +use bytes::BufMut; +use bytes::BytesMut; +use camino::Utf8Path; +use camino::Utf8PathBuf; +use flate2::read::GzDecoder; +use futures_util::TryStreamExt; +use tokio::fs::File; +use tokio::io::AsyncReadExt; +use tufaceous_artifact::ArtifactVersion; +use tufaceous_artifact::OsVariant; + +use crate::COSMO_PHASE_1_PATH; +use crate::GIMLET_PHASE_1_PATH; +use crate::PHASE_2_PATH; +use crate::edit::KIB; +use crate::edit::MIB; +use crate::edit::input::Input; +use crate::edit::source::BytesSource; +use crate::edit::source::FileSource; +use crate::edit::source::TargetSource; +use crate::error::Error; +use crate::error::ErrorKind; +use crate::error::try_path; + +impl Input> { + pub(crate) async fn os_images( + os_variant: OsVariant, + dir: &Utf8Path, + phase_2: Option, + version: ArtifactVersion, + ) -> Result { + let cosmo_phase_1 = + FileSource::open(dir.join(COSMO_PHASE_1_PATH)).await?.into(); + let gimlet_phase_1 = + FileSource::open(dir.join(GIMLET_PHASE_1_PATH)).await?.into(); + let phase_2 = match phase_2 { + Some(phase_2) => phase_2, + None => FileSource::open(dir.join(PHASE_2_PATH)).await?, + }; + + // If `os.tar.gz` is present, read the `image/*.txt` files from it + // as extra sources. Stop once we get to `zfs.img`, which comes after + // the additional metadata. (If a GzDecoder was seekable we could + // conceivably skip past any files we don't want to read, but alas.) + // https://github.com/oxidecomputer/helios/blob/f145457f6ccb13a139b8d93408b9b4de5db57bd6/tools/helios-build/src/main.rs#L1881-L1885 + let tarball_path = dir.join("os.tar.gz"); + let mut extra_targets = match tokio::fs::File::open(&tarball_path).await + { + Ok(file) => { + let file = file.into_std().await; + tokio::task::spawn_blocking(move || { + read_os_tarball_metadata_blocking(file, tarball_path) + }) + .await?? + } + Err(source) if source.kind() == std::io::ErrorKind::NotFound => { + // We might be reading from an unpacked Tufaceous archive; look + // for any `*.txt` files and include them. + let mut extra_targets = BTreeMap::new(); + let mut read_dir = crate::util::read_dir(dir.into()).await?; + while let Some(entry) = read_dir.try_next().await? { + if entry.path().extension() != Some("txt") { + continue; + } + let file_name = entry.file_name().to_owned(); + let source = FileSource::open(entry.into_path()).await?; + extra_targets.insert(file_name, source.into()); + } + extra_targets + } + Err(source) => { + return Err(ErrorKind::OpenFile { + source, + path: Some(tarball_path), + } + .into()); + } + }; + for path in ["unix.z", "cpio.z"] { + let source = FileSource::open(dir.join(path)).await?; + extra_targets.insert(path.into(), source.into()); + } + + Ok(Self::OsImages { + cosmo_phase_1, + gimlet_phase_1, + phase_2: phase_2.into(), + extra_targets, + os_variant, + version, + }) + } + + pub(crate) async fn guess_os_images( + path: &Utf8Path, + version: &ArtifactVersion, + ) -> Option> { + let phase_2_path = path.join(PHASE_2_PATH); + let mut file = File::open(&phase_2_path).await.ok()?; + // Read the header block from the image and guess whether it's a + // recovery image based on the image name. + let mut buf = [0; 4096]; + file.read_exact(&mut buf).await.ok()?; + if !buf.starts_with(&0x1DEB0075_u32.to_le_bytes()) { + return None; + } + // see https://github.com/oxidecomputer/boot-image-tools/blob/main/src/diskimage.rs + let image_name = &buf[200..328]; + let variant = if image_name.starts_with(b"recovery") { + OsVariant::Recovery + } else { + OsVariant::Host + }; + + let phase_2 = FileSource::from_file(file, phase_2_path); + Some( + Self::os_images(variant, path, Some(phase_2), version.clone()) + .await, + ) + } +} + +fn read_os_tarball_metadata_blocking( + file: std::fs::File, + tarball_path: Utf8PathBuf, +) -> Result>, Error> { + let mut metadata = BTreeMap::new(); + let mut archive = tar::Archive::new(GzDecoder::new(file)); + for entry in try_path!(archive.entries(), ReadFile, tarball_path) { + let mut entry = try_path!(entry, ReadFile, tarball_path); + if entry.header().entry_type() != tar::EntryType::Regular { + continue; + } + let path = try_path!(entry.path(), ReadFile, tarball_path); + let Some(parent) = path.parent() else { continue }; + if parent != "image" { + continue; + }; + let Some(extension) = path.extension() else { continue }; + if extension != "txt" { + continue; + } + let file_name = Path::new( + path.file_name() + .expect("a path with an extension must have a file name"), + ); + if file_name == "zfs.img" { + break; + } + let file_name = try_path!( + Utf8PathBuf::try_from(file_name.to_owned()) + .map_err(|error| error.into_io_error()), + ReadFile, + tarball_path + ); + let mut writer = BytesMut::new().writer(); + try_path!( + std::io::copy(&mut entry, &mut writer), + ReadFile, + tarball_path + ); + let source = BytesSource::new(writer.into_inner().freeze()); + metadata.insert(file_name.into(), source.into()); + } + Ok(metadata) +} + +impl Input { + pub(crate) fn fake_os_images( + os_variant: OsVariant, + version: ArtifactVersion, + ) -> Self { + let cosmo_phase_1 = BytesSource::fake_padded( + format!("cosmo {os_variant} OS phase 1 image version {version}\n"), + MIB, + ); + let gimlet_phase_1 = BytesSource::fake_padded( + format!("gimlet {os_variant} OS phase 1 image version {version}\n"), + MIB, + ); + let phase_2 = BytesSource::fake_padded( + format!("{os_variant} OS phase 2 image version {version}\n"), + 4 * MIB, + ); + + let mut extra_targets = BTreeMap::new(); + extra_targets.insert( + String::from("unix.z"), + BytesSource::fake_padded( + format!("{os_variant} OS unix.z version {version}\n"), + 64 * KIB, + ), + ); + extra_targets.insert( + String::from("cpio.z"), + BytesSource::fake_padded( + format!("{os_variant} OS cpio.z version {version}\n"), + 256 * KIB, + ), + ); + + Self::OsImages { + cosmo_phase_1, + gimlet_phase_1, + phase_2, + extra_targets, + os_variant, + version, + } + } +} diff --git a/lib/src/edit/source.rs b/lib/src/edit/source.rs index 3bf6db5..9243762 100644 --- a/lib/src/edit/source.rs +++ b/lib/src/edit/source.rs @@ -6,6 +6,7 @@ use std::convert::Infallible; use std::pin::Pin; use bytes::Bytes; +use bytes::BytesMut; use camino::Utf8PathBuf; use futures_util::Stream; use futures_util::StreamExt; @@ -38,14 +39,13 @@ pub(crate) enum TargetSource<'a> { Bytes(BytesSource), File(FileSource), Repository(RepositorySource<'a>), - Fake(FakeSource), } impl TargetSource<'_> { pub(crate) fn cost(&self) -> usize { match self { - TargetSource::Bytes(_) => 0, - TargetSource::Fake(_) => 1, + TargetSource::Bytes(BytesSource { fake_length: None, .. }) => 0, + TargetSource::Bytes(_) => 1, TargetSource::File(_) => 2, TargetSource::Repository(_) => 3, } @@ -55,14 +55,11 @@ impl TargetSource<'_> { &mut self, ) -> Pin> + '_>> { match self { - TargetSource::Bytes(BytesSource(bytes)) => { - Box::pin(stream::once(std::future::ready(Ok(bytes.clone())))) + TargetSource::Bytes(source) => { + Box::pin(source.stream().err_into::()) } TargetSource::File(source) => Box::pin(source.stream()), TargetSource::Repository(source) => Box::pin(source.stream()), - TargetSource::Fake(source) => { - Box::pin(source.stream().err_into::()) - } } } } @@ -85,28 +82,83 @@ impl<'a> From> for TargetSource<'a> { } } -impl From for TargetSource<'_> { - fn from(source: FakeSource) -> Self { - TargetSource::Fake(source) - } -} - #[derive(Debug, Clone)] -pub(crate) struct BytesSource(pub(crate) Bytes); +pub(crate) struct BytesSource { + bytes: Bytes, + fake_length: Option, + sha256: Option, +} impl BytesSource { + pub(crate) fn new(bytes: impl Into) -> Self { + Self { bytes: bytes.into(), fake_length: None, sha256: None } + } + pub(crate) fn json( data: &T, ) -> Result { let mut s = serde_json::to_string_pretty(data)?; s.push('\n'); - Ok(Self(s.into())) + Ok(Self { bytes: s.into(), fake_length: None, sha256: None }) + } + + pub(crate) fn fake_padded(prefix: impl Into, length: usize) -> Self { + Self { bytes: prefix.into(), fake_length: Some(length), sha256: None } + } + + pub(crate) fn iter_bytes(&self) -> impl Iterator + 'static { + const CHUNK_SIZE: usize = 8192; + + let mut bytes = self.bytes.clone(); + let mut remaining = match self.fake_length { + Some(length) => { + bytes.truncate(length); + length - bytes.len() + } + None => 0, + }; + let zero = BytesMut::zeroed(remaining.min(CHUNK_SIZE)).freeze(); + std::iter::once(bytes).chain(std::iter::from_fn(move || { + if remaining > 0 { + let slice = zero.slice(..remaining.min(CHUNK_SIZE)); + remaining = remaining.saturating_sub(slice.len()); + Some(slice) + } else { + None + } + })) } - pub(crate) fn into_target(self) -> Target<'static> { + pub(crate) fn stream( + &self, + ) -> impl Stream> + 'static { + stream::iter(self.iter_bytes().map(Ok)) + } + + pub(crate) fn length(&self) -> usize { + self.fake_length.unwrap_or_else(|| self.bytes.len()) + } + + async fn sha256(&mut self) -> ArtifactHash { + if let Some(sha256) = self.sha256 { + return sha256; + } + let mut stream = self.stream(); + let mut hasher = Sha256::new(); + while let Some(Ok(bytes)) = stream.next().await { + hasher.update(&bytes); + } + *self.sha256.insert(ArtifactHash(hasher.finalize().into())) + } + + pub(crate) async fn into_target(mut self) -> Target<'static> { Target { - length: self.0.len().try_into().unwrap(), - sha256: Sha256::digest(&self.0).to_vec(), + length: self + .fake_length + .unwrap_or(self.bytes.len()) + .try_into() + .unwrap(), + sha256: self.sha256().await.0.to_vec(), source: self.into(), } } @@ -114,7 +166,7 @@ impl BytesSource { #[derive(Debug)] pub(crate) struct FileSource { - file: File, + file: Box, pub(crate) path: Utf8PathBuf, length_sha256: Option<(u64, ArtifactHash)>, } @@ -126,7 +178,7 @@ impl FileSource { } pub(crate) fn from_file(file: File, path: Utf8PathBuf) -> Self { - Self { file, path, length_sha256: None } + Self { file: Box::new(file), path, length_sha256: None } } pub(crate) async fn into_target( @@ -234,58 +286,3 @@ impl<'a> RepositorySource<'a> { stream::once(self.repo.read_target(&self.target_name)).try_flatten() } } - -#[derive(Debug)] -pub(crate) struct FakeSource { - prefix: Bytes, - length: usize, - sha256: Option, -} - -impl FakeSource { - pub(crate) fn new(prefix: String, length: usize) -> Self { - Self { prefix: prefix.into(), length, sha256: None } - } - - async fn sha256(&mut self) -> ArtifactHash { - if let Some(sha256) = self.sha256 { - return sha256; - } - let mut stream = self.stream(); - let mut hasher = Sha256::new(); - while let Some(Ok(bytes)) = stream.next().await { - hasher.update(&bytes); - } - *self.sha256.insert(ArtifactHash(hasher.finalize().into())) - } - - pub(crate) async fn into_target(mut self) -> Target<'static> { - Target { - length: self.length.try_into().unwrap(), - sha256: self.sha256().await.0.to_vec(), - source: self.into(), - } - } - - pub(crate) fn stream( - &self, - ) -> impl Stream> + 'static { - const ZERO: Bytes = Bytes::from_static(&[0; 8192]); - - let mut prefix = self.prefix.clone(); - prefix.truncate(self.length); - let suffix = stream::unfold( - self.length.saturating_sub(prefix.len()), - |remaining| { - std::future::ready(if remaining == 0 { - None - } else if remaining < ZERO.len() { - Some((Ok(ZERO.slice(..remaining)), 0)) - } else { - Some((Ok(ZERO), remaining - ZERO.len())) - }) - }, - ); - stream::once(std::future::ready(Ok(prefix))).chain(suffix) - } -} diff --git a/lib/src/edit/zone_image.rs b/lib/src/edit/zone_image.rs new file mode 100644 index 0000000..4884e7c --- /dev/null +++ b/lib/src/edit/zone_image.rs @@ -0,0 +1,98 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::ops::ControlFlow; + +use bytes::BufMut; +use bytes::BytesMut; +use camino::Utf8PathBuf; +use flate2::Compression; +use flate2::read::GzDecoder; +use flate2::write::GzEncoder; +use tufaceous_artifact::ArtifactVersion; +use tufaceous_artifact::ZoneTags; +use tufaceous_brand_metadata::ArchiveType; +use tufaceous_brand_metadata::LayerInfo; +use tufaceous_brand_metadata::Metadata; + +use crate::edit::guess::GuessInput; +use crate::edit::guess::GuessResult; +use crate::edit::input::Input; +use crate::edit::source::BytesSource; +use crate::edit::source::FileSource; +use crate::edit::source::TargetSource; +use crate::error::Error; +use crate::error::ErrorKind; +use crate::error::try_path; + +impl Input> { + pub(crate) async fn zone_image(path: Utf8PathBuf) -> Result { + let cloned_path = path.clone(); + let (file, layer_info) = tokio::task::spawn_blocking(move || { + let file = try_path!(std::fs::File::open(&path), ReadFile, path); + let mut archive = tar::Archive::new(GzDecoder::new(file)); + let layer_info = try_path!( + Metadata::read_from_tar(&mut archive) + .and_then(|metadata| metadata.layer_info().cloned()), + ReadZoneOxideJson, + path + ); + Ok::<_, Error>((archive.into_inner().into_inner(), layer_info)) + }) + .await??; + let source = FileSource::from_file(file.into(), cloned_path); + Ok(Self::Zone { + source: source.into(), + tags: ZoneTags { zone_name: layer_info.pkg }, + version: layer_info.version, + }) + } + + pub(crate) async fn guess_zone_image(input: GuessInput) -> GuessResult { + // `oxide.json` is the first file of a zone image and is relatively + // small, so it should be contained entirely within the first 4K of the + // compressed tarball. + let mut archive = tar::Archive::new(GzDecoder::new(&*input.file_start)); + let Ok(layer_info) = Metadata::read_from_tar(&mut archive) + .and_then(Metadata::into_layer_info) + else { + return Ok(ControlFlow::Continue(input)); + }; + Ok(ControlFlow::Break(Self::Zone { + source: input.source.into(), + tags: ZoneTags { zone_name: layer_info.pkg }, + version: layer_info.version, + })) + } +} + +impl Input { + pub(crate) fn fake_zone_image( + zone_name: String, + version: ArtifactVersion, + ) -> Result { + let mut archive = tar::Builder::new(GzEncoder::new( + BytesMut::new().writer(), + Compression::best(), + )); + let metadata = Metadata::new(ArchiveType::Layer(LayerInfo { + pkg: zone_name.clone(), + version: version.clone(), + })); + metadata + .append_to_tar(&mut archive, 0) + .map_err(ErrorKind::GenerateFakeZoneImage)?; + let bytes = archive + .into_inner() + .and_then(GzEncoder::finish) + .map_err(ErrorKind::GenerateFakeZoneImage)? + .into_inner() + .freeze(); + Ok(Input::Zone { + source: BytesSource::new(bytes), + tags: ZoneTags { zone_name }, + version, + }) + } +} diff --git a/lib/src/error.rs b/lib/src/error.rs index 35c6b0b..f0c7368 100644 --- a/lib/src/error.rs +++ b/lib/src/error.rs @@ -118,6 +118,8 @@ pub enum ErrorKind { SerializeFakeMeasurementCorpus( #[source] ciborium::ser::Error, ), + #[error("failed to generate fake zone image")] + GenerateFakeZoneImage(#[source] std::io::Error), #[error("unspecified failure when generating ed25519 key")] Ed25519Generate, diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 1fc75bf..0ca62bf 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -14,3 +14,7 @@ mod zip_writer; pub use loader::*; pub use repo::*; pub use zip_transport::*; + +pub(crate) const COSMO_PHASE_1_PATH: &str = "cosmo.rom"; +pub(crate) const GIMLET_PHASE_1_PATH: &str = "gimlet.rom"; +pub(crate) const PHASE_2_PATH: &str = "zfs.img"; diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs index ef4341a..7dfa451 100644 --- a/lib/src/repo/v1.rs +++ b/lib/src/repo/v1.rs @@ -36,9 +36,15 @@ use tufaceous_artifact::ArtifactVersion; use tufaceous_artifact::Artifacts; use tufaceous_artifact::KnownArtifactTags; use tufaceous_artifact::OsBoard; +use tufaceous_artifact::OsPhase1Tags; +use tufaceous_artifact::OsPhase2Tags; use tufaceous_artifact::OsVariant; use tufaceous_artifact::RotSlot; +use tufaceous_artifact::ZoneTags; +use crate::COSMO_PHASE_1_PATH; +use crate::GIMLET_PHASE_1_PATH; +use crate::PHASE_2_PATH; use crate::Repository; use crate::error::Error; use crate::error::ErrorKind; @@ -244,9 +250,9 @@ pub(crate) async fn from_loaded( artifacts.insert(Artifact { target_name, version: layer_info.version, - tags: KnownArtifactTags::Zone { - name: layer_info.pkg, - } + tags: KnownArtifactTags::Zone(ZoneTags { + zone_name: layer_info.pkg, + }) .to_tags(), hash, length, @@ -407,17 +413,25 @@ fn unpack_os( entries: &mut HashMap, artifacts: &mut Artifacts, artifact: &V1Artifact, - variant: OsVariant, + os_variant: OsVariant, ) { for (tar_path, inner) in unpacked.entries { - let tags = match tar_path.as_ref() { - "image/cosmo.rom" => { - KnownArtifactTags::OsPhase1 { variant, board: OsBoard::Cosmo } + let tags = match tar_path.as_str().strip_prefix("image/") { + Some(COSMO_PHASE_1_PATH) => { + KnownArtifactTags::OsPhase1(OsPhase1Tags { + os_variant, + os_board: OsBoard::Cosmo, + }) + } + Some(GIMLET_PHASE_1_PATH) => { + KnownArtifactTags::OsPhase1(OsPhase1Tags { + os_variant, + os_board: OsBoard::Gimlet, + }) } - "image/gimlet.rom" => { - KnownArtifactTags::OsPhase1 { variant, board: OsBoard::Gimlet } + Some(PHASE_2_PATH) => { + KnownArtifactTags::OsPhase2(OsPhase2Tags { os_variant }) } - "image/zfs.img" => KnownArtifactTags::OsPhase2 { variant }, _ => continue, }; let target_name = format!("{}/{tar_path}", artifact.target); diff --git a/lib/src/schema.rs b/lib/src/schema.rs index b13c965..f0c9cae 100644 --- a/lib/src/schema.rs +++ b/lib/src/schema.rs @@ -28,3 +28,9 @@ pub(crate) struct ArtifactSchema { pub(crate) version: ArtifactVersion, pub(crate) tags: BTreeMap, } + +impl AsRef for ArtifactSchema { + fn as_ref(&self) -> &ArtifactSchema { + self + } +} From 20893298c7b018ade9818aa132a08dbcea8835ed Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 27 Jan 2026 21:19:12 -0800 Subject: [PATCH 018/137] clippy pedantic cleanup and explicit unwraps --- Cargo.lock | 11 - Cargo.toml | 4 +- artifact/src/installinator.rs | 2 +- artifact/src/version.rs | 16 +- bin/Cargo.toml | 1 - bin/src/show_target.rs | 3 +- bin/src/sign.rs | 14 +- brand-metadata/src/lib.rs | 3 +- lib/src/edit/editor.rs | 13 +- lib/src/edit/guess.rs | 2 +- lib/src/edit/input.rs | 2 +- lib/src/edit/measurement_corpus.rs | 15 +- lib/src/edit/mod.rs | 6 +- lib/src/edit/os_images.rs | 7 +- lib/src/edit/sign.rs | 6 +- lib/src/edit/source.rs | 43 ++-- lib/src/edit/zone_image.rs | 25 +- lib/src/loader.rs | 13 +- lib/src/repo.rs | 15 +- lib/src/repo/v1.rs | 401 ++++++++++++++--------------- lib/src/util.rs | 3 +- lib/src/zip_transport.rs | 18 +- lib/tests/tests.rs | 2 +- 23 files changed, 316 insertions(+), 309 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9e60eb5..22344c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -813,16 +813,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fs-err" -version = "3.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824f08d01d0f496b3eca4f001a13cf17690a6ee930043d20817f547455fd98f8" -dependencies = [ - "autocfg", - "tokio", -] - [[package]] name = "fs_extra" version = "1.3.0" @@ -2897,7 +2887,6 @@ dependencies = [ "camino", "chrono", "clap", - "fs-err", "futures-util", "semver", "slog", diff --git a/Cargo.toml b/Cargo.toml index a015585..aa022d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,6 @@ clap = { version = "4.5.31", features = ["cargo", "derive", "wrap_help"] } daft = { version = "0.1.1", features = ["derive"] } expectorate = "1.1.0" flate2 = "1.0.35" -fs-err = "3.2.0" futures-util = "0.3.31" hex = "0.4.3" hubtools = { git = "https://github.com/oxidecomputer/hubtools.git", branch = "main" } @@ -58,6 +57,9 @@ tufaceous-brand-metadata.path = "brand-metadata" url = "2.5.4" [workspace.lints.clippy] +pedantic = { level = "warn", priority = -1 } +must-use-candidate = "allow" +missing-errors-doc = "allow" [profile.dev.package.flate2] opt-level = 3 diff --git a/artifact/src/installinator.rs b/artifact/src/installinator.rs index c201cab..88bdf17 100644 --- a/artifact/src/installinator.rs +++ b/artifact/src/installinator.rs @@ -81,7 +81,7 @@ impl InstallinatorArtifactKind { /// Historically this was called `ArtifactHashId` and consists of two /// strings: `kind` and `hash`. It was developed before the present system /// of tags representing an artifact kind but remains in use for any -/// Installinator-related interfaces (namely Wicket and MUPdate overrides). +/// Installinator-related interfaces (namely Wicket and mupdate overrides). /// /// This schema is stored to disk and should not change. #[derive( diff --git a/artifact/src/version.rs b/artifact/src/version.rs index fc88499..4693bee 100644 --- a/artifact/src/version.rs +++ b/artifact/src/version.rs @@ -2,6 +2,11 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +#![cfg_attr( + any(test, feature = "proptest"), + expect(clippy::explicit_deref_methods) +)] + use std::borrow::Cow; use std::fmt; use std::str::FromStr; @@ -68,6 +73,10 @@ impl ArtifactVersion { } /// Creates a new `ArtifactVersion` from a static string. + /// + /// # Errors + /// + /// Returns an error if the string is not a valid artifact version. pub fn new_static( version: &'static str, ) -> Result { @@ -78,8 +87,11 @@ impl ArtifactVersion { } } - /// Creates a new `ArtifactVersion` at compile time, panicking if it is - /// invalid. + /// Creates a new `ArtifactVersion` from a static string. + /// + /// # Panics + /// + /// Panics if the string is not a valid artifact version. pub const fn new_const(s: &'static str) -> Self { match validate_version(s) { Ok(()) => Self(Cow::Borrowed(s)), diff --git a/bin/Cargo.toml b/bin/Cargo.toml index 73cdbee..583569e 100644 --- a/bin/Cargo.toml +++ b/bin/Cargo.toml @@ -14,7 +14,6 @@ anyhow.workspace = true camino.workspace = true chrono.workspace = true clap.workspace = true -fs-err = { workspace = true, features = ["tokio"] } futures-util.workspace = true semver.workspace = true slog.workspace = true diff --git a/bin/src/show_target.rs b/bin/src/show_target.rs index a395777..2823f92 100644 --- a/bin/src/show_target.rs +++ b/bin/src/show_target.rs @@ -4,6 +4,7 @@ use std::io::Write; +use anyhow::Context; use anyhow::Result; use camino::Utf8PathBuf; use clap::Parser; @@ -28,7 +29,7 @@ impl Args { let mut stream = repo.read_target(&self.target_name).await?; let mut stdout = std::io::stdout().lock(); while let Some(bytes) = stream.try_next().await? { - stdout.write_all(&bytes)?; + stdout.write_all(&bytes).context("failed to write to stdout")?; } Ok(()) } diff --git a/bin/src/sign.rs b/bin/src/sign.rs index b8739ef..ed9c4c9 100644 --- a/bin/src/sign.rs +++ b/bin/src/sign.rs @@ -5,10 +5,11 @@ use anyhow::Result; use camino::Utf8PathBuf; use clap::Parser; -use fs_err::tokio as fs; use tough::key_source::LocalKeySource; use tufaceous::edit::SignedRepository; use tufaceous::edit::UnsignedRepository; +use tufaceous::error::Error; +use tufaceous::error::ErrorKind; #[derive(Debug, Parser)] #[cfg_attr(test, derive(PartialEq))] @@ -30,7 +31,16 @@ impl SignOptions { unsigned = unsigned.generate_root(); } if let Some(path) = self.root { - let root = fs::read(path).await?; + let root = match tokio::fs::read(&path).await { + Ok(root) => root, + Err(source) => { + return Err(Error::from(ErrorKind::ReadFile { + source, + path: Some(path), + }) + .into()); + } + }; unsigned = unsigned.root(root); } for path in self.key { diff --git a/brand-metadata/src/lib.rs b/brand-metadata/src/lib.rs index c0e283c..fd0a48b 100644 --- a/brand-metadata/src/lib.rs +++ b/brand-metadata/src/lib.rs @@ -56,6 +56,7 @@ impl Metadata { Metadata { v: "1".into(), t: archive_type } } + #[expect(clippy::missing_panics_doc)] pub fn append_to_tar( &self, a: &mut tar::Builder, @@ -72,7 +73,7 @@ impl Metadata { h.set_gid(0); h.set_path("oxide.json")?; h.set_mode(0o444); - h.set_size(b.len().try_into().unwrap()); + h.set_size(b.len().try_into().expect("usize fits in u64")); h.set_mtime(mtime); h.set_cksum(); diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index e83e9a0..21d2933 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -38,6 +38,7 @@ use crate::schema::ArtifactSchema; use crate::schema::ArtifactsSchema; #[derive(Debug)] +#[must_use] pub struct RepositoryEditor<'a> { system_version: Version, artifact_version: Result, @@ -244,8 +245,8 @@ impl<'a> RepositoryEditor<'a> { mut self, target_name: String, version: ArtifactVersion, - tags: KnownArtifactTags, - length: usize, + tags: &KnownArtifactTags, + length: u64, ) -> Self { let prefix = format!("{target_name}\n{version}\n{tags:?}\n"); self.targets @@ -267,16 +268,12 @@ impl<'a> RepositoryEditor<'a> { Ok(editor) } - pub async fn metadata( - mut self, - key: String, - value: serde_json::Value, - ) -> Self { + pub fn metadata(mut self, key: String, value: serde_json::Value) -> Self { self.metadata.insert(key, value); self } - pub async fn remove_metadata(mut self, key: &str) -> Self { + pub fn remove_metadata(mut self, key: &str) -> Self { self.metadata.remove(key); self } diff --git a/lib/src/edit/guess.rs b/lib/src/edit/guess.rs index 78a9596..eeafbf7 100644 --- a/lib/src/edit/guess.rs +++ b/lib/src/edit/guess.rs @@ -59,7 +59,7 @@ impl Input> { ControlFlow::Break(input) => return Ok(input), ControlFlow::Continue(guess_input) => guess_input, }; - match Self::guess_zone_image(guess_input).await? { + match Self::guess_zone_image(guess_input)? { ControlFlow::Break(input) => Ok(input), ControlFlow::Continue(_) => { Err(ErrorKind::GuessArtifact { path }.into()) diff --git a/lib/src/edit/input.rs b/lib/src/edit/input.rs index 38d0377..d5c0f01 100644 --- a/lib/src/edit/input.rs +++ b/lib/src/edit/input.rs @@ -207,7 +207,7 @@ impl Output { version: data.version, tags: data.tags.to_tags(), hash: ArtifactHash(hasher.finalize().into()), - length: self.source.length().try_into().unwrap(), + length: self.source.length(), }) } } diff --git a/lib/src/edit/measurement_corpus.rs b/lib/src/edit/measurement_corpus.rs index 615a5eb..a4bd133 100644 --- a/lib/src/edit/measurement_corpus.rs +++ b/lib/src/edit/measurement_corpus.rs @@ -29,16 +29,11 @@ impl Input> { corim: Option, version: ArtifactVersion, ) -> Result { - let Corim { id, .. } = match corim { - Some(corim) => corim, - None => { - let v = source.read_to_end().await?; - try_path!( - ciborium::from_reader(v.as_slice()), - Corim, - &source.path - ) - } + let Corim { id, .. } = if let Some(corim) = corim { + corim + } else { + let v = source.read_to_end().await?; + try_path!(ciborium::from_reader(v.as_slice()), Corim, &source.path) }; let sha256 = source.sha256().await?; Ok(Self::MeasurementCorpus { diff --git a/lib/src/edit/mod.rs b/lib/src/edit/mod.rs index 90029d4..f66d762 100644 --- a/lib/src/edit/mod.rs +++ b/lib/src/edit/mod.rs @@ -21,5 +21,7 @@ pub use key::*; pub use root::*; pub use sign::*; -const KIB: usize = 1024; -const MIB: usize = 1024 * KIB; +const KIB: u64 = 1024; +const MIB: u64 = 1024 * KIB; +#[expect(clippy::unreadable_literal)] +const OXIDE_BOOT_MAGIC: [u8; 4] = 0x1DEB0075_u32.to_le_bytes(); diff --git a/lib/src/edit/os_images.rs b/lib/src/edit/os_images.rs index 182d957..2fd0913 100644 --- a/lib/src/edit/os_images.rs +++ b/lib/src/edit/os_images.rs @@ -21,6 +21,7 @@ use crate::GIMLET_PHASE_1_PATH; use crate::PHASE_2_PATH; use crate::edit::KIB; use crate::edit::MIB; +use crate::edit::OXIDE_BOOT_MAGIC; use crate::edit::input::Input; use crate::edit::source::BytesSource; use crate::edit::source::FileSource; @@ -108,7 +109,7 @@ impl Input> { // recovery image based on the image name. let mut buf = [0; 4096]; file.read_exact(&mut buf).await.ok()?; - if !buf.starts_with(&0x1DEB0075_u32.to_le_bytes()) { + if !buf.starts_with(&OXIDE_BOOT_MAGIC) { return None; } // see https://github.com/oxidecomputer/boot-image-tools/blob/main/src/diskimage.rs @@ -142,7 +143,7 @@ fn read_os_tarball_metadata_blocking( let Some(parent) = path.parent() else { continue }; if parent != "image" { continue; - }; + } let Some(extension) = path.extension() else { continue }; if extension != "txt" { continue; @@ -156,7 +157,7 @@ fn read_os_tarball_metadata_blocking( } let file_name = try_path!( Utf8PathBuf::try_from(file_name.to_owned()) - .map_err(|error| error.into_io_error()), + .map_err(camino::FromPathBufError::into_io_error), ReadFile, tarball_path ); diff --git a/lib/src/edit/sign.rs b/lib/src/edit/sign.rs index e47cdff..fcde750 100644 --- a/lib/src/edit/sign.rs +++ b/lib/src/edit/sign.rs @@ -20,6 +20,7 @@ use tough::schema::Root; use tough::schema::Signed; use crate::edit::Ed25519Key; +use crate::edit::OXIDE_BOOT_MAGIC; use crate::edit::source::FileSource; use crate::edit::source::Target; use crate::edit::source::TargetSource; @@ -32,6 +33,7 @@ pub(crate) const DEFAULT_VALIDITY: Duration = Duration::from_secs(60 * 60 * 24 * 7 /* 1 week */); #[derive(Debug)] +#[must_use] pub struct UnsignedRepository<'a> { targets: BTreeMap>, root: Option>, @@ -263,7 +265,7 @@ impl SignedRepository<'_> { } } writer.finish().await.map_err(|source| { - let archive_path = archive_path.map(|p| p.to_owned()); + let archive_path = archive_path.map(Utf8Path::to_owned); ErrorKind::WriteZip { source, archive_path }.into() }) } @@ -291,7 +293,7 @@ fn deflate_heuristic(buf: &[u8]) -> Compression { // ZIP archive, e.g. hubris archive. not necessarily compressed based on // this heuristic alone but in our case it's very likely. Compression::none() - } else if buf.starts_with(&0x1DEB0075_u32.to_le_bytes()) { + } else if buf.starts_with(&OXIDE_BOOT_MAGIC) { // oxide phase 2 OS image. images are zlib-compressed after the header // if the least-significant bit of the flags starting at byte 8 is set: if buf.get(8).is_some_and(|b| *b & 0x1 == 0x1) { diff --git a/lib/src/edit/source.rs b/lib/src/edit/source.rs index 9243762..e8a8b7d 100644 --- a/lib/src/edit/source.rs +++ b/lib/src/edit/source.rs @@ -6,7 +6,6 @@ use std::convert::Infallible; use std::pin::Pin; use bytes::Bytes; -use bytes::BytesMut; use camino::Utf8PathBuf; use futures_util::Stream; use futures_util::StreamExt; @@ -85,7 +84,7 @@ impl<'a> From> for TargetSource<'a> { #[derive(Debug, Clone)] pub(crate) struct BytesSource { bytes: Bytes, - fake_length: Option, + fake_length: Option, sha256: Option, } @@ -102,29 +101,30 @@ impl BytesSource { Ok(Self { bytes: s.into(), fake_length: None, sha256: None }) } - pub(crate) fn fake_padded(prefix: impl Into, length: usize) -> Self { + pub(crate) fn fake_padded(prefix: impl Into, length: u64) -> Self { Self { bytes: prefix.into(), fake_length: Some(length), sha256: None } } pub(crate) fn iter_bytes(&self) -> impl Iterator + 'static { - const CHUNK_SIZE: usize = 8192; - let mut bytes = self.bytes.clone(); let mut remaining = match self.fake_length { Some(length) => { - bytes.truncate(length); - length - bytes.len() + if let Ok(length) = usize::try_from(length) { + bytes.truncate(length); + } + length - u64::try_from(bytes.len()).expect("usize fits in u64") } None => 0, }; - let zero = BytesMut::zeroed(remaining.min(CHUNK_SIZE)).freeze(); std::iter::once(bytes).chain(std::iter::from_fn(move || { - if remaining > 0 { - let slice = zero.slice(..remaining.min(CHUNK_SIZE)); - remaining = remaining.saturating_sub(slice.len()); - Some(slice) - } else { + static ZERO: &[u8] = &[0; 8192]; + if remaining == 0 { None + } else { + let end = remaining.min(8192); + remaining -= end; + let end = usize::try_from(end).expect("8192 <= usize::MAX"); + Some(Bytes::from_static(&ZERO[..end])) } })) } @@ -135,8 +135,10 @@ impl BytesSource { stream::iter(self.iter_bytes().map(Ok)) } - pub(crate) fn length(&self) -> usize { - self.fake_length.unwrap_or_else(|| self.bytes.len()) + pub(crate) fn length(&self) -> u64 { + self.fake_length.unwrap_or_else(|| { + self.bytes.len().try_into().expect("usize fits in u64") + }) } async fn sha256(&mut self) -> ArtifactHash { @@ -153,11 +155,7 @@ impl BytesSource { pub(crate) async fn into_target(mut self) -> Target<'static> { Target { - length: self - .fake_length - .unwrap_or(self.bytes.len()) - .try_into() - .unwrap(), + length: self.length(), sha256: self.sha256().await.0.to_vec(), source: self.into(), } @@ -200,10 +198,11 @@ impl FileSource { .try_fold( (0u64, Sha256::new()), |(mut length, mut hasher), bytes| { - length += u64::try_from(bytes.len()).unwrap(); + length += + u64::try_from(bytes.len()).expect("usize fits in u64"); hasher.update(&bytes); if let Some(vec) = vec.as_mut() { - vec.extend_from_slice(&bytes) + vec.extend_from_slice(&bytes); } std::future::ready(Ok((length, hasher))) }, diff --git a/lib/src/edit/zone_image.rs b/lib/src/edit/zone_image.rs index 4884e7c..7b8c97a 100644 --- a/lib/src/edit/zone_image.rs +++ b/lib/src/edit/zone_image.rs @@ -28,20 +28,14 @@ use crate::error::try_path; impl Input> { pub(crate) async fn zone_image(path: Utf8PathBuf) -> Result { - let cloned_path = path.clone(); - let (file, layer_info) = tokio::task::spawn_blocking(move || { - let file = try_path!(std::fs::File::open(&path), ReadFile, path); - let mut archive = tar::Archive::new(GzDecoder::new(file)); - let layer_info = try_path!( - Metadata::read_from_tar(&mut archive) - .and_then(|metadata| metadata.layer_info().cloned()), - ReadZoneOxideJson, - path - ); - Ok::<_, Error>((archive.into_inner().into_inner(), layer_info)) - }) - .await??; - let source = FileSource::from_file(file.into(), cloned_path); + let file = + try_path!(tokio::fs::File::open(&path).await, OpenFile, path); + let (file, layer_info) = crate::util::read_zone_layer_info( + file.into_std().await, + path.clone(), + ) + .await?; + let source = FileSource::from_file(file.into(), path); Ok(Self::Zone { source: source.into(), tags: ZoneTags { zone_name: layer_info.pkg }, @@ -49,7 +43,8 @@ impl Input> { }) } - pub(crate) async fn guess_zone_image(input: GuessInput) -> GuessResult { + #[expect(clippy::unnecessary_wraps)] + pub(crate) fn guess_zone_image(input: GuessInput) -> GuessResult { // `oxide.json` is the first file of a zone image and is relatively // small, so it should be contained entirely within the first 4K of the // compressed tarball. diff --git a/lib/src/loader.rs b/lib/src/loader.rs index 3deeb8e..57ad22e 100644 --- a/lib/src/loader.rs +++ b/lib/src/loader.rs @@ -39,6 +39,7 @@ pub enum TrustStoreBehavior { } #[derive(Debug, Clone, Default)] +#[must_use] pub struct RepositoryLoader { compute_archive_sha256: bool, expiration_enforcement: ExpirationEnforcement, @@ -126,8 +127,16 @@ impl RepositoryLoader { } fn zip_base_urls(self) -> Self { - self.metadata_base_url("zip:///repo/metadata/".parse().unwrap()) - .targets_base_url("zip:///repo/targets/".parse().unwrap()) + self.metadata_base_url( + "zip:///repo/metadata/" + .parse() + .expect("`zip:///repo/metadata/` is a valid URL"), + ) + .targets_base_url( + "zip:///repo/targets/" + .parse() + .expect("`zip:///repo/targets/` is a valid URL"), + ) } /// Load a Tufaceous-generated ZIP archive from an owned buffer. diff --git a/lib/src/repo.rs b/lib/src/repo.rs index 96d0421..4c1c93f 100644 --- a/lib/src/repo.rs +++ b/lib/src/repo.rs @@ -103,10 +103,19 @@ impl Repository { read_target_json(&repo, ArtifactsSchema::TARGET_NAME).await? else { if v1_compatibility - && let Some(repo) = - v1::from_loaded(repo, log, trust_root).await? + && let Some((system_version, artifacts, v1_unpacked)) = + v1::from_loaded(&repo, log).await? { - return Ok(repo); + return Ok(Repository { + inner: repo, + trust_root, + archive_path, + archive_sha256, + system_version, + artifacts, + metadata: BTreeMap::new(), + v1_unpacked: Some(v1_unpacked), + }); } return Err(ErrorKind::TargetNotFound { diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs index 7dfa451..31ce5e9 100644 --- a/lib/src/repo/v1.rs +++ b/lib/src/repo/v1.rs @@ -2,7 +2,6 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use std::collections::BTreeMap; use std::collections::HashMap; use std::io::BufRead; use std::io::Read; @@ -19,6 +18,7 @@ use futures_util::Stream; use futures_util::TryStreamExt; use futures_util::pin_mut; use futures_util::stream; +use hubtools::Caboose; use hubtools::RawHubrisArchive; use rawzip::FileReader; use rawzip::RangeReader; @@ -41,13 +41,14 @@ use tufaceous_artifact::OsPhase2Tags; use tufaceous_artifact::OsVariant; use tufaceous_artifact::RotSlot; use tufaceous_artifact::ZoneTags; +use tufaceous_artifact::hubris::ReadCabooseError; use crate::COSMO_PHASE_1_PATH; use crate::GIMLET_PHASE_1_PATH; use crate::PHASE_2_PATH; -use crate::Repository; use crate::error::Error; use crate::error::ErrorKind; +use crate::error::try_path; use crate::repo::read_target; use crate::repo::read_target_json; use crate::repo::read_target_vec; @@ -59,230 +60,108 @@ pub(crate) struct Unpacked { } pub(crate) async fn from_loaded( - repo: tough::Repository, + repo: &tough::Repository, log: &Logger, - trust_root: Vec, -) -> Result, Error> { +) -> Result, Error> { let Some(V1ArtifactsSchema { system_version, artifacts: v1_artifacts }) = - read_target_json(&repo, V1ArtifactsSchema::TARGET_NAME).await? + read_target_json(repo, V1ArtifactsSchema::TARGET_NAME).await? else { return Ok(None); }; let mut artifacts = Artifacts::default(); - let mut entries = HashMap::new(); - for artifact in v1_artifacts { - let kind = match artifact.kind { + let mut unpacked = Unpacked { entries: HashMap::new() }; + for V1Artifact { version, kind, target } in v1_artifacts { + let Some((hash, length)) = sha256_length(repo, log, &target) else { + continue; + }; + let kind = match kind { V1ArtifactKind::Known(kind) => kind, V1ArtifactKind::Unknown(kind) => { warn!( log, "skipping artifact"; - "target_name" => &artifact.target, + "target_name" => &target, "error" => "unknown v1 kind", "kind" => kind, ); continue; } }; - let Some((hash, length)) = sha256_length(&repo, log, &artifact.target) - else { - continue; - }; - - match kind { + let tags = match kind { V1KnownArtifactKind::GimletSp | V1KnownArtifactKind::PscSp | V1KnownArtifactKind::SwitchSp => { - let Some(image) = - read_target_vec(&repo, &artifact.target).await? - else { - continue; - }; - let caboose = RawHubrisArchive::from_vec(image) - .and_then(|image| image.read_caboose()) - .map_err(|source| ErrorKind::ReadHubrisArchive { - source, - path: artifact.target.clone().into(), - })?; - let tags = KnownArtifactTags::from_sp_caboose(&caboose) - .map_err(|source| ErrorKind::ReadCaboose { - source, - path: artifact.target.clone().into(), - })?; - artifacts.insert(Artifact { - target_name: artifact.target, - version: artifact.version, - tags: tags.to_tags(), - hash, - length, - }); + let image = read_target_vec(repo, &target).await?; + let Some(image) = image else { continue }; + caboose_tags( + image, + &target, + KnownArtifactTags::from_sp_caboose, + )? } V1KnownArtifactKind::GimletRotBootloader | V1KnownArtifactKind::PscRotBootloader | V1KnownArtifactKind::SwitchRotBootloader => { - let Some(image) = - read_target_vec(&repo, &artifact.target).await? - else { - continue; - }; - let caboose = RawHubrisArchive::from_vec(image) - .and_then(|image| image.read_caboose()) - .map_err(|source| ErrorKind::ReadHubrisArchive { - source, - path: artifact.target.clone().into(), - })?; - let tags = - KnownArtifactTags::from_rot_bootloader_caboose(&caboose) - .map_err(|source| ErrorKind::ReadCaboose { - source, - path: artifact.target.clone().into(), - })?; - artifacts.insert(Artifact { - target_name: artifact.target, - version: artifact.version, - tags: tags.to_tags(), - hash, - length, - }); + let image = read_target_vec(repo, &target).await?; + let Some(image) = image else { continue }; + caboose_tags( + image, + &target, + KnownArtifactTags::from_rot_bootloader_caboose, + )? } - V1KnownArtifactKind::GimletRot | V1KnownArtifactKind::PscRot | V1KnownArtifactKind::SwitchRot => { - let target = artifact.target.clone(); - let unpacked = - CompositeArtifact::unpack(&repo, &target).await?; - for (tar_path, inner) in unpacked.entries { - let UnpackedArtifact { file, hash, length } = inner; - let slot = match tar_path.as_ref() { - "archive-a.zip" => RotSlot::A, - "archive-b.zip" => RotSlot::B, - _ => continue, - }; - let mut reader = RangeReader::new(file.clone(), 0..length); - let mut vec = length - .try_into() - .map(Vec::with_capacity) - .unwrap_or_default(); - let image = tokio::task::spawn_blocking(move || { - reader.read_to_end(&mut vec).map(|_| vec).map_err( - |source| ErrorKind::ReadFile { source, path: None }, - ) - }) - .await??; - let caboose = RawHubrisArchive::from_vec(image) - .and_then(|image| image.read_caboose()) - .map_err(|source| ErrorKind::ReadHubrisArchive { - source, - path: target.clone().into(), - })?; - let tags = - KnownArtifactTags::from_rot_caboose(&caboose, slot) - .map_err(|source| ErrorKind::ReadCaboose { - source, - path: target.clone().into(), - })?; - let target_name = format!("{}/{tar_path}", artifact.target); - entries.insert( - target_name.clone(), - UnpackedArtifact { file, hash, length }, - ); - artifacts.insert(Artifact { - target_name, - version: artifact.version.clone(), - tags: tags.to_tags(), - hash, - length, - }); - } + CompositeArtifact::unpack(repo, target) + .await? + .read_rot(&mut artifacts, &mut unpacked, version) + .await?; + continue; } V1KnownArtifactKind::Host => { - unpack_os( - CompositeArtifact::unpack(&repo, &artifact.target).await?, - &mut entries, + CompositeArtifact::unpack(repo, target).await?.read_os_image( &mut artifacts, - &artifact, + &mut unpacked, OsVariant::Host, + &version, ); + continue; } V1KnownArtifactKind::Trampoline => { - unpack_os( - CompositeArtifact::unpack(&repo, &artifact.target).await?, - &mut entries, + CompositeArtifact::unpack(repo, target).await?.read_os_image( &mut artifacts, - &artifact, + &mut unpacked, OsVariant::Recovery, + &version, ); + continue; } V1KnownArtifactKind::InstallinatorDocument => { - artifacts.insert(Artifact { - target_name: artifact.target, - version: artifact.version, - tags: KnownArtifactTags::InstallinatorDocument {}.to_tags(), - hash, - length, - }); + KnownArtifactTags::InstallinatorDocument } V1KnownArtifactKind::ControlPlane => { - let unpacked = - CompositeArtifact::unpack(&repo, &artifact.target).await?; - for (tar_path, inner) in unpacked.entries { - let UnpackedArtifact { file, hash, length } = inner; - if tar_path.starts_with("zones/") { - let path = tar_path.to_string(); - let (file, layer_info) = - crate::util::read_zone_layer_info( - RangeReader::new(file, 0..length), - path.into(), - ) - .await?; - let file = file.into_inner(); - let target_name = - format!("{}/{tar_path}", artifact.target); - entries.insert( - target_name.clone(), - UnpackedArtifact { file, hash, length }, - ); - artifacts.insert(Artifact { - target_name, - version: layer_info.version, - tags: KnownArtifactTags::Zone(ZoneTags { - zone_name: layer_info.pkg, - }) - .to_tags(), - hash, - length, - }); - } - } + CompositeArtifact::unpack(repo, target) + .await? + .read_control_plane(&mut artifacts, &mut unpacked) + .await?; + continue; } V1KnownArtifactKind::MeasurementCorpus => { - artifacts.insert(Artifact { - target_name: artifact.target, - version: artifact.version, - tags: KnownArtifactTags::MeasurementCorpus {}.to_tags(), - hash, - length, - }); + KnownArtifactTags::MeasurementCorpus } - } - } + }; - Ok(Some(Repository { - inner: repo, - system_version, - trust_root, - archive_path: None, - archive_sha256: None, - artifacts: Artifacts::new(artifacts), - metadata: BTreeMap::new(), - v1_unpacked: Some(Unpacked { entries }), - })) + let target_name = target; + let tags = tags.to_tags(); + artifacts.insert(Artifact { target_name, version, tags, hash, length }); + } + Ok(Some((system_version, artifacts, unpacked))) } #[derive(Debug, Clone)] @@ -330,32 +209,46 @@ impl UnpackedArtifact { ); } hasher.update(&bytes); - bytes_read += u64::try_from(bytes.len()).unwrap(); + bytes_read += + u64::try_from(bytes.len()).expect("usize fits in u64"); Ok(Some((bytes, (buf, hasher, bytes_read)))) }, ) } } +fn caboose_tags( + image: Vec, + target_name: &str, + f: impl FnOnce(&Caboose) -> Result, +) -> Result { + let caboose = try_path!( + RawHubrisArchive::from_vec(image) + .and_then(|image| image.read_caboose()), + ReadHubrisArchive, + target_name + ); + Ok(try_path!(f(&caboose), ReadCaboose, target_name)) +} + #[derive(Debug)] struct CompositeArtifact { entries: HashMap, + original_target_name: String, } impl CompositeArtifact { async fn unpack( repo: &tough::Repository, - target_name: &str, + target_name: String, ) -> Result { let stream = - read_target(repo, target_name).await?.ok_or_else(|| { - ErrorKind::TargetNotFound { - target_name: target_name.to_owned(), - } + read_target(repo, &target_name).await?.ok_or_else(|| { + ErrorKind::TargetNotFound { target_name: target_name.clone() } })?; pin_mut!(stream); let (tx, rx) = mpsc::channel(1); - let target_name = target_name.to_owned(); + let target_name = target_name.clone(); let task = tokio::task::spawn_blocking(move || { let mut archive = tar::Archive::new(GzDecoder::new(MpscReader::new(rx))); @@ -393,56 +286,138 @@ impl CompositeArtifact { ErrorKind::WriteFile { source, path: None } })?; hasher.update(&buf[..n]); - length += u64::try_from(n).unwrap(); + length += u64::try_from(n).expect("usize fits in u64"); } let file = Arc::new(file.into()); let hash = ArtifactHash(hasher.finalize().into()); entries.insert(path, UnpackedArtifact { file, hash, length }); } - Ok(Self { entries }) + Ok(Self { entries, original_target_name: target_name }) }); while let Some(item) = stream.try_next().await? { let Ok(()) = tx.send(item).await else { break }; } task.await? } -} -fn unpack_os( - unpacked: CompositeArtifact, - entries: &mut HashMap, - artifacts: &mut Artifacts, - artifact: &V1Artifact, - os_variant: OsVariant, -) { - for (tar_path, inner) in unpacked.entries { - let tags = match tar_path.as_str().strip_prefix("image/") { - Some(COSMO_PHASE_1_PATH) => { + async fn read_rot( + mut self, + artifacts: &mut Artifacts, + unpacked: &mut Unpacked, + version: ArtifactVersion, + ) -> Result<(), Error> { + for slot in [RotSlot::A, RotSlot::B] { + let path = Utf8PathBuf::from(format!("archive-{slot}.zip")); + let Some(UnpackedArtifact { file, hash, length }) = + self.entries.remove(&path) + else { + continue; + }; + + let mut reader = RangeReader::new(file.clone(), 0..length); + let image = tokio::task::spawn_blocking(move || { + let capacity = usize::try_from(length).unwrap_or_default(); + let mut vec = Vec::with_capacity(capacity); + reader.read_to_end(&mut vec).map(|_| vec).map_err(|source| { + ErrorKind::ReadFile { source, path: None } + }) + }) + .await??; + + let target_name = format!("{}/{path}", self.original_target_name); + let tags = caboose_tags(image, &target_name, |caboose| { + KnownArtifactTags::from_rot_caboose(caboose, slot) + })?; + artifacts.insert(Artifact { + target_name: target_name.clone(), + version: version.clone(), + tags: tags.to_tags(), + hash, + length, + }); + unpacked + .entries + .insert(target_name, UnpackedArtifact { file, hash, length }); + } + Ok(()) + } + + fn read_os_image( + mut self, + artifacts: &mut Artifacts, + unpacked: &mut Unpacked, + os_variant: OsVariant, + version: &ArtifactVersion, + ) { + for (file_name, tags) in [ + ( + COSMO_PHASE_1_PATH, KnownArtifactTags::OsPhase1(OsPhase1Tags { os_variant, os_board: OsBoard::Cosmo, - }) - } - Some(GIMLET_PHASE_1_PATH) => { + }), + ), + ( + GIMLET_PHASE_1_PATH, KnownArtifactTags::OsPhase1(OsPhase1Tags { os_variant, os_board: OsBoard::Gimlet, - }) - } - Some(PHASE_2_PATH) => { - KnownArtifactTags::OsPhase2(OsPhase2Tags { os_variant }) + }), + ), + ( + PHASE_2_PATH, + KnownArtifactTags::OsPhase2(OsPhase2Tags { os_variant }), + ), + ] { + let path = Utf8PathBuf::from(format!("image/{file_name}")); + let Some(entry) = self.entries.remove(&path) else { + continue; + }; + let target_name = format!("{}/{path}", self.original_target_name); + artifacts.insert(Artifact { + target_name: target_name.clone(), + version: version.clone(), + tags: tags.to_tags(), + hash: entry.hash, + length: entry.length, + }); + unpacked.entries.insert(target_name, entry); + } + } + + async fn read_control_plane( + self, + artifacts: &mut Artifacts, + unpacked: &mut Unpacked, + ) -> Result<(), Error> { + for (tar_path, UnpackedArtifact { file, hash, length }) in self.entries + { + if !tar_path.starts_with("zones") { + continue; } - _ => continue, - }; - let target_name = format!("{}/{tar_path}", artifact.target); - artifacts.insert(Artifact { - target_name: target_name.clone(), - version: artifact.version.clone(), - tags: tags.to_tags(), - hash: inner.hash, - length: inner.length, - }); - entries.insert(target_name, inner); + let target_name = + format!("{}/{tar_path}", self.original_target_name); + let (file, layer_info) = crate::util::read_zone_layer_info( + RangeReader::new(file, 0..length), + target_name.clone().into(), + ) + .await?; + let file = file.into_inner(); + artifacts.insert(Artifact { + target_name: target_name.clone(), + version: layer_info.version, + tags: KnownArtifactTags::Zone(ZoneTags { + zone_name: layer_info.pkg, + }) + .to_tags(), + hash, + length, + }); + unpacked + .entries + .insert(target_name, UnpackedArtifact { file, hash, length }); + } + Ok(()) } } diff --git a/lib/src/util.rs b/lib/src/util.rs index 14c9f42..681abbc 100644 --- a/lib/src/util.rs +++ b/lib/src/util.rs @@ -67,6 +67,7 @@ pub(crate) struct DirEntry { } impl DirEntry { + #[expect(clippy::needless_pass_by_value)] fn new(inner: tokio::fs::DirEntry) -> Result { let path = inner.path().try_into().map_err(FromPathBufError::into_io_error)?; @@ -84,6 +85,6 @@ impl DirEntry { pub(crate) fn file_name(&self) -> &str { self.path .file_name() - .expect("path created through DirEntry must have a filename") + .expect("path created through DirEntry must have a file name") } } diff --git a/lib/src/zip_transport.rs b/lib/src/zip_transport.rs index b194ddb..daca627 100644 --- a/lib/src/zip_transport.rs +++ b/lib/src/zip_transport.rs @@ -39,7 +39,7 @@ use url::Url; use crate::error::Error; use crate::error::ErrorKind; -const MAX_SYMLINK_TARGET_LEN: usize = 1024; +const MAX_SYMLINK_TARGET_LEN: usize = 1024 - 1; const MAX_SYMLINK_TRAVERSAL: usize = 8; /// Implementation of [`tough::Transport`] that operates on a Zip archive. @@ -278,10 +278,15 @@ impl ZipTransport { ) -> Result { let target = tokio::task::spawn_blocking(move || { let mut reader = self.reader_blocking(entry_data)?; - let mut v = vec![0; MAX_SYMLINK_TARGET_LEN]; + let mut v = vec![0; MAX_SYMLINK_TARGET_LEN + 1]; let len = - std::io::copy(&mut reader, &mut Cursor::new(v.as_mut_slice()))?; - v.truncate(len.try_into().unwrap()); + std::io::copy(&mut reader, &mut Cursor::new(v.as_mut_slice()))? + .try_into() + .expect("MAX_SYMLINK_TARGET_LEN <= usize::MAX"); + if v.len() > MAX_SYMLINK_TARGET_LEN { + return Err(ZipTransportError::SymlinkTargetLengthLimit); + } + v.truncate(len); String::from_utf8(v).map_err(|source| { ZipTransportError::SymlinkUtf8(source.utf8_error()) }) @@ -391,6 +396,8 @@ pub enum ZipTransportError { FileNotFound, #[error("is a directory")] IsADirectory, + #[error("symlink target name exceeded byte limit")] + SymlinkTargetLengthLimit, #[error("reached symlink traversal limit")] SymlinkTraversalLimit, #[error("symlink target is not valid UTF-8")] @@ -409,6 +416,7 @@ impl ZipTransportError { | ZipTransportError::CompressionMethod(_) | ZipTransportError::Duplicate | ZipTransportError::IsADirectory + | ZipTransportError::SymlinkTargetLengthLimit | ZipTransportError::SymlinkTraversalLimit | ZipTransportError::SymlinkUtf8(_) => TransportErrorKind::Other, }; @@ -429,7 +437,7 @@ fn path_to_url(path: ZipFilePath>, log: &Logger) -> Option { .ok() .and_then(|path| { Url::parse("zip:///") - .unwrap() + .expect("`zip:///` is a valid URL") .join(path.as_str()) .inspect_err(|err| { warn!( diff --git a/lib/tests/tests.rs b/lib/tests/tests.rs index e30563b..c0a01c5 100644 --- a/lib/tests/tests.rs +++ b/lib/tests/tests.rs @@ -72,7 +72,7 @@ async fn empty_artifact() -> Result<(), Error> { .fake_artifact( "empty.img".to_owned(), "1.0.0".parse().unwrap(), - tufaceous_artifact::KnownArtifactTags::InstallinatorDocument {}, + &tufaceous_artifact::KnownArtifactTags::InstallinatorDocument, 0, ) .finish() From 1f329c4291a5e8d5a55f80631cdedb1df5fbe6da Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 27 Jan 2026 21:21:26 -0800 Subject: [PATCH 019/137] small nit --- bin/src/edit.rs | 5 ++--- lib/src/edit/input.rs | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/bin/src/edit.rs b/bin/src/edit.rs index 327a050..57121f9 100644 --- a/bin/src/edit.rs +++ b/bin/src/edit.rs @@ -43,9 +43,8 @@ impl Args { .generate_installinator_document(!self.no_installinator_document); // always remove an existing installinator document - for artifact in repo - .artifacts() - .get_all(KnownArtifactTags::InstallinatorDocument {}) + for artifact in + repo.artifacts().get_all(KnownArtifactTags::InstallinatorDocument) { editor = editor.remove_target(&artifact.target_name); } diff --git a/lib/src/edit/input.rs b/lib/src/edit/input.rs index d5c0f01..66aaa6a 100644 --- a/lib/src/edit/input.rs +++ b/lib/src/edit/input.rs @@ -73,7 +73,7 @@ impl Input { "measurements/{corim_id}-{}.cbor", hex::encode(sha256) ); - let tags = KnownArtifactTags::MeasurementCorpus {}; + let tags = KnownArtifactTags::MeasurementCorpus; vec![Output::new(target_name, version, tags, source)] } Input::OsImages { From 9a48652e508de16aac60976501b1cef9e9df5c05 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 27 Jan 2026 21:39:03 -0800 Subject: [PATCH 020/137] ugh clippy --- Cargo.toml | 2 +- clippy.toml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 clippy.toml diff --git a/Cargo.toml b/Cargo.toml index aa022d7..6a51ac6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -58,8 +58,8 @@ url = "2.5.4" [workspace.lints.clippy] pedantic = { level = "warn", priority = -1 } -must-use-candidate = "allow" missing-errors-doc = "allow" +must-use-candidate = "allow" [profile.dev.package.flate2] opt-level = 3 diff --git a/clippy.toml b/clippy.toml new file mode 100644 index 0000000..8a4e1a8 --- /dev/null +++ b/clippy.toml @@ -0,0 +1 @@ +doc-valid-idents = ["OpenAPI", ".."] From fb729bc09261061b2edfafacd256ce5bdf6d0cf7 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Wed, 28 Jan 2026 11:10:57 -0800 Subject: [PATCH 021/137] add a `Repository::verify_targets` method --- Cargo.lock | 1 + Cargo.toml | 1 + lib/Cargo.toml | 1 + lib/src/repo.rs | 75 ++++++++++++++++++++++++++++++++++++++++++++-- lib/tests/tests.rs | 38 +++++++++++++++++++++++ 5 files changed, 114 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 22344c2..9d483ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2843,6 +2843,7 @@ dependencies = [ "futures-util", "hex", "hubtools", + "memchr", "rats-corim", "rawzip", "semver", diff --git a/Cargo.toml b/Cargo.toml index 6a51ac6..2ebebbe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,7 @@ flate2 = "1.0.35" futures-util = "0.3.31" hex = "0.4.3" hubtools = { git = "https://github.com/oxidecomputer/hubtools.git", branch = "main" } +memchr = "2.7.4" proptest = "1.5.0" rats-corim = { git = "https://github.com/oxidecomputer/rats-corim.git", branch = "main" } rawzip = "0.4.2" diff --git a/lib/Cargo.toml b/lib/Cargo.toml index b275aba..adf94e7 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -34,6 +34,7 @@ tufaceous-brand-metadata.workspace = true url.workspace = true [dev-dependencies] +memchr.workspace = true tokio = { workspace = true, features = ["macros"] } tokio-test.workspace = true diff --git a/lib/src/repo.rs b/lib/src/repo.rs index 4c1c93f..7895065 100644 --- a/lib/src/repo.rs +++ b/lib/src/repo.rs @@ -6,7 +6,9 @@ mod v1; use std::collections::BTreeMap; use std::collections::HashMap; +use std::num::NonZero; use std::pin::Pin; +use std::sync::Arc; use bytes::Bytes; use camino::Utf8Path; @@ -17,6 +19,9 @@ use semver::Version; use serde::de::DeserializeOwned; use slog::Logger; use slog::warn; +use tokio::sync::Semaphore; +use tokio::sync::TryAcquireError; +use tokio::task::JoinSet; use tough::TargetName; use tough::schema::Hashes; use tough::schema::Target; @@ -186,10 +191,11 @@ impl Repository { pub async fn read_target<'a>( &'a self, target: &str, - ) -> Result> + use<'a>, Error> { + ) -> Result> + Send + use<'a>, Error> + { if let Some(stream) = read_target(&self.inner, target).await? { return Ok(Box::pin(stream) - as Pin>>>); + as Pin> + Send>>); } if let Some(unpacked) = &self.v1_unpacked @@ -200,6 +206,71 @@ impl Repository { Err(ErrorKind::TargetNotFound { target_name: target.to_owned() }.into()) } + + /// Reads all targets in the repository and verifies they have the correct + /// length and checksum. + /// + /// The repository must be wrapped in [`Arc`] to call this method. + /// + /// This method is *not* necessary to safely read the repository. All + /// streams returned from [`Repository::read_target`] and read completely + /// are verified to have the same correct length and checksum. + /// + /// However, this verification is useful to complete before using contents + /// for any operations if you have the entire repository available locally. + /// In the past, Tufaceous archives were used by completely unpacking the + /// archive, which verified the contents against the CRC-32 checksums in the + /// ZIP archive; this caught binaries that were corrupted in transit between + /// CI and destination hardware. + /// + /// `parallelism` controls how many targets are read at a time. Consider + /// using [`std::thread::available_parallelism`], or [`NonZero::::MIN`] + /// as a fallback or if you do not want parallelism. + #[expect(clippy::missing_panics_doc)] + pub async fn verify_targets( + self: &Arc, + parallelism: NonZero, + ) -> Result<(), Error> { + // This is a reimplementation of `parallel-task-set` from Omicron, + // and could potentially be replaced with that if it's published to + // crates.io. + let semaphore = Arc::new(Semaphore::new(parallelism.get())); + let mut set: JoinSet> = JoinSet::new(); + + for target_name in self.targets().keys() { + let target = target_name.raw().to_owned(); + let permit = match Arc::clone(&semaphore).try_acquire_owned() { + Ok(permit) => permit, + Err(TryAcquireError::Closed) => { + unreachable!("we never close the semaphore") + } + Err(TryAcquireError::NoPermits) => { + if let Some(result) = set.join_next().await { + result??; + } + Arc::clone(&semaphore) + .acquire_owned() + .await + .expect("we never close the semaphore") + } + }; + let this = Arc::clone(self); + set.spawn(async move { + let _permit = permit; + let mut stream = this.read_target(&target).await?; + // Read the stream to the end. There's no need to do anything + // with the data; the underlying stream performs verification. + while stream.try_next().await?.is_some() {} + Ok(()) + }); + } + + while let Some(result) = set.join_next().await { + result??; + } + + Ok(()) + } } async fn read_target<'a>( diff --git a/lib/tests/tests.rs b/lib/tests/tests.rs index c0a01c5..8907316 100644 --- a/lib/tests/tests.rs +++ b/lib/tests/tests.rs @@ -4,6 +4,8 @@ use std::convert::Infallible; use std::io::Write; +use std::num::NonZero; +use std::sync::Arc; use bytes::BufMut; use bytes::BytesMut; @@ -43,6 +45,42 @@ async fn it_works() -> Result<(), Error> { Ok(()) } +#[tokio::test] +async fn verify_targets() -> Result<(), Error> { + let log = slog::Logger::root(slog::Discard, slog::o!()); + let mut zip = RepositoryEditor::fake(VERSION)? + .finish() + .await? + .generate_root() + .sign() + .await? + .write_zip(Vec::new(), Utc::now()) + .await?; + let repo = RepositoryLoader::new() + .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .load_zip_buffer(zip.clone(), &log) + .await?; + let repo = Arc::new(repo); + let parallelism = + std::thread::available_parallelism().unwrap_or(NonZero::::MIN); + repo.verify_targets(parallelism).await?; + + // Now intentionally fuck up the archive, and ensure verification fails. + let pos = memchr::memmem::find(&zip, b"hubris") + .expect("b\"hubris\" not found in archive"); + zip[pos] = b'H'; // flip! that! bit! + let repo = RepositoryLoader::new() + .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .load_zip_buffer(zip.clone(), &log) + .await?; + let repo = Arc::new(repo); + let err = repo.verify_targets(parallelism).await.unwrap_err(); + // This error ultimately comes from `rawzip`'s CRC-32 checking. + assert!(err.to_string().contains("Invalid checksum")); + + Ok(()) +} + #[tokio::test] async fn no_artifacts() -> Result<(), Error> { let log = slog::Logger::root(slog::Discard, slog::o!()); From 846b468381e4c583ea11e98059a173c056f6a082 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 29 Jan 2026 13:38:26 -0800 Subject: [PATCH 022/137] ensure target streams impl `Sync` --- Cargo.lock | 6 ++---- Cargo.toml | 3 ++- lib/src/repo.rs | 9 +++++---- lib/src/zip_transport.rs | 7 ++----- 4 files changed, 11 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9d483ff..eb4ecbe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1560,8 +1560,7 @@ dependencies = [ [[package]] name = "olpc-cjson" version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "696183c9b5fe81a7715d074fd632e8bd46f4ccc0231a3ed7fc580a80de5f7083" +source = "git+https://github.com/iliana/tough.git?branch=transport-is-sync#47aff7d174914f46db6166dbade64124849f2468" dependencies = [ "serde", "serde_json", @@ -2797,8 +2796,7 @@ dependencies = [ [[package]] name = "tough" version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e88d0ee9525696569cc2af5d46f8a739028c0268895071e0386957195b0c9161" +source = "git+https://github.com/iliana/tough.git?branch=transport-is-sync#47aff7d174914f46db6166dbade64124849f2468" dependencies = [ "async-recursion", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 2ebebbe..7bc0c3d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,7 +51,8 @@ thiserror = "2.0.12" tokio = "1.47.0" tokio-test = "0.4.5" tokio-util = "0.7.15" -tough = "0.21.0" +# https://github.com/awslabs/tough/pull/917 +tough = { git = "https://github.com/iliana/tough.git", branch = "transport-is-sync" } tufaceous.path = "lib" tufaceous-artifact.path = "artifact" tufaceous-brand-metadata.path = "brand-metadata" diff --git a/lib/src/repo.rs b/lib/src/repo.rs index 7895065..3baf8ad 100644 --- a/lib/src/repo.rs +++ b/lib/src/repo.rs @@ -35,6 +35,9 @@ use crate::error::ErrorKind; use crate::schema::ArtifactSchema; use crate::schema::ArtifactsSchema; +pub type TargetStream<'a> = + Pin> + Send + Sync + 'a>>; + /// A loaded TUF repository. #[derive(Debug, Clone)] pub struct Repository { @@ -191,11 +194,9 @@ impl Repository { pub async fn read_target<'a>( &'a self, target: &str, - ) -> Result> + Send + use<'a>, Error> - { + ) -> Result, Error> { if let Some(stream) = read_target(&self.inner, target).await? { - return Ok(Box::pin(stream) - as Pin> + Send>>); + return Ok(Box::pin(stream)); } if let Some(unpacked) = &self.v1_unpacked diff --git a/lib/src/zip_transport.rs b/lib/src/zip_transport.rs index daca627..546be84 100644 --- a/lib/src/zip_transport.rs +++ b/lib/src/zip_transport.rs @@ -7,7 +7,6 @@ use std::fmt::Debug; use std::fs::File; use std::io::Cursor; use std::io::Read; -use std::pin::Pin; use std::sync::Arc; use bytes::Bytes; @@ -33,6 +32,7 @@ use tokio::sync::mpsc; use tough::Transport; use tough::TransportError; use tough::TransportErrorKind; +use tough::TransportStream; use tough::async_trait; use url::Url; @@ -307,10 +307,7 @@ impl Transport async fn fetch( &self, original_url: Url, - ) -> Result< - Pin> + Send>>, - TransportError, - > { + ) -> Result { let mut url = original_url.clone(); if url.scheme() != "zip" { return Err(TransportError::new( From 8afcc64449dae5be2fa7afbdd56b36f1445e5ecd Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 3 Feb 2026 10:41:23 -0800 Subject: [PATCH 023/137] add ErrorKind::is_repository_error --- lib/src/error.rs | 165 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 149 insertions(+), 16 deletions(-) diff --git a/lib/src/error.rs b/lib/src/error.rs index f0c7368..fcefc9b 100644 --- a/lib/src/error.rs +++ b/lib/src/error.rs @@ -3,17 +3,43 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use std::convert::Infallible; +use std::error::Error as _; use std::fmt; use std::fmt::Debug; use std::fmt::Display; +use std::ops::Deref; use std::ops::Range; +use crate::ZipTransportError; use camino::Utf8PathBuf; +use tough::TransportErrorKind; + +macro_rules! try_path { + ($result:expr, $kind:ident, $path:expr) => { + match $result { + Ok(value) => value, + Err(source) => { + return Err( + ErrorKind::$kind { source, path: $path.into() }.into() + ) + } + } + }; +} +pub(crate) use try_path; #[derive(Debug, thiserror::Error)] #[error(transparent)] pub struct Error(pub Box); +impl Deref for Error { + type Target = ErrorKind; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + #[derive(Debug, thiserror::Error)] #[non_exhaustive] pub enum ErrorKind { @@ -22,7 +48,7 @@ pub enum ErrorKind { #[error(transparent)] Join(#[from] tokio::task::JoinError), #[error(transparent)] - Tough(#[from] tough::error::Error), + Tough(tough::error::Error), #[error("error while manipulating key")] ToughKey(#[source] Box), @@ -164,12 +190,134 @@ pub enum ErrorKind { ImportV1Repo, } +impl ErrorKind { + /// Returns `true` if the error is due to a problem loading or reading + /// a repository, where retrying the operation with the same input would + /// result in the same error. + /// + /// Note that errors that can return `true` here can also happen during + /// other operations, such as editing or signing repositories. + pub fn is_repository_error(&self) -> bool { + match self { + ErrorKind::ZipEntryCount { .. } + | ErrorKind::ZipOverlappingRanges { .. } + | ErrorKind::ZipPathMismatch { .. } + | ErrorKind::ZipRangeOverrun { .. } + | ErrorKind::StreamLimit { .. } + | ErrorKind::ReadHubrisArchive { .. } + | ErrorKind::ReadCaboose { .. } + | ErrorKind::MetadataBaseUrlUnset + | ErrorKind::TargetsBaseUrlUnset + | ErrorKind::UrlJoin { .. } + | ErrorKind::NoTrustRoots + | ErrorKind::TargetNotFound { .. } + | ErrorKind::ParseTargetJson { .. } + | ErrorKind::ArtifactVersion(_) + | ErrorKind::ReadCompositeArtifact { .. } + | ErrorKind::ReadZoneOxideJson { .. } => true, + + ErrorKind::Join(_) + | ErrorKind::ToughKey(_) + | ErrorKind::WriteZip { .. } + | ErrorKind::CreateTempDir(_) + | ErrorKind::CreateTempFile(_) + | ErrorKind::OpenFile { .. } + | ErrorKind::ReadDir { .. } + | ErrorKind::ReadFile { .. } + | ErrorKind::SeekFile { .. } + | ErrorKind::WriteFile { .. } + | ErrorKind::ReadStream(_) + | ErrorKind::GenerateFakeHubrisArchive(_) + | ErrorKind::GenerateFakeMeasurementCorpus(_) + | ErrorKind::SerializeFakeMeasurementCorpus(_) + | ErrorKind::GenerateFakeZoneImage(_) + | ErrorKind::Ed25519Generate + | ErrorKind::KeyId(_) + | ErrorKind::Corim { .. } + | ErrorKind::GuessArtifact { .. } + | ErrorKind::TargetNameCollision { .. } + | ErrorKind::SerializeArtifacts(_) + | ErrorKind::SerializeInstallinator(_) + | ErrorKind::NoSigningRoot + | ErrorKind::ParseSigningRoot(_) + | ErrorKind::ImportV1Repo => false, + + ErrorKind::Fetch(error) => { + // A transport error might be due to a broken repository (e.g. + // HTTP Not Found, ZIP CRC mismatch), but it might also be due + // to a retryable problem (e.g. HTTP timeout). We will try and + // classify errors we can introspect but otherwise we'll return + // false. + match error.kind() { + TransportErrorKind::UnsupportedUrlScheme + | TransportErrorKind::FileNotFound => return true, + _ => {} + } + if let Some(source) = error.source().and_then(|source| { + source.downcast_ref::() + }) { + return match source { + ZipTransportError::UrlJoin { .. } + | ZipTransportError::CompressionMethod(_) + | ZipTransportError::Duplicate + | ZipTransportError::FileNotFound + | ZipTransportError::IsADirectory + | ZipTransportError::SymlinkTargetLengthLimit + | ZipTransportError::SymlinkTraversalLimit + | ZipTransportError::SymlinkUtf8(_) => true, + + ZipTransportError::Io(_) + | ZipTransportError::Join(_) => false, + + ZipTransportError::Zip(source) => { + // All of rawzip's errors are related to broken zip + // files, except for IO errors. + !matches!(source.kind(), rawzip::ErrorKind::IO(_)) + } + }; + } + false + } + + ErrorKind::Tough(error) => { + // tough's errors are... tough to classify. We have a pretty + // simple heuristic: if any error in the source chain is + // `std::io::Error`, return false. Otherwise, return true. + for source in std::iter::successors(error.source(), |source| { + (*source).source() + }) { + if source.downcast_ref::().is_some() { + return false; + } + } + true + } + + ErrorKind::ReadZip { source, .. } => { + // All of rawzip's errors are related to broken zip files, + // except for IO errors. + !matches!(source.kind(), rawzip::ErrorKind::IO(_)) + } + } + } +} + impl> From for Error { fn from(kind: T) -> Self { Error(Box::new(kind.into())) } } +impl From for ErrorKind { + fn from(error: tough::error::Error) -> Self { + if let tough::error::Error::Transport { source, .. } = error { + ErrorKind::Fetch(source) + } else { + ErrorKind::Tough(error) + } + } +} + impl From for ErrorKind { fn from(error: Infallible) -> Self { match error {} @@ -199,21 +347,6 @@ impl Debug for DebugByteString<'_> { } } -macro_rules! try_path { - ($result:expr, $kind:ident, $path:expr) => { - match $result { - Ok(value) => value, - Err(source) => { - return Err( - ErrorKind::$kind { source, path: $path.into() }.into() - ) - } - } - }; -} - -pub(crate) use try_path; - #[cfg(test)] mod tests { use std::error::Error as _; From b487607b686c7b5dd13bfb08673472970f096bf3 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 3 Feb 2026 10:41:53 -0800 Subject: [PATCH 024/137] add ArtifactId and ArtifactHandle --- artifact/src/artifact.rs | 34 ++++++++++++++++++++++ lib/src/repo.rs | 61 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) diff --git a/artifact/src/artifact.rs b/artifact/src/artifact.rs index 9dab798..e639840 100644 --- a/artifact/src/artifact.rs +++ b/artifact/src/artifact.rs @@ -28,6 +28,11 @@ pub struct Artifact { } impl Artifact { + /// Clones this artifact's `version` and `tags` into an [`ArtifactId`]. + pub fn id(&self) -> ArtifactId { + ArtifactId { version: self.version.clone(), tags: self.tags.clone() } + } + pub fn known_tags(&self) -> Option { KnownArtifactTags::from_tags(&self.tags).ok() } @@ -53,6 +58,29 @@ impl Display for DisplayTags<'_> { } } +#[derive(Debug, Clone, Deserialize, Serialize)] +#[cfg_attr(any(test, feature = "schemars"), derive(schemars::JsonSchema))] +pub struct ArtifactId { + pub version: ArtifactVersion, + pub tags: BTreeMap, +} + +impl ArtifactId { + pub fn known_tags(&self) -> Option { + KnownArtifactTags::from_tags(&self.tags).ok() + } + + pub fn display_tags(&self) -> DisplayTags<'_> { + DisplayTags(&self.tags) + } +} + +impl From for ArtifactId { + fn from(artifact: Artifact) -> Self { + ArtifactId { version: artifact.version, tags: artifact.tags } + } +} + #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct Artifacts { inner: BTreeMap, BTreeSet>, @@ -94,6 +122,12 @@ impl Artifacts { self.inner.get(&Some(tags)).map(BTreeSet::iter).unwrap_or_default() } + pub fn contains(&self, artifact: &Artifact) -> bool { + self.inner + .get(&artifact.known_tags()) + .is_some_and(|set| set.contains(artifact)) + } + pub fn filter_tags( &self, mut predicate: impl FnMut(&KnownArtifactTags) -> bool, diff --git a/lib/src/repo.rs b/lib/src/repo.rs index 3baf8ad..4e6cafb 100644 --- a/lib/src/repo.rs +++ b/lib/src/repo.rs @@ -208,6 +208,47 @@ impl Repository { Err(ErrorKind::TargetNotFound { target_name: target.to_owned() }.into()) } + /// Returns an [`ArtifactHandle`] that can be used to read `artifact`. + /// + /// This simply clones `artifact` and `self` and keeps them together in a + /// struct as a convenience. + /// + /// The repository must be wrapped in [`Arc`] to call this method. + /// + /// # Errors + /// + /// Returns an error if `artifact` is not found in this repository. + pub fn artifact_handle( + self: &Arc, + artifact: &Artifact, + ) -> Result { + if self.artifacts.contains(artifact) { + Ok(ArtifactHandle { + artifact: artifact.clone(), + repo: Arc::clone(self), + }) + } else { + Err(ErrorKind::TargetNotFound { + target_name: artifact.target_name.clone(), + } + .into()) + } + } + + /// Returns an iterator of [`ArtifactHandle`]s for every artifact + /// in the repository. + /// + /// An artifact handle simply clones `artifact` and `self` and keeps them + /// together in a struct as a convenience. + /// + /// The repository must be wrapped in [`Arc`] to call this method. + pub fn handles(self: &Arc) -> impl Iterator { + self.artifacts.iter().map(|artifact| ArtifactHandle { + artifact: artifact.clone(), + repo: Arc::clone(self), + }) + } + /// Reads all targets in the repository and verifies they have the correct /// length and checksum. /// @@ -274,6 +315,26 @@ impl Repository { } } +#[derive(Debug, Clone)] +pub struct ArtifactHandle { + artifact: Artifact, + repo: Arc, +} + +impl ArtifactHandle { + pub fn artifact(&self) -> &Artifact { + &self.artifact + } + + pub fn into_artifact(self) -> Artifact { + self.artifact + } + + pub async fn stream(&self) -> Result, Error> { + self.repo.read_target(&self.artifact.target_name).await + } +} + async fn read_target<'a>( repo: &'a tough::Repository, target: &str, From e69319a46196e2f9fe68da3802b1c2495f2c6e0c Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 3 Feb 2026 14:19:45 -0800 Subject: [PATCH 025/137] i fought the borrow checker and won --- lib/src/repo.rs | 20 ++++++++++---------- lib/src/repo/v1.rs | 21 +++++++++++---------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/lib/src/repo.rs b/lib/src/repo.rs index 4e6cafb..332403d 100644 --- a/lib/src/repo.rs +++ b/lib/src/repo.rs @@ -35,8 +35,8 @@ use crate::error::ErrorKind; use crate::schema::ArtifactSchema; use crate::schema::ArtifactsSchema; -pub type TargetStream<'a> = - Pin> + Send + Sync + 'a>>; +pub type TargetStream = + Pin> + Send + Sync + 'static>>; /// A loaded TUF repository. #[derive(Debug, Clone)] @@ -191,16 +191,16 @@ impl Repository { self.v1_unpacked.is_some() } - pub async fn read_target<'a>( - &'a self, + pub async fn read_target( + &self, target: &str, - ) -> Result, Error> { + ) -> Result { if let Some(stream) = read_target(&self.inner, target).await? { return Ok(Box::pin(stream)); } if let Some(unpacked) = &self.v1_unpacked - && let Some(entry) = unpacked.entries.get(target) + && let Some(entry) = unpacked.entries.get(target).cloned() { return Ok(Box::pin(entry.stream())); } @@ -330,15 +330,15 @@ impl ArtifactHandle { self.artifact } - pub async fn stream(&self) -> Result, Error> { + pub async fn stream(&self) -> Result { self.repo.read_target(&self.artifact.target_name).await } } -async fn read_target<'a>( - repo: &'a tough::Repository, +async fn read_target( + repo: &tough::Repository, target: &str, -) -> Result> + use<'a>>, Error> { +) -> Result> + 'static>, Error> { let target_name = target.parse()?; // Ensure the target is in the top-level targets.json role and not a // delegated target; we don't permit the use of delegated targets in diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs index 31ce5e9..e2859c5 100644 --- a/lib/src/repo/v1.rs +++ b/lib/src/repo/v1.rs @@ -172,30 +172,31 @@ pub(crate) struct UnpackedArtifact { } impl UnpackedArtifact { - pub(crate) fn stream(&self) -> impl Stream> { + pub(crate) fn stream( + self, + ) -> impl Stream> + 'static { stream::try_unfold( - (BytesMut::new(), Sha256::new(), 0), - async |(mut buf, mut hasher, mut bytes_read)| { + (self, BytesMut::new(), Sha256::new(), 0), + async |(this, mut buf, mut hasher, mut bytes_read)| { if buf.capacity() == 0 { buf.reserve(8192); } buf.resize(buf.capacity(), 0); - let file = self.file.clone(); - let mut buf = tokio::task::spawn_blocking(move || { - let n = file.read_at(&mut buf, bytes_read).map_err( + let (this, mut buf) = tokio::task::spawn_blocking(move || { + let n = this.file.read_at(&mut buf, bytes_read).map_err( |source| ErrorKind::ReadFile { source, path: None }, )?; buf.truncate(n); - Ok::<_, Error>(buf) + Ok::<_, Error>((this, buf)) }) .await??; let bytes = buf.split().freeze(); if bytes.is_empty() { - let msg = if self.hash + let msg = if this.hash != ArtifactHash(hasher.finalize().into()) { "invalid checksum" - } else if self.length != bytes_read { + } else if this.length != bytes_read { "invalid length" } else { return Ok(None); @@ -211,7 +212,7 @@ impl UnpackedArtifact { hasher.update(&bytes); bytes_read += u64::try_from(bytes.len()).expect("usize fits in u64"); - Ok(Some((bytes, (buf, hasher, bytes_read)))) + Ok(Some((bytes, (this, buf, hasher, bytes_read)))) }, ) } From 3d4876047a3730814b80a95db994698473b2445f Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 3 Feb 2026 15:00:25 -0800 Subject: [PATCH 026/137] create `Root` struct --- lib/src/edit/root.rs | 44 ++++++++++++++++++++++++++++++++++++++---- lib/src/error.rs | 3 +++ lib/src/repo.rs | 46 ++++++++++++++++++-------------------------- 3 files changed, 62 insertions(+), 31 deletions(-) diff --git a/lib/src/edit/root.rs b/lib/src/edit/root.rs index eebcae0..ed5dee0 100644 --- a/lib/src/edit/root.rs +++ b/lib/src/edit/root.rs @@ -8,21 +8,42 @@ use std::num::NonZero; use aws_lc_rs::rand::SystemRandom; use chrono::DateTime; use chrono::Utc; +use serde::Deserialize; +use serde::Serialize; use tough::editor::signed::SignedRole; use tough::key_source::KeySource; +use tough::schema; use tough::schema::KeyHolder; use tough::schema::RoleKeys; use tough::schema::RoleType; -use tough::schema::Root; +use tough::schema::Signed; use crate::error::Error; use crate::error::ErrorKind; -pub async fn generate_root( +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] +#[serde(transparent)] +pub struct Root(Signed); + +impl Root { + pub async fn generate( + keys: &[Box], + expires: DateTime, + ) -> Result { + Ok(Self(generate_root(keys, expires).await?.signed().clone())) + } + + pub fn verify_self_signed(&self) -> Result<(), Error> { + self.0.signed.verify_role(&self.0).map_err(ErrorKind::RoleVerify)?; + Ok(()) + } +} + +pub(crate) async fn generate_root( keys: &[Box], expires: DateTime, -) -> Result, Error> { - let mut root = Root { +) -> Result, Error> { + let mut root = schema::Root { spec_version: "1.0.0".into(), consistent_snapshot: false, version: NonZero::::MIN, @@ -60,3 +81,18 @@ pub async fn generate_root( ) .await?) } + +#[cfg(test)] +mod tests { + use chrono::Utc; + + use crate::edit::Ed25519Key; + use crate::edit::Root; + + #[tokio::test] + async fn generate_verify() { + let key = Ed25519Key::generate().unwrap(); + let root = Root::generate(&[Box::new(key)], Utc::now()).await.unwrap(); + root.verify_self_signed().unwrap(); + } +} diff --git a/lib/src/error.rs b/lib/src/error.rs index fcefc9b..d2a4e6d 100644 --- a/lib/src/error.rs +++ b/lib/src/error.rs @@ -151,6 +151,8 @@ pub enum ErrorKind { Ed25519Generate, #[error("failed to calculate TUF key ID")] KeyId(#[source] tough::schema::Error), + #[error("role verification failed")] + RoleVerify(#[source] tough::schema::Error), #[error("metadata base URL unset")] MetadataBaseUrlUnset, @@ -233,6 +235,7 @@ impl ErrorKind { | ErrorKind::GenerateFakeZoneImage(_) | ErrorKind::Ed25519Generate | ErrorKind::KeyId(_) + | ErrorKind::RoleVerify(_) | ErrorKind::Corim { .. } | ErrorKind::GuessArtifact { .. } | ErrorKind::TargetNameCollision { .. } diff --git a/lib/src/repo.rs b/lib/src/repo.rs index 332403d..cb22820 100644 --- a/lib/src/repo.rs +++ b/lib/src/repo.rs @@ -28,6 +28,8 @@ use tough::schema::Target; use tufaceous_artifact::Artifact; use tufaceous_artifact::ArtifactHash; use tufaceous_artifact::Artifacts; +use tufaceous_artifact::GetError; +use tufaceous_artifact::KnownArtifactTags; use crate::RepositoryLoader; use crate::error::Error; @@ -208,45 +210,35 @@ impl Repository { Err(ErrorKind::TargetNotFound { target_name: target.to_owned() }.into()) } - /// Returns an [`ArtifactHandle`] that can be used to read `artifact`. + /// Returns an [`ArtifactHandle`] for the one (and only one) artifact + /// matching `tags` that can be used to stream the artifact. /// - /// This simply clones `artifact` and `self` and keeps them together in a - /// struct as a convenience. - /// - /// The repository must be wrapped in [`Arc`] to call this method. + /// An artifact handle simply clones `artifact` and `self` and keeps them + /// together in a struct as a convenience. The repository must be wrapped in + /// [`Arc`] to call this method. /// /// # Errors /// - /// Returns an error if `artifact` is not found in this repository. - pub fn artifact_handle( + /// Returns an error if there is not exactly one artifact matching `tags`. + pub fn get_handle( self: &Arc, - artifact: &Artifact, - ) -> Result { - if self.artifacts.contains(artifact) { - Ok(ArtifactHandle { - artifact: artifact.clone(), - repo: Arc::clone(self), - }) - } else { - Err(ErrorKind::TargetNotFound { - target_name: artifact.target_name.clone(), - } - .into()) - } + tags: KnownArtifactTags, + ) -> Result { + let artifact = self.artifacts.get(tags)?.clone(); + Ok(ArtifactHandle { artifact, repo: Arc::clone(self) }) } /// Returns an iterator of [`ArtifactHandle`]s for every artifact /// in the repository. /// /// An artifact handle simply clones `artifact` and `self` and keeps them - /// together in a struct as a convenience. - /// - /// The repository must be wrapped in [`Arc`] to call this method. + /// together in a struct as a convenience. The repository must be wrapped in + /// [`Arc`] to call this method. pub fn handles(self: &Arc) -> impl Iterator { - self.artifacts.iter().map(|artifact| ArtifactHandle { - artifact: artifact.clone(), - repo: Arc::clone(self), - }) + self.artifacts + .iter() + .cloned() + .map(|artifact| ArtifactHandle { artifact, repo: Arc::clone(self) }) } /// Reads all targets in the repository and verifies they have the correct From 6225be429b0e3ca71058fac0e545edf42ab18c5d Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 3 Feb 2026 20:15:51 -0800 Subject: [PATCH 027/137] various tweaks --- artifact/src/artifact.rs | 7 ++++--- artifact/src/tags.rs | 8 +++++++- lib/src/edit/fake.rs | 16 +++++++++++++++- lib/src/edit/input.rs | 4 ++-- lib/src/repo.rs | 14 +++++--------- lib/tests/tests.rs | 4 ++-- 6 files changed, 35 insertions(+), 18 deletions(-) diff --git a/artifact/src/artifact.rs b/artifact/src/artifact.rs index e639840..27ea028 100644 --- a/artifact/src/artifact.rs +++ b/artifact/src/artifact.rs @@ -2,6 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use std::borrow::Cow; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::collections::btree_map::IntoValues; @@ -38,12 +39,12 @@ impl Artifact { } pub fn display_tags(&self) -> DisplayTags<'_> { - DisplayTags(&self.tags) + DisplayTags(Cow::Borrowed(&self.tags)) } } #[derive(Debug, Clone)] -pub struct DisplayTags<'a>(&'a BTreeMap); +pub struct DisplayTags<'a>(pub(crate) Cow<'a, BTreeMap>); impl Display for DisplayTags<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -71,7 +72,7 @@ impl ArtifactId { } pub fn display_tags(&self) -> DisplayTags<'_> { - DisplayTags(&self.tags) + DisplayTags(Cow::Borrowed(&self.tags)) } } diff --git a/artifact/src/tags.rs b/artifact/src/tags.rs index bfe8949..e277482 100644 --- a/artifact/src/tags.rs +++ b/artifact/src/tags.rs @@ -2,12 +2,14 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use std::borrow::Cow; use std::collections::BTreeMap; use serde::Deserialize; use serde::Serialize; use serde::de::value::MapDeserializer; +use crate::DisplayTags; use crate::InstallinatorArtifactKind; use crate::Sign; @@ -46,6 +48,10 @@ pub enum KnownArtifactTags { } impl KnownArtifactTags { + pub fn display(&self) -> DisplayTags<'static> { + DisplayTags(Cow::Owned(self.to_tags())) + } + pub fn from_tags( tags: &BTreeMap, ) -> Result { @@ -102,8 +108,8 @@ macro_rules! display_serialize { )] #[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] pub struct OsPhase1Tags { - pub os_variant: OsVariant, pub os_board: OsBoard, + pub os_variant: OsVariant, } impl From for KnownArtifactTags { diff --git a/lib/src/edit/fake.rs b/lib/src/edit/fake.rs index ddb4f70..b17e851 100644 --- a/lib/src/edit/fake.rs +++ b/lib/src/edit/fake.rs @@ -17,6 +17,20 @@ use crate::edit::source::BytesSource; use crate::error::Error; use crate::schema::ArtifactSchema; +const FAKE_ZONES: [&str; 11] = [ + "clickhouse", + "clickhouse_keeper", + "clickhouse_server", + "cockroachdb", + "crucible-zone", + "crucible-pantry-zone", + "external-dns", + "internal-dns", + "ntp", + "nexus", + "oximeter", +]; + pub trait ArtifactsExt: Sized { fn fake(version: ArtifactVersion) -> Result; } @@ -82,7 +96,7 @@ impl Input { version.clone(), )?); } - for name in ["zone1", "zone2"] { + for name in FAKE_ZONES { inputs.push(Self::fake_zone_image(name.into(), version.clone())?); } Ok(inputs) diff --git a/lib/src/edit/input.rs b/lib/src/edit/input.rs index 66aaa6a..c5565d7 100644 --- a/lib/src/edit/input.rs +++ b/lib/src/edit/input.rs @@ -89,14 +89,14 @@ impl Input { vec.push(Output::new( base.join("image").join(COSMO_PHASE_1_PATH).into(), version.clone(), - OsPhase1Tags { os_variant, os_board: OsBoard::Cosmo } + OsPhase1Tags { os_board: OsBoard::Cosmo, os_variant } .into(), cosmo_phase_1, )); vec.push(Output::new( base.join("image").join(GIMLET_PHASE_1_PATH).into(), version.clone(), - OsPhase1Tags { os_variant, os_board: OsBoard::Gimlet } + OsPhase1Tags { os_board: OsBoard::Gimlet, os_variant } .into(), gimlet_phase_1, )); diff --git a/lib/src/repo.rs b/lib/src/repo.rs index cb22820..ea3d03e 100644 --- a/lib/src/repo.rs +++ b/lib/src/repo.rs @@ -6,7 +6,6 @@ mod v1; use std::collections::BTreeMap; use std::collections::HashMap; -use std::num::NonZero; use std::pin::Pin; use std::sync::Arc; @@ -257,18 +256,16 @@ impl Repository { /// ZIP archive; this caught binaries that were corrupted in transit between /// CI and destination hardware. /// - /// `parallelism` controls how many targets are read at a time. Consider - /// using [`std::thread::available_parallelism`], or [`NonZero::::MIN`] - /// as a fallback or if you do not want parallelism. + /// `parallelism` controls how many targets are read at a time. #[expect(clippy::missing_panics_doc)] pub async fn verify_targets( - self: &Arc, - parallelism: NonZero, + &self, + parallelism: usize, ) -> Result<(), Error> { // This is a reimplementation of `parallel-task-set` from Omicron, // and could potentially be replaced with that if it's published to // crates.io. - let semaphore = Arc::new(Semaphore::new(parallelism.get())); + let semaphore = Arc::new(Semaphore::new(parallelism.max(1))); let mut set: JoinSet> = JoinSet::new(); for target_name in self.targets().keys() { @@ -288,10 +285,9 @@ impl Repository { .expect("we never close the semaphore") } }; - let this = Arc::clone(self); + let mut stream = self.read_target(&target).await?; set.spawn(async move { let _permit = permit; - let mut stream = this.read_target(&target).await?; // Read the stream to the end. There's no need to do anything // with the data; the underlying stream performs verification. while stream.try_next().await?.is_some() {} diff --git a/lib/tests/tests.rs b/lib/tests/tests.rs index 8907316..61ff959 100644 --- a/lib/tests/tests.rs +++ b/lib/tests/tests.rs @@ -63,7 +63,7 @@ async fn verify_targets() -> Result<(), Error> { let repo = Arc::new(repo); let parallelism = std::thread::available_parallelism().unwrap_or(NonZero::::MIN); - repo.verify_targets(parallelism).await?; + repo.verify_targets(parallelism.get()).await?; // Now intentionally fuck up the archive, and ensure verification fails. let pos = memchr::memmem::find(&zip, b"hubris") @@ -74,7 +74,7 @@ async fn verify_targets() -> Result<(), Error> { .load_zip_buffer(zip.clone(), &log) .await?; let repo = Arc::new(repo); - let err = repo.verify_targets(parallelism).await.unwrap_err(); + let err = repo.verify_targets(parallelism.get()).await.unwrap_err(); // This error ultimately comes from `rawzip`'s CRC-32 checking. assert!(err.to_string().contains("Invalid checksum")); From 1aeda4a39a9b255e05b65e18dae3d3965b95f9f5 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Wed, 4 Feb 2026 14:46:25 -0800 Subject: [PATCH 028/137] add v1_compatibility tests --- Cargo.lock | 4 +- Cargo.toml | 2 +- lib/src/repo/v1.rs | 4 +- lib/tests/data/v1-fake.zip | Bin 0 -> 1097510 bytes lib/tests/tests.rs | 8 ++- lib/tests/v1_compatibility.rs | 99 ++++++++++++++++++++++++++++++++++ 6 files changed, 107 insertions(+), 10 deletions(-) create mode 100644 lib/tests/data/v1-fake.zip create mode 100644 lib/tests/v1_compatibility.rs diff --git a/Cargo.lock b/Cargo.lock index eb4ecbe..ae5c828 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -319,9 +319,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.10.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "bzip2" diff --git a/Cargo.toml b/Cargo.toml index 7bc0c3d..6506aea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,7 @@ anyhow = "1.0.96" assert_cmd = "2.0.16" atomicwrites = "0.4.4" aws-lc-rs = "1.12.4" -bytes = "1.10.0" +bytes = "1.11.1" camino = "1.1.9" camino-tempfile = "1.1.1" chrono = { version = "0.4.40", default-features = false, features = ["std"] } diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs index e2859c5..36c89fe 100644 --- a/lib/src/repo/v1.rs +++ b/lib/src/repo/v1.rs @@ -13,7 +13,7 @@ use bytes::Bytes; use bytes::BytesMut; use camino::FromPathBufError; use camino::Utf8PathBuf; -use flate2::read::GzDecoder; +use flate2::bufread::GzDecoder; use futures_util::Stream; use futures_util::TryStreamExt; use futures_util::pin_mut; @@ -437,7 +437,7 @@ impl Read for MpscReader { fn read(&mut self, buf: &mut [u8]) -> std::io::Result { self.fill_buf()?; let len = self.buf.len().min(buf.len()); - self.buf.split_to(len).copy_to_slice(buf); + self.buf.split_to(len).copy_to_slice(&mut buf[..len]); Ok(len) } } diff --git a/lib/tests/data/v1-fake.zip b/lib/tests/data/v1-fake.zip new file mode 100644 index 0000000000000000000000000000000000000000..a0eb1a9f98449a257bd14fe9476c891c0c710a2d GIT binary patch literal 1097510 zcmeI*1$Y!!8}R*&5HtiWBv1&!3xwU-SwjN^hoXhztjv2w+FCI>I8=#>wuVOdcZ~`U>yypb*Dor#OPJ#DtK_R* z6W2RM_l{7+Rliq{^(&uzPgI0r@6U}AwvuFQoB9lDB3~s#O<0&UA}T06Iw`5??hto{C}CZq zgT!q{$(;NmNl6c~Mg=9kPI6Y%+tlX?vPuS{`sz-rR&TTzOm>6TW|qx%hsES@TAXI5 z&aTrroRU*-vMENpMNt%|!)7!Z3>JspYLIPCo6T<1IsKB#L4DGG>0b5ka-Q$zs!6^)|b# z*z^{i)@qSVW}VZdHycfo#bPok2FYoa6~$ytDNi-$!CllR)S0!C#U}Ejmu*_T-6Gqp z4xQd4>ns+F)@pP}R--|7Dw4>iRnl3kMw3pb*cFT2X4e{ZPKTt3;;|TE4MbcYE3MEBrNG>+D6gp(bhGQ z-zi#DRdp-%Ms-cAO8yyYgIZ|+Tyaw$np&~`FW1{>Yh)KCI_hoJR&UULNWHb_C9_eh zSL{~FVYG{jHgUjo7P};>uw*is6_Zv}SiM$nwb-prNiPnF!C{jn#cVgpvQzgzT5pZA zBpEG6*`jyY)odCiyH2Y!=*%YBp*L!EcB4~viXWZXY1fJBYqgq06R}zqQ4FGl%%Yha z^iIj)G#MnT*vYPSTCI}aE^bsD7K72Q=uJAaU2hgm!)UM=6tkk&%SM~kU=$5TYkOP0 zbv)W8t?<-n)UD*(>TUYk@b+q(^2q;7wKnzDQu}?iLG=udwg+{J43Dnhw0@yfNWNY{ zy`I$7`-^@2yTphtCAHHMJH6_P%_c*-PVrq`vFO(pwrq6lELnD7$uP@hu-9HI1DDE*=%!|MPFo=oerzhCfUUJVYbR|%SB2p zOU*?}C40|Y$l4Fch0Uq6*>w&vKp8AflTl}u?0T)%B>s~v7Fp*sn?+h}ibGKcevx0D z)@U~>Ca1|RYNIUM-_gCxBzy=gt~8=Y3VPWZ!)hFL3rz|k;i&3dClXVO}24ueq=*NnPP8N$nZ!_5?hd3Hqt;3?%S%vB)t7OsJYzC)V&n$YA-KiBFt|U4zQQyQ+V-_8@ z=pzhfa}5y+vzRoRaJ?neBGPW_FmJT8Cm)yIvte zgJf}tPFa>@MXPlR9U07$Ss3Zf=$cs)Zn?=w6 z0r@dowTh(I8B8{-BIZ<1F$~&GqQ|vJVy6Ex9fCvlg;FGn)L=z^=z`mARDZr z0-DlKyWSY5mBAv{)1Kwsk76hDdM}dV%Aw?p+%e1 zs#C-?N9fyVlpJ=Q%`8AJ8jV@26VoM&RxA1|i`FDbl0(d-3=X5nrB>3ubvz6Tk4o`Y z(xvw~0(zZf`GE3Q^jed_Xwj;pwN(*Q2GQx-C9SwgGTO9aXq3&OleH?6WD`|QSVXHc z*=>0*r*i#6)fODfsbb%vztua+?UM81<&P1MXSB_t5uCkzR{80 zmE`vCmu~SX<(BtcZA+~NDJ|jOuR5hwF)=F?&z`!Bs5m4seXzV*l}f(1hczT!rA+FC zBf8r{g6-l~Kb=h$Y_~gQMKUPj*a`b;6|ul2TV*ksQdglAqgeXV>vUp`$!Jjw2CLKR zbSQQ)f}6F5^wdh%O-9*+#JHzEe`=Ca@~$50SM$eLt>IOJNmuZxwHSmz+i7|0~NcsWJX{r7gKa zivRzg_QOuSShTP^#R7`nA%-Y1&I&|W_3FB_Q?ZIM*DS^?qioWPMGmb^(b}wH02BO{ z#cGGRLCn3aVv$Gm$?CL0tfkw`cAH5|Uu_n#04>?=V#4Pz>kLkZLu(e3TQQnDbw;CL zuHGU#_O~v-Wjb@N^Fgs>tZt#H{cuV{sQ7Q^&HnG0+^Z|t{^CV*h^?5Wafb_=eOxq} z^X?jrSPawzhjyvx6dn!nS3;aM+XfkRHQJPF(Z;wW=WvZCE;cc7Pw{}Qc85pxxo2E9 zc5SbT)3|qTpa~3c_jU;oxhb<$vue{(DQNTn@m!k20iKdiXn?%aV5s|5kjo&C0iG^% z1{@IgcpA!hsv}U+k4vyu%`6E`JhL{elvwlLj2iBp#}~|9#r<~=+fUzh9$c_gjjWn1 z+n&a^DJ~6m2`JF{lfMGjH!C)2Xz0k!M?WvCIa#Cdpn(Pa3U>Rb;mU*Cd+w;^<)7cH z@wwv0_&NHx*7Lhy^*JTEITfLV=lipZ|N&teYQ6#UN`J$<$9HR92}fA zu$4!L7DGmlYqWXuZcU|MA8v^mwd`n-Yza$SR@?Ak<5vS3j{PR{j(a^zv>2Mzt?S+E z12Pt(U)67KZBVoFMj_gQ!_o@zQNW6N8jV|Q;?+R6fWgH(dsLfs#$`#@Mxqci1*9*; z8JcLH88I_wXf!T2G!L^J{CU8mDl?88%`tk?;6|eZWSw`xNm)GQEI)O6T4CUjhn5*R zwffl}_jSK~d+TX6+u_z1Dvn&gKWp6yt2WQ^uM?Md+kLmYgRDi`%6>OW#9F6+)xL4r zJYV%1mixzI3BxPpDqjEC!V-n?&~$$3e|coY%wjhN290al(8bl? z?VsyQ(g76z%^s*;%20b=-}UuUO53%zsDNp^iZW~3Fq4ORx2yQqL;Up@RjYs|sJksP zILg=7JvhYSYmKxA1@};VduTJhCDUN|=`JB|8jZSzi(Ql6K}>C1|8{Pg)V9>wLDCAd zx^njVRH5joENa3v?uoZs^{dySS-$G=zm}9&?~!cZ8ceP0Syr!YlU?7zIke#N&m7)= zPJL?pcHy-Z75y4_s6Ddbmrd`ryXUri(~5eB8_gVea$=r3bsAWo6(4ut=OHU9FKbl% zS>4(Z!=lW?`kY;R?ew{I*CM)iIkF(jhEjD3ct+K64YVyQIPT}W(G_wH@BggS^O`dQ z{aju4xNX~heQTMPd44?nXO{*E)e{>n%YMA)^t?MxY|kBOjCxqFMWw3OT>H69{GvkW zx4E;-+ETxJ`yuTLCQf~NxqSSKFDf@|vHw6~&1IdlcMaIx`qH3~nOQ0fTA66wSHDQW zfhT3MyN%H2h$%cibgL#NSkq|557lyCDKdXd&cpYMuNePx@!4|^n0gKNnxz=~vtuE#E4$d)s7>nB(;|SM#6GEcYp&@2gFpX_gcT-kGEJ+>0d-AGC)rb9`B| z_@OyrEB376f~M7r@4q}|-QCOO)^z^#V*R_liZ^~-e8DHze_66&Q>$I)D{g#vGCp8b zyS-2E-}*uM^lASl<6GREGrCWu9;RPsFIykn;b!wL!Dl*HypNW?zPRwt3l}e(8>_bu z+jC`B?;dkX$KSqx`ttqUrO#A&-f-vKS<%}p^Y<=mnYB$!n@gSREm^uP@41*S&Bq^f zxIBJy%~g+E2G=U^=+U3seY%ZLxSRi~?}RIretD2>_mS@&|FvS2U+=Q}vfm#v-?!Y7 z+rJ(hnlp3kz(p;FwSI zR2!0bC-R3Db;7Ro7*VkIy?pL_w$EMoVYV4D|jxVTm zx_^cBlXv$y{NirY2W{tOJ~B*d6F;Kz%az4$w)*2_t1cVtvu7`Re7W>DxsCaUY@Pl% zrblG4FJ_O7IkvRHjpn@*7wFdh_@(oYiOF-8kH# zXld`7b)MC{;!<#5>6iU>cOKLrPx^qlN{B1Ox65VpmdXf^K;rqJm{3=!Tr`B{q-#TM$pgu zVsAcQy)X9jKu6UD`CZDnjw&$l`(nEbd28}%G#MMd;l;i=eMF;Ix9+bu{C_+7_J7Zn zg7nQlxKZ9_xu%EaE?2W@m3d$8YMy^|-Qb(6rq}b@_DP-v^Rj-@py6i!vOhikxz`T? z`M32dbg6dn&li+Bd$8#9Ty6zx1da&G;&XXqL2cK-p+WV|Wcp_7@Yn%zx67W{Ohq~nTFIF(lW!bn{3A?V`k3CiCc9rP?n=1 z<(}tG@9&xT%b1WG)t-Jo?4C2PXTAPM7ymYWN9mjcZ^Rai7(eXb8FOI1CN*DNxK*;a z-Nm)(ZIAP{r=A}%WzfZLKh@ki;_z<`k1ZHlrCZAYN6QV?WiID0hs|C7ThYA}7c44f zxi$7q7G8P_F!2Z!zYi_f)$QZWUoRapqRpZEpD!yOyw=a} z=k}APPal7HQl7PkY*9@fZ<$f+^nUa8AI4PB?f!A8>GPN-dA|0(ckAZlTDGf~b8OGL zdRdFGCSUX!`lIfCua{dty7uF)IaR*D-E2;-g`pi6tuRR4M%R+cUEegnbf!W(hPTM} z;>Db^6AFaxY+UoZp$#u?`7V3E&-cz9yWjJxz=<9aRlm=+U{&9>yL?uM|2pH&4Bfp8 zHxFdb9k*)5FDq{jIel|sA^pPg6PGqWJo5RTxQlxyx^1=pG~W<4=Rw`A34!|uU%Hlc z?&`QhV?u)SRTwe7Qz!2{k9yRt(c@d!*ySOE0=L~=_`-Mhy}Q<~NB^+JZ0UP=wfFoI z(=*3zTJ+n>4%(u9`ZmfEaPD@gzVq-DVZo-h9WYX2J?)y+nja&-Ra$-%ziF|kkkKDpjt>$*Hcm4%)BJK91{8t(LK zn0HrupMG8ET|ZW7UBw^z)ukn6JSPT1Q{=<7(lkewWUgMM4A~AMwvyxRu zjJOy(GO_H-qN6uNO(@v1NIutmu89?0tGi^m96asK-ayl)bHmF*4eA#4dT;P=-&*s3 zuXh168u7%Suu>)84Q*ZSFIuaZxI4XAF1wJK91S48I#n%Wgx_t>N> zRx#J`b1$I?B1mlh77$hy3>wpznuN*ud2TUJ{rBZOxqH9c3yPLu{5fW zCammtt8$cI@KKrQMr-d*IdbvJ=P$~9bJ(?Pz|>Yh=?}N--fC&li+AeH&6dZ*Pj@^Z zV|lhO8&Q0_nwhto`+%I_NH({-OO0V}ojo?pnkLMxmZwWV`ttPEbkE(QPNO3`eS&_I zWyz;V*&E*9JRJD7?~#LzC5^`8OIddFJrKOoBY0k5Ov5a$E1r60-BhTV=1;e%@=un{ zpEqSd&C}g#ZTf!ArGZDvkEuSq#sfxxlWAu2vSE8x1*F(}M(~}TuLJt^DwKRI-^<>wmsZ-Y zP2Xkj8lnZgr@dP&&vU$l7=G1llz52@_Wrl;(WN#zA@2X+ZN`k1$1V3-_j>Pcj4y)< zywGSSC#thvjcAN{ZGA2E7B-o{nt&S+fK(`SmI#)6nxL5&8^a0}3=jZpl_b+mYZtR9BUv(^7I?b%as z?4myQVjjgd3>(v;Y3@zGMff?gly7M6zo1Tz;pg+XcP?V{sGFry&;{#H9t{TjHJe{z zK$l*5TqX~E;u!Kxg&|v9|Ed?btMbSK6>e6$>e9OUqzU``FV9?Tz;8!yb?`p9q`X;w z;$Cp>)q`Ren$E@_+Fv>Ep7p!u)p-20`r!E|_AhCEW17c?Hqz?u4|Hq$eATGx^zAp+ zeq)F&w5qld(s!V zuVzAE%SV#zlXp~i zje1vO9{B!^xonXAWl7PH)orMFi42YT-+oXibt3&nOHQ3r|HG#RGS;b47jHdi^6qB* z`>ArRHJV9DW2)v-EoWSHbxe&v?K@hdxh}>4%@DD8r5U0rtJR!vnb$a9c)i>=LW_M} zYO=0sojP4(eBwfz-Ei!0xq05hn3>xfO_n{r)Oq_9jB`IY$M?4&?Ep8A$+I8Unbmmy zh&eu`G^0ytCTU!Jg6Cw)bUM?n4WC_Jmute|d9l|nT+HT?&9#-gvE3Jc6pF9ox=rI4 zQc@a}yO6d>^@x#WD&%vyRO5@1nxz3b_egsTb0_>cU`v)9HETA#P`ml&9uM66uK4li z?TfY-G?pFIC3<6<&?*NuSkB+L<`~h%euyE z>&JbrjlQw}-1P*jQfb5f@hd-_y7Oj^EhE}qDmwY=_I2woy(Z-{4EyaHXYL|byIFh0 z^r~lSIqc!4&~q==-@M<>rKI1EC+of{yJbk+rzhI3$yoCFzn^71qt?5(x8|Wshc8HO z&4DhOS#GwtYGU+FE_s)o=}TU;=4^9D$7Ly~8L(MC?B+Y!V}QnI&XC{dO!kP28{knV zz^$=daI)-pl%>{>&vUv?{-oQ~$(lc0W)wW(GJ2fLv{hsF4$GUTLiRniGq)|?GJm@e zuP)s+-G1pA5ZA0%Ue_!~_f`wjXloy^`p-;gactg_uvIlmwLW|N>l=+%dG#xsE3ih6 za>G7rBrT}5^GmnF#cH&AzTSQ|e%Qc#8y!`C%6I1M}mLmRdG ztYFx-m5yTegGFm){^_RI3;oQGJ2@*&Y9r11{rK=9-uWxVogZ+!W2>%ZSM)8v-Pj<@ zy|FKS-Ta(WijA&N>2yf&FvmjoVDF5aTiiOH$Y8O*=ZUw!5ymD)lBmVbO zeGW(MuX^pQDWUh-Yf}@(9U3`ySMzdT)EjhfXSSjfMvuMW&FxW%X05)C8#}0_`;a4JZx(H^x@Y%6HC`k>Dd2Eq&vmTZ ztm|F_n~n27Uwp%wg%j&OyjFHc_0=V-T|C|;cuDZMUjE-NtDG29eBq6PH$OkRrpKwu z&tlHh@9?aDr`ZoPeUf0B+i~CIj!naz6E^0XxHIeBpSIl}aoOXGo0~tX)Uet3Jy+&T z^O^8Rhepi`x()1XZgYIf#GXrkyzCq_>t)?yy-(J=G3IdWnyp_Q=v|=8ohm(Np9%h` z!#J--w^q;Yy|c#>^Yw1id$e;imAke(>bG-K`ps|M=aWXUANRh!ZAs+NvQbOU`NSX3 zeeM3}(HnNpJbd!guzL%)o}Q|yv0I;a^!^ghqVJ!7ID1>KD^pu`YhUQjiUMVR>~wrV z&Eh4i%o`fA-`DW@-BqpHoI6^o=(VIQMT)$_ZKC@*T>+@@-xJ`F5)!kfR=t1{28rRLwy=S%gZS{ccn=<8G zGGJ8ZO_{djE|Y6?@#^&qxu*J>yY(8nX=GLx_tNgIvMdVv&h?`*&x87TTz1*%HZHTT zSFZelImG3`rOS8?xpL&AZP}l%k+UuwZ^<7y-@A}Wf2?W5)ryl7Pc7SbKekNiHE{`R zCh>0aT69CDpjk~%sUgN*Q<7C zizEE;di|*pzXGbal?j zLvzkw{JGzgyM4lQba+0%*fKU@<--T37aRz!-)wIDqL-CgUt8SrV5a3m9M+b>PmWa2 zUnTl+yRQ2eFIycsru>UM(G{DRg1%^x(Dhb3!yk`tJ@qVL*RQu*tpHh?+w*#j@7SGo(bR%!huN-I%%@jni~)5_h|Mg7-4@PsCd`*N3C$FFXB?piL}q)r2>Y#K74%juwx3wob- zjTpG?!KL47A8_RC-)hU#N0Iw~&ui=++vJkZgV^zxy$Ww~J3nylA-6=0p~IhFXaZhe zHjG~UAnpr69CdqR+3??f&O6(3vPq{Ea7vo;A+d&1@+@_blkMfWqW*DN0xu8yZQOuCKh4Ne=egUEA=6Hu>T~jNRQ%db@s|0))qmPM?$$oZ z+j`o)%g_?_J7-?}W#Mj9pM(}#(eOvf+r`{yfVNGml|wy$w@Gd4w!0Ji_+a1C%PM4E zZY`%RyDG=>pJ#1zY|GiJ$;F5>Gr!citW@SsU!@FLSn|Y*-lMO*G<-bq$gIm7zxi#} zaret-pPp#bp+UXkvo=pl)ZVH2qpxe$btAn_4W6kj8Jm5xTSQjPkXv~xMYSF>#<;-q zs~m&<`!B54U_m{v`VABP+vdtPuIIds8>M{BHh+_OY2{dTa&;6?J1sv*aob{PG**@E*m zCcJpEU{~WM_Fr1IebDh}wOP9xG(9xo;Nm5fyM7$Hd*kt`U5qp46j~Nl?)eM%tJUi7 zF8g5R`67Pb`s7ULb~C4r7qE<)*Q#y}YUpnm1Nk|_bpxeAS}4M`&W6}C|GEK|;L3+j_AlKd_&1+!b9XkskYn)apEHelwx?tGlI%6g{ciX?e*M}4tz%m> zcv%07bJI3G|0@5eTW4`5Mytz#1)A+dM zw6K)Ay|GOAZ@)*7`ldnZC+$-{n*Q2{?K8qU*Hza{Mc+N83AwOXJZPl4GEp{Qt5_yX zn$p~9?%rOXoBPeCntOQc;+usLgVSy zSfg>M?(~@ajoT6Rg8Cr<%}04ImpItX>(Kmqo$hwbyJN=AGCRH3cx;aI-SN5a@baBU z1lGIq^z6@dluQ=~SfcZ7t}!*&=B&BI2t7CM=PkoMwu~H_E&rf_{x35xsxfJFha4Yw z^E0k1TYi1+(faS&_4^^T|CGvC?{>M?`I^u3+`rGNTU|fmhNEjak0yoc8&}qydgGTY z-99QdDO={ehd*xaJ1AFNFV`o5)gENMRR2o<&u#~7KJ+xxlWavR{;^?W??+kZR3Ek0 zZ)%x(#)v;Oue~X6Z{KC*Z)#?QdozBrA;X*UjTZ#AOoKZ%JKpE;GiJS90$63kM!2c^?ia4c*G4ZHGR&&Sx4l|^Jn~i!+Yj2j{z%A zEzTB@&)FdJjLnhq<=s_x6!5A)@sF7C{mTprF5PEjo05xXR5|oRRE_a51>9_Z?r}b{ zRh`|?rHaQ4^TFSjm8d!6OIPJu$&N3oYYxUd$^EdP^Pxra}q<81j zUH^xn_Vng3pUj%t&5pguv80^T^s5HXJsT{m{Y91BqZiKmEb-S`Z7VjPzpvD)MZ3M* zY_<+rHtxGV(+d=-UbO#-im_G4dW|YjIqt9HA;wXEUA=$r+$rnli4UXCZM@ccVM2WC zQPz9ec3kRxa@y6o)xW&-@buw+H=jq1?YH&X%k_V~aQ0dL%U}1Llf!pB*eiOw3-QNR zKmN7Tz^&bSu8vLUR;B2UE5TLz$5%|ea%q~O2i6T8 z`}TgAJu&J^eBz%gD);w^_wssnyQ`)Au-g-h4!!-b*ScqGiWc3!dfmClhp+vf_-m2= zMb;N>@73PxQGi!(--|9AF16dRI{a4EK7X7|c$Txsd6$&m>wN;^A9YOqou0YZfRx{- zMGo}&^^erwYpIVvKX60lhp!&rF}`=|<9p4BcS(EvAANWU3QE)yY8354?H1@j=xNna zhVg=dRYe6nfCunE;Q4=)2QMr2i8bH3^Em!a&K=KQtf=zkj&0csZ|xgmoU*NN)L&0- z^ys&?OTp=P3SG6o?7Q;i(*7F@ezYW_%JjKsg8T0}Z2okA)ibO2RZlo>>Kj(+_L%Rt zKij&s|GKt0cUzzBAG@VTWQ{fZvb*MTEtbo5=FP;<+9&Q`oRIdf|AqJhFRN^FOZgp_ zFyO_;!>JRMKl(iTK6Mh~8}IV$+{<$}e?6Q0dn)m9b+5Y`_4jhs&WYk`VwaRD%j(42 zu_-$y@1@=l8-KHvI%bKx&b?G`C~|(_`pmaduRTkgC7$?2xcWpJ#h(WOXWf!6q+A~x z^Xe{fOKiqFPbH@1;6TEFloYDzteTQUk;;cvXC%ZdN&VfQn&VAw{k*QHW=ibxASp{~ z#sbc{r9FO00xv;9QKM)VDSZMR2tBPj&@f&wu&Stl2k-zMyv~Eze!o^P+_UGip0TH= zPaJmX%td94wVH9tUfYJXefxG_xAwF?N0S842fEk|ZI>LYUNLT4xx!^X@iIjD~Bu&&)TM1%-o*h z_vzfkrwq#s%M8ovte0V#VOd?|U|42YetlSmtF()xRZ`YRS>;t1v|uxAhRv`!_5Cqc zvtTo9hRwX$mC_T!19$)r;6dsqrT#S!+E2{+@YZ4czErFPrU-=31d|D9(}P=hu5MfDcZ$3y@Atbcy5PP6hV40dXOXTLCOa;WX# zP-_>ZVvIA&Uj%Y^6$9Mwi{P-NbKR+@Jm?f5PQRc?X?Ln;V-U~i=dj4KI4gnHU^D9M zA{w|}l4Mx~e-|g4)9I|D0K}QR92T=IPN`&ZC|aXYTo-4-5z&Svt2lu~%K2|n;@rQH zvZSv#Wr*>gmV(seWr)p?c6Vw5|84At54~kt8Z$f~Ihh$=>_h8i z@=1$6qzvEIZKkF3%}1psHtB>(A6TAhK(6$?$GZzMIsSZFzS2^bd{1&}lArfO%S_!X zLx=Hx$;q%w#`L`EU{ap?KTOeo*tMu7B2tj9yZEPa%lN3YGE~$5pZ6#!@kl=G0KbZU z`gdu4b!T!CU%kMGmTSt6B714C^6rD3^6DA(N;}?Z_hv|(T7)0iD$-JyuJHe}gudh0 zr=>aF1K&vge^dkh-q0VCi+4RkTG9XgBU6e!=}c#@&Ke?4vz(D@vj(30=3^0_N8R2C zEBbFw#F!eSLsWt{3OLO*>5g)Ss**8yN7q@y%hq{!@Q&LhH?+`bzDtfrsD>34!8_Ej zqP|0N<$B_(apVwTMFU=i6>V{_an`R7>Sp@{kDjCPZc=t~oaEW0MID#VjJMTI#mHzbiOq<7-&+Q)UzNK6-?cxR=o_Xq$T@Udt{+MT)vFg+DLKYRkFU;^eJ#evnHh6FJFDjX z?J+)nT)h9Ul%S$6NkK(Lj1RZfNd=z!c=`fQ3Pbp%rxaw9v;%f->3lfAXR??k7_$6! zz()l08|SH!GSBvr9Dd!(4DxOIsqN?4?q>HK)^7$*E>X{WQINjJgItGuAFEn)N|?tn zXF<0znx3B)taZfie#zRoGmWh3=CaJY7l7(U7S#G?euxp6<_x%bY#4IYEsEtjgHRw^;x6U?V}HUHKo+ZoZsi4kgr_H znv)*9h<@HT@o={n59F7tVwTxIA9F@TTMzT^l=$n%akst3CcNm=e|hi3{>yz@i7&M^ zdT?QVmFK++Uz=Ox&e{h}Ufe2FE#cbUc+;~fd-o=o9z<6D)D+t zxAA55FAuLjw`ka-rHS|bj_>zcJ#6fbd)8m7K98JQegDflf8C9VsM@Rg-Tw13$Lw?~ z>vi|z`1>1vwa1@W_jtqp%C7=?{I7n~Fe%swUmNFZ<9uyg1mPk4jvB3qs51YfUmK@0 zPl}gCXHI8MXPy#wj?VnQyEA`w^VW+Oe@67|+p}lv(?{kd`!5AQa^11C@VYZ$D^IuI zu;G_89q)L@?RXY<qrhartX#$*(3HyQhbo4%jaII zccu&Eky!t&K|VwXj&yN6s*DriM$%oc`)kURCjF_N6mcZoEooUG{FhqF%;OmP8TuLe z8TwP!<^H>e{-jZp(4CCkjNNKbImYh)zl_}*H=I59ea@acM39e9Z$3MG=7tFJVd=5% zOxLw*`}XU;Zk-77F*d<-_p-hlPlT%H+o{W6IUQezGh;2Y*c*B`}JgFPnw5;B938HCJ8j_*q7R06;z4Kb-pgM^Y1 zUx?*9v3w_%@5J(**yOGt>3;&i!e-bEn_+XxJer7FMATA+K*$V2W)L!ikQx8SLT3C= zt_5AgsVzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@Bkjb19$)r-~l{<2k-zMzyo*y58wejfCumZ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q{=oxH8@jk=bIYbl{wb#EJa&El?yee*x)n~j zRsA!2p!$EL5)oc8REf4atkKqr(ZQigRJ1iT!oO=&c-ZUDv|Ab9DOx;>x|LFQPWhQV zaEZ7j<1^_@CaomvwUW_4D$E)Y6%-!*x6kQ#v`t#!snMug$+}|bU2C!%Wl=CDli6u>n8dGt zM3lW#R73@*=-4WV%N4|BoxfK66R+UkCFb=5(O~%LE+K9jjk<+rNLo#Cgp&?LYSJub zyGiGCYV{VaU2#gX!(_JG6`R>%c4{4p)#PwEEPBZxSsYrML6&4it96>pMuS;0n~jp$ zn3S~0@aVL(saM{XIKzv5bNYycsay99iL31UTH+jfyH#tIC9A-7emQL*Su7DZ=v*!;T$hlVK8onECc`Fe$SB~jC+ zbHmHxu&P^Bh9uT6@mdmXPMyuJb65>pgT-kw>dcZ|uN6m7uazwpS?4sHEoQ4taVU~g zZ!#KeI<3)eR7_5jU6vfOZ1<1q860g7dUbS@ufFR5w^*L%cnMK{>NYAv0ymU?ErBMj zS#NacL;=|x2BV}CHO*#|CCOkhn{1NFrg!Ka4x`R!R!n-k!>Cnc(JC!Ahf}dxWW{1m zO`s^VPBzgNLc*;MB{KPNrzBP_&UAOZ?ckRUvcD`T60L4SGbDUexz`e|Guth8lS#HY z71<^ZwNWt}Ee5T{VYl0@c11DECZp3~HW^J?qeZfb!nG=jq%%4tyUD55>Qigx8>v=wOQTyjb#MN)K< z(`J>7dP%Q$7#!&;#Tzx@ZAtWLeB5$cG%9t=*svnZuO-oFmPKH zAju%wzflsk!J$|CCWFanHrpI#gRC>lqGxd0B%7oYO-@d2`>EY#O4ENw&lzpr{#iS{ zMx$;S%kWI)*Ai&47(~BgmmD^&QL7cbq|R!xi9TMk=xsKGQ+C)a7QM;t)QajTiBxJO zQQ9W6sLrCFFo>QrI#LXz;UU4PM>^g0x0PY7`I$QA6y>LG89Vpg;%bJ@-K?|7c13S< zT6Kz1b{Gw^(I`3WI-6M%OB1OpNv+ zB8k@M@W@V%aC`SqB`mdj|2qO48a*<`kX55mxAqwti!<`I#ECJ^A?qwoyVj_PCaySb ziY$gao2YAwPBa{?RT3pD8_jw}FH)e=d{_(Mw5Ev>?VWGDvp@hrjwm!F+j^|f1`EEPLs`O zHaZo%O|r`t#jX<_YHC&cyDl=_i1VIZ(O)JJD*(}m*RV2wK#&yv(YIP>N%_)W^SrWFe`UizarHtH3m)=(G zN|~?r)@X8x1m)6*3$K1=4_vtXwPcyCT1C?93?`8%qru=5oweO0s+35T-C{DB#o-pW z>vVRL&E#~N^#(B_*<_1BHdw{DWODk4D%Pm(k&0T^I@!Y`Bf3YW_Jv88GbS^WhnkkW O;@>`j+L|lWB>sOld|l80 literal 0 HcmV?d00001 diff --git a/lib/tests/tests.rs b/lib/tests/tests.rs index 61ff959..f4db75b 100644 --- a/lib/tests/tests.rs +++ b/lib/tests/tests.rs @@ -4,7 +4,6 @@ use std::convert::Infallible; use std::io::Write; -use std::num::NonZero; use std::sync::Arc; use bytes::BufMut; @@ -61,9 +60,8 @@ async fn verify_targets() -> Result<(), Error> { .load_zip_buffer(zip.clone(), &log) .await?; let repo = Arc::new(repo); - let parallelism = - std::thread::available_parallelism().unwrap_or(NonZero::::MIN); - repo.verify_targets(parallelism.get()).await?; + let parallelism = std::thread::available_parallelism().unwrap().get(); + repo.verify_targets(parallelism).await?; // Now intentionally fuck up the archive, and ensure verification fails. let pos = memchr::memmem::find(&zip, b"hubris") @@ -74,7 +72,7 @@ async fn verify_targets() -> Result<(), Error> { .load_zip_buffer(zip.clone(), &log) .await?; let repo = Arc::new(repo); - let err = repo.verify_targets(parallelism.get()).await.unwrap_err(); + let err = repo.verify_targets(parallelism).await.unwrap_err(); // This error ultimately comes from `rawzip`'s CRC-32 checking. assert!(err.to_string().contains("Invalid checksum")); diff --git a/lib/tests/v1_compatibility.rs b/lib/tests/v1_compatibility.rs new file mode 100644 index 0000000..44d5a1c --- /dev/null +++ b/lib/tests/v1_compatibility.rs @@ -0,0 +1,99 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use camino::Utf8Path; +use tufaceous::ExpirationEnforcement; +use tufaceous::RepositoryLoader; +use tufaceous::TrustStoreBehavior; +use tufaceous::error::Error; +use tufaceous_artifact::Artifacts; +use tufaceous_artifact::KnownArtifactTags; +use tufaceous_artifact::OsBoard; +use tufaceous_artifact::OsPhase1Tags; +use tufaceous_artifact::OsPhase2Tags; +use tufaceous_artifact::OsVariant; +use tufaceous_artifact::RotBootloaderTags; +use tufaceous_artifact::RotSlot; +use tufaceous_artifact::RotTags; +use tufaceous_artifact::Sign; +use tufaceous_artifact::SpTags; +use tufaceous_artifact::ZoneTags; + +#[tokio::test] +async fn v1_fake() -> Result<(), Error> { + let log = slog::Logger::root(slog::Discard, slog::o!()); + let path = Utf8Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/data/v1-fake.zip"); + + let repo = RepositoryLoader::new() + .expiration_enforcement(ExpirationEnforcement::Unsafe) + .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .v1_compatibility(true) + .load_zip_path(path, &log) + .await?; + let parallelism = std::thread::available_parallelism().unwrap().get(); + repo.verify_targets(parallelism).await?; + + // We expect to see one of each of these artifacts: + let mut expected = vec![ + KnownArtifactTags::InstallinatorDocument, + KnownArtifactTags::MeasurementCorpus, + ]; + for os_variant in [OsVariant::Host, OsVariant::Recovery] { + for os_board in [OsBoard::Cosmo, OsBoard::Gimlet] { + expected.push(OsPhase1Tags { os_board, os_variant }.into()); + } + expected.push(OsPhase2Tags { os_variant }.into()); + } + for rot_sign in ["sign-gimlet", "sign-psc", "sign-switch"] { + for rot_slot in [RotSlot::A, RotSlot::B] { + expected.push( + RotTags { + rot_board: "SimRot".into(), + rot_sign: Sign(Some(rot_sign.into())), + rot_slot, + } + .into(), + ); + } + expected.push( + RotBootloaderTags { + rot_board: "SimRot".into(), + rot_sign: Sign(Some(rot_sign.into())), + } + .into(), + ); + } + for sp_board in ["SimGimletSp", "SimPscSp", "SimSidecarSp"] { + expected.push(SpTags { sp_board: sp_board.into() }.into()); + } + for zone_name in ["zone-1", "zone-2"] { + expected.push(ZoneTags { zone_name: zone_name.into() }.into()); + } + let seen = expected + .into_iter() + .map(|tags| repo.artifacts().get(tags).unwrap().clone()) + .collect::(); + // And there should be no unexpected artifacts: + assert_eq!(&seen, repo.artifacts()); + + Ok(()) +} + +#[tokio::test] +#[should_panic(expected = "target artifacts-v2.json not found")] +async fn v1_fake_without_compat() { + let log = slog::Logger::root(slog::Discard, slog::o!()); + let path = Utf8Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/data/v1-fake.zip"); + + let _ = RepositoryLoader::new() + .expiration_enforcement(ExpirationEnforcement::Unsafe) + .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .v1_compatibility(false) + .load_zip_path(path, &log) + .await + .map_err(|error| error.to_string()) + .unwrap(); +} From 24868eda39d2884a059f56cad5b07f86aa799746 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Wed, 4 Feb 2026 16:10:19 -0800 Subject: [PATCH 029/137] a few tweaks and tests --- lib/src/repo.rs | 14 +++----- lib/src/repo/v1.rs | 86 +++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 87 insertions(+), 13 deletions(-) diff --git a/lib/src/repo.rs b/lib/src/repo.rs index ea3d03e..c0c4e27 100644 --- a/lib/src/repo.rs +++ b/lib/src/repo.rs @@ -344,14 +344,10 @@ async fn read_target_vec( repo: &tough::Repository, target: &str, ) -> Result>, Error> { - let Some(mut stream) = read_target(repo, target).await? else { + let Some(stream) = read_target(repo, target).await? else { return Ok(None); }; - let mut buf = Vec::new(); - while let Some(item) = stream.try_next().await? { - buf.extend_from_slice(item.as_ref()); - } - Ok(Some(buf)) + stream.map_ok(Vec::from).try_concat().await.map(Some) } async fn read_target_json( @@ -361,9 +357,9 @@ async fn read_target_json( let Some(vec) = read_target_vec(repo, target).await? else { return Ok(None); }; - Ok(serde_json::from_slice(&vec).map_err(|source| { - ErrorKind::ParseTargetJson { source, target: target.into() } - })?) + serde_json::from_slice(&vec).map_err(|source| { + ErrorKind::ParseTargetJson { source, target: target.into() }.into() + }) } fn sha256_length( diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs index 36c89fe..18a9c7c 100644 --- a/lib/src/repo/v1.rs +++ b/lib/src/repo/v1.rs @@ -248,8 +248,9 @@ impl CompositeArtifact { ErrorKind::TargetNotFound { target_name: target_name.clone() } })?; pin_mut!(stream); + let (tx, rx) = mpsc::channel(1); - let target_name = target_name.clone(); + let target_name_clone = target_name.clone(); let task = tokio::task::spawn_blocking(move || { let mut archive = tar::Archive::new(GzDecoder::new(MpscReader::new(rx))); @@ -295,10 +296,32 @@ impl CompositeArtifact { } Ok(Self { entries, original_target_name: target_name }) }); + + let mut stream_interrupted = false; while let Some(item) = stream.try_next().await? { - let Ok(()) = tx.send(item).await else { break }; + let Ok(()) = tx.send(item).await else { + // The receiver hung up early. We are not allowed to return `Ok` + // from this function, otherwise we have not actually verified + // any of the data we just read against its hash. + stream_interrupted = true; + break; + }; + } + drop(tx); + let result = task.await?; + if stream_interrupted && result.is_ok() { + // No, it isn't ok. + Err(ErrorKind::ReadCompositeArtifact { + source: std::io::Error::new( + std::io::ErrorKind::Interrupted, + "stream unexpectedly interrupted", + ), + target: target_name_clone, + } + .into()) + } else { + result } - task.await? } async fn read_rot( @@ -437,7 +460,7 @@ impl Read for MpscReader { fn read(&mut self, buf: &mut [u8]) -> std::io::Result { self.fill_buf()?; let len = self.buf.len().min(buf.len()); - self.buf.split_to(len).copy_to_slice(&mut buf[..len]); + self.buf.copy_to_slice(&mut buf[..len]); Ok(len) } } @@ -499,3 +522,58 @@ enum V1KnownArtifactKind { SwitchRot, SwitchRotBootloader, } + +#[cfg(test)] +mod tests { + use std::io::Read; + + use bytes::Bytes; + use tokio::sync::mpsc; + + use crate::repo::v1::MpscReader; + + #[tokio::test] + async fn mpsc_reader() { + static CHUNKS: [Bytes; 5] = [ + Bytes::from_static(b"hello world"), + Bytes::from_static(&[0x5a; 512]), + Bytes::from_static(b"meow meow meow meow\0"), + Bytes::from_static(&[0; 12345]), + Bytes::from_static(&[0x5a; 256]), + ]; + + let expected = CHUNKS.concat(); + + let (tx, rx) = mpsc::channel(1); + let task = tokio::task::spawn_blocking(move || { + let mut bytes_read = Vec::new(); + let mut reader = MpscReader::new(rx); + let mut buf = [0; 2048]; + while let n = reader.read(&mut buf).unwrap() + && n > 0 + { + bytes_read.extend_from_slice(&buf[..n]); + } + bytes_read + }); + for chunk in &CHUNKS { + tx.send(chunk.clone()).await.unwrap(); + } + drop(tx); + let bytes_read = task.await.unwrap(); + assert_eq!(bytes_read, expected); + } + + #[tokio::test] + async fn mpsc_reader_empty() { + let (tx, rx) = mpsc::channel(1); + let task = tokio::task::spawn_blocking(move || { + let mut bytes_read = Vec::new(); + MpscReader::new(rx).read_to_end(&mut bytes_read).unwrap(); + bytes_read + }); + drop(tx); + let bytes_read = task.await.unwrap(); + assert!(bytes_read.is_empty()); + } +} From 8d889488df84b0a86814a8a373a8a24c4d8323af Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Wed, 4 Feb 2026 16:12:04 -0800 Subject: [PATCH 030/137] metadata will always be string:string map --- lib/src/edit/editor.rs | 14 +------------- lib/src/repo.rs | 4 ++-- lib/src/schema.rs | 2 +- 3 files changed, 4 insertions(+), 16 deletions(-) diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index 21d2933..0bc0e7a 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -45,7 +45,6 @@ pub struct RepositoryEditor<'a> { generate_installinator_document: bool, targets: HashMap>>, artifacts: HashMap>, - metadata: BTreeMap, } impl<'a> RepositoryEditor<'a> { @@ -57,7 +56,6 @@ impl<'a> RepositoryEditor<'a> { generate_installinator_document: true, targets: HashMap::new(), artifacts: HashMap::new(), - metadata: BTreeMap::new(), } } @@ -268,16 +266,6 @@ impl<'a> RepositoryEditor<'a> { Ok(editor) } - pub fn metadata(mut self, key: String, value: serde_json::Value) -> Self { - self.metadata.insert(key, value); - self - } - - pub fn remove_metadata(mut self, key: &str) -> Self { - self.metadata.remove(key); - self - } - pub fn from_repo(repo: &'a Repository) -> Result { Self::new(repo.system_version().clone()).import_repo(repo) } @@ -397,7 +385,7 @@ impl<'a> RepositoryEditor<'a> { let document = ArtifactsSchema { system_version: self.system_version, artifacts: artifacts.into_values().collect(), - metadata: self.metadata, + metadata: BTreeMap::new(), }; let target = BytesSource::json(&document) .map_err(ErrorKind::SerializeArtifacts)? diff --git a/lib/src/repo.rs b/lib/src/repo.rs index c0c4e27..007abea 100644 --- a/lib/src/repo.rs +++ b/lib/src/repo.rs @@ -48,7 +48,7 @@ pub struct Repository { pub(crate) archive_path: Option, pub(crate) archive_sha256: Option<[u8; 32]>, artifacts: Artifacts, - metadata: BTreeMap, + metadata: BTreeMap, v1_unpacked: Option, } @@ -184,7 +184,7 @@ impl Repository { &self.artifacts } - pub fn metadata(&self) -> &BTreeMap { + pub fn metadata(&self) -> &BTreeMap { &self.metadata } diff --git a/lib/src/schema.rs b/lib/src/schema.rs index f0c9cae..604c117 100644 --- a/lib/src/schema.rs +++ b/lib/src/schema.rs @@ -15,7 +15,7 @@ pub(crate) struct ArtifactsSchema { #[serde(default)] pub(crate) artifacts: Vec, #[serde(default)] - pub(crate) metadata: BTreeMap, + pub(crate) metadata: BTreeMap, } impl ArtifactsSchema { From f5ced8fa42514c14fef206881385be8dfe45a92f Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Wed, 4 Feb 2026 16:33:32 -0800 Subject: [PATCH 031/137] don't bother with symlinks in zip files i initially wrote symlink support in `ZipTransport` because i had hoped it would be useful for deduplicating targets. however it's only really useful if the zip archives remain useful outside of the context of tufaceous. in practice, symlinks are dubious: windows does not handle them at all (as one might expect), and unix tooling is mediocre at handling them properly (particularly when wanting to extract a single file -- there's usually no option to dereference the link). --- lib/src/error.rs | 4 +- lib/src/zip_transport.rs | 143 ++++++++++----------------------------- 2 files changed, 37 insertions(+), 110 deletions(-) diff --git a/lib/src/error.rs b/lib/src/error.rs index d2a4e6d..811090f 100644 --- a/lib/src/error.rs +++ b/lib/src/error.rs @@ -265,9 +265,7 @@ impl ErrorKind { | ZipTransportError::Duplicate | ZipTransportError::FileNotFound | ZipTransportError::IsADirectory - | ZipTransportError::SymlinkTargetLengthLimit - | ZipTransportError::SymlinkTraversalLimit - | ZipTransportError::SymlinkUtf8(_) => true, + | ZipTransportError::IsASymlink => true, ZipTransportError::Io(_) | ZipTransportError::Join(_) => false, diff --git a/lib/src/zip_transport.rs b/lib/src/zip_transport.rs index 546be84..67f370d 100644 --- a/lib/src/zip_transport.rs +++ b/lib/src/zip_transport.rs @@ -22,8 +22,6 @@ use rawzip::FileReader; use rawzip::ReaderAt; use rawzip::ZipArchive; use rawzip::ZipArchiveEntryWayfinder; -use rawzip::ZipReader; -use rawzip::ZipVerifier; use rawzip::path::RawPath; use rawzip::path::ZipFilePath; use slog::Logger; @@ -39,9 +37,6 @@ use url::Url; use crate::error::Error; use crate::error::ErrorKind; -const MAX_SYMLINK_TARGET_LEN: usize = 1024 - 1; -const MAX_SYMLINK_TRAVERSAL: usize = 8; - /// Implementation of [`tough::Transport`] that operates on a Zip archive. /// /// URLs used with this transport must use the `zip:///` protocol. For example, @@ -69,8 +64,8 @@ struct Inner { #[derive(Debug, Clone, Copy)] enum Entry { File(EntryData), - Symlink(EntryData), Dir, + Symlink, Duplicate, } @@ -145,16 +140,14 @@ impl ZipTransport { std::collections::hash_map::Entry::Vacant(e) => { if record.is_dir() { e.insert(Entry::Dir); + } else if record.mode().is_symlink() { + e.insert(Entry::Symlink); } else { let data = EntryData { compression_method: record.compression_method(), wayfinder: record.wayfinder(), }; - if record.mode().is_symlink() { - e.insert(Entry::Symlink(data)); - } else { - e.insert(Entry::File(data)); - } + e.insert(Entry::File(data)); } } } @@ -219,23 +212,6 @@ impl ZipTransport { Ok(Self { inner: Arc::new(Inner { archive, entries }) }) } - fn reader_blocking( - &self, - EntryData { compression_method, wayfinder }: EntryData, - ) -> Result, ZipTransportError> { - let entry = self.inner.archive.get_entry(wayfinder)?; - let reader = entry.reader(); - match compression_method { - CompressionMethod::Store => { - Ok(Reader::Store(entry.verifying_reader(reader))) - } - CompressionMethod::Deflate => Ok(Reader::Deflate( - entry.verifying_reader(DeflateDecoder::new(reader)), - )), - other => Err(ZipTransportError::CompressionMethod(other)), - } - } - fn stream( self, entry_data: EntryData, @@ -243,7 +219,21 @@ impl ZipTransport { ) -> impl Stream> { let (tx, mut rx) = mpsc::channel(1); let task = tokio::task::spawn_blocking(move || { - let mut reader = self.reader_blocking(entry_data)?; + let entry = self.inner.archive.get_entry(entry_data.wayfinder)?; + let mut reader = match entry_data.compression_method { + CompressionMethod::Store => { + let reader = entry.reader(); + Box::new(entry.verifying_reader(reader)) as Box + } + CompressionMethod::Deflate => { + let reader = DeflateDecoder::new(entry.reader()); + Box::new(entry.verifying_reader(reader)) as Box + } + other => { + return Err(ZipTransportError::CompressionMethod(other)); + } + }; + let mut buf = BytesMut::zeroed(8192); while let n = reader.read(&mut buf)? && n > 0 @@ -270,88 +260,33 @@ impl ZipTransport { }), ) } - - async fn read_symlink_target( - self, - entry_data: EntryData, - base: &Url, - ) -> Result { - let target = tokio::task::spawn_blocking(move || { - let mut reader = self.reader_blocking(entry_data)?; - let mut v = vec![0; MAX_SYMLINK_TARGET_LEN + 1]; - let len = - std::io::copy(&mut reader, &mut Cursor::new(v.as_mut_slice()))? - .try_into() - .expect("MAX_SYMLINK_TARGET_LEN <= usize::MAX"); - if v.len() > MAX_SYMLINK_TARGET_LEN { - return Err(ZipTransportError::SymlinkTargetLengthLimit); - } - v.truncate(len); - String::from_utf8(v).map_err(|source| { - ZipTransportError::SymlinkUtf8(source.utf8_error()) - }) - }) - .await??; - base.join(&target).map_err(|source| ZipTransportError::UrlJoin { - source, - url: target, - base: base.to_string(), - }) - } } #[async_trait] impl Transport for ZipTransport { - async fn fetch( - &self, - original_url: Url, - ) -> Result { - let mut url = original_url.clone(); + async fn fetch(&self, url: Url) -> Result { if url.scheme() != "zip" { return Err(TransportError::new( TransportErrorKind::UnsupportedUrlScheme, url, )); } - for _ in 0..MAX_SYMLINK_TRAVERSAL { - match self.inner.entries.get(&url).copied() { - Some(Entry::File(entry_data)) => { - return Ok(Box::pin(self.clone().stream(entry_data, url))); - } - Some(Entry::Symlink(entry_data)) => { - url = self - .clone() - .read_symlink_target(entry_data, &url) - .await - .map_err(|error| error.upgrade(url))?; - } - Some(Entry::Dir) => { - return Err(ZipTransportError::IsADirectory.upgrade(url)); - } - Some(Entry::Duplicate) => { - return Err(ZipTransportError::Duplicate.upgrade(url)); - } - None => { - return Err(ZipTransportError::FileNotFound.upgrade(url)); - } + match self.inner.entries.get(&url).copied() { + Some(Entry::File(entry_data)) => { + Ok(Box::pin(self.clone().stream(entry_data, url))) } - } - Err(ZipTransportError::SymlinkTraversalLimit.upgrade(original_url)) - } -} - -enum Reader<'a, T: ReaderAt> { - Store(ZipVerifier, &'a T>), - Deflate(ZipVerifier>, &'a T>), -} - -impl Read for Reader<'_, T> { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - match self { - Reader::Store(reader) => reader.read(buf), - Reader::Deflate(reader) => reader.read(buf), + Some(Entry::Dir) => { + Err(ZipTransportError::IsADirectory.upgrade(url)) + } + Some(Entry::Symlink) => { + Err(ZipTransportError::IsASymlink.upgrade(url)) + } + Some(Entry::Duplicate) => { + Err(ZipTransportError::Duplicate.upgrade(url)) + } + None => Err(ZipTransportError::FileNotFound.upgrade(url)), } } } @@ -393,12 +328,8 @@ pub enum ZipTransportError { FileNotFound, #[error("is a directory")] IsADirectory, - #[error("symlink target name exceeded byte limit")] - SymlinkTargetLengthLimit, - #[error("reached symlink traversal limit")] - SymlinkTraversalLimit, - #[error("symlink target is not valid UTF-8")] - SymlinkUtf8(std::str::Utf8Error), + #[error("is a symlink")] + IsASymlink, } impl ZipTransportError { @@ -413,9 +344,7 @@ impl ZipTransportError { | ZipTransportError::CompressionMethod(_) | ZipTransportError::Duplicate | ZipTransportError::IsADirectory - | ZipTransportError::SymlinkTargetLengthLimit - | ZipTransportError::SymlinkTraversalLimit - | ZipTransportError::SymlinkUtf8(_) => TransportErrorKind::Other, + | ZipTransportError::IsASymlink => TransportErrorKind::Other, }; TransportError::new_with_cause(kind, url, self) } From ebd9aa3451094a7414dca9d129e6fc0359eec304 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Wed, 4 Feb 2026 17:07:36 -0800 Subject: [PATCH 032/137] oops --- lib/src/edit/input.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/src/edit/input.rs b/lib/src/edit/input.rs index c5565d7..0565362 100644 --- a/lib/src/edit/input.rs +++ b/lib/src/edit/input.rs @@ -87,21 +87,21 @@ impl Input { let base = Utf8PathBuf::from(format!("os-{os_variant}")); let mut vec = Vec::new(); vec.push(Output::new( - base.join("image").join(COSMO_PHASE_1_PATH).into(), + base.join(COSMO_PHASE_1_PATH).into(), version.clone(), OsPhase1Tags { os_board: OsBoard::Cosmo, os_variant } .into(), cosmo_phase_1, )); vec.push(Output::new( - base.join("image").join(GIMLET_PHASE_1_PATH).into(), + base.join(GIMLET_PHASE_1_PATH).into(), version.clone(), OsPhase1Tags { os_board: OsBoard::Gimlet, os_variant } .into(), gimlet_phase_1, )); vec.push(Output::new( - base.join("image").join(PHASE_2_PATH).into(), + base.join(PHASE_2_PATH).into(), version, OsPhase2Tags { os_variant }.into(), phase_2, From 4515d8efeff94bc769f96bdb0d88ea53be55ce70 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Wed, 4 Feb 2026 19:41:27 -0800 Subject: [PATCH 033/137] write a big comment about zip archive safety --- lib/src/zip_transport.rs | 67 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 3 deletions(-) diff --git a/lib/src/zip_transport.rs b/lib/src/zip_transport.rs index 67f370d..546260b 100644 --- a/lib/src/zip_transport.rs +++ b/lib/src/zip_transport.rs @@ -2,6 +2,57 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +// ZIP is a somewhat dreadful archive format. We use it as the repository +// format of choice for Tufaceous because it has two qualities: +// +// 1. It is well-supported on most operating systems without any additional +// software, so anybody who finds themselves needing to deal with the +// repository can inspect it in a reasonable way. +// 2. Files can be accessed in random order. +// +// We use the `rawzip` crate to deal with parsing the structures in a ZIP +// archive; any higher-level use cases are an exercise left to the consumer. +// This allows us to create a very strict ZIP archive reader. A list of common +// problems with using ZIP files, and the defenses we take: +// +// - Directory traversal, symlinks, and similar naughty file operations related +// to extracting ZIP archives to a directory (e.g. "Zip Slip"). This module's +// defense is to never extract any files to disk, avoiding recreating any +// problems related to extraction. (The available API is to read specific +// files in the archive as a stream.) Symlinks in ZIP files largely do not +// work in the real world so they are ignored here (attempts to read a file +// which is a symlink result in an error). +// - Central directory confusion due to differences in readers. APPNOTE.TXT is +// full of doublespeak: one section says that archives must be read "from the +// back" (starting from searching for the end-of-central-directory record), +// and another suggests that archives can be streamed, or read "from the +// front". `rawzip` always reads "from the back". Additionally, APPNOTE.TXT +// arguably underspecifies comments, making it possible for the comment to +// contain something that is misparsed as an end-of-central-directory record. +// This module always expects the end-of-central-directory record to come at +// the very end of the file, rejecting any archives that contain comments or +// any extra content at the end. +// https://web.archive.org/web/20250131021721/https://games.greggman.com/game/zip-rant/ +// - Multiple entries with the same file name, leading to confusion due to +// differences in readers. These are detected and those files return an error +// if reading is attempted. +// - Quines and zip bombs which abuse overlapping file headers leading to a +// denial of service. This module takes the defense suggested by `rawzip`, +// which is to reject archives that contain files with overlapping file +// data. We additionally reject archives where file data overlaps the central +// directory. +// https://www.bamsoftware.com/hacks/zipbomb/ +// - Zip bombs which use extreme compression ratios leading to a denial of +// service. This module takes no defense against this; instead this defense +// is implemented in `tough`. The library places default size limits on +// repository metadata, and all targets in the repository have a file size in +// the signed metadata. A malicious archive designed to run a system out of +// memory or temporary disk space would therefore also need to be signed and +// trusted by the user (in the context of the Oxide control plane, this user +// is either an administrator over the entire control plane, or has physical +// access to the technician port). +// https://docs.rs/tough/0.21.0/tough/struct.Limits.html + use std::collections::HashMap; use std::fmt::Debug; use std::fs::File; @@ -37,6 +88,10 @@ use url::Url; use crate::error::Error; use crate::error::ErrorKind; +// The length of the end-of-central-directory record if the comment is zero +// bytes. Archives with a "comment" (data after the EOCD) are rejected. +const EOCD_MAX_SEARCH_SPACE: u64 = 22; + /// Implementation of [`tough::Transport`] that operates on a Zip archive. /// /// URLs used with this transport must use the `zip:///` protocol. For example, @@ -84,7 +139,10 @@ impl Clone for ZipTransport { impl + Debug + Send + Sync + 'static> ZipTransport> { pub fn from_slice(data: T, log: &Logger) -> Result { - let archive = ZipArchive::from_slice(data).upgrade(None)?; + let archive = ZipArchive::with_max_search_space(EOCD_MAX_SEARCH_SPACE) + .locate_in_slice(data) + .map_err(|(_, error)| error) + .upgrade(None)?; Self::from_impl_blocking(archive.into_zip_archive(), None, None, log) } } @@ -99,8 +157,11 @@ impl ZipTransport { tokio::task::spawn_blocking(move || { let archive_path = archive_path; let mut buffer = vec![0; rawzip::RECOMMENDED_BUFFER_SIZE]; - let archive = ZipArchive::from_file(file, &mut buffer) - .upgrade(archive_path.as_ref())?; + let archive = + ZipArchive::with_max_search_space(EOCD_MAX_SEARCH_SPACE) + .locate_in_file(file, &mut buffer) + .map_err(|(_, error)| error) + .upgrade(archive_path.as_ref())?; Self::from_impl_blocking(archive, archive_path, Some(buffer), &log) }) .await? From 32c790dbbe69def64f2cdfbfe9089e90963d9cdd Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 5 Feb 2026 12:41:28 -0800 Subject: [PATCH 034/137] regenerate installinator documents for v1 repos --- lib/src/repo/v1.rs | 125 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 103 insertions(+), 22 deletions(-) diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs index 18a9c7c..581d230 100644 --- a/lib/src/repo/v1.rs +++ b/lib/src/repo/v1.rs @@ -4,6 +4,7 @@ use std::collections::HashMap; use std::io::BufRead; +use std::io::BufReader; use std::io::Read; use std::io::Write; use std::sync::Arc; @@ -12,6 +13,7 @@ use bytes::Buf; use bytes::Bytes; use bytes::BytesMut; use camino::FromPathBufError; +use camino::Utf8Path; use camino::Utf8PathBuf; use flate2::bufread::GzDecoder; use futures_util::Stream; @@ -34,6 +36,8 @@ use tufaceous_artifact::Artifact; use tufaceous_artifact::ArtifactHash; use tufaceous_artifact::ArtifactVersion; use tufaceous_artifact::Artifacts; +use tufaceous_artifact::InstallinatorArtifact; +use tufaceous_artifact::InstallinatorDocument; use tufaceous_artifact::KnownArtifactTags; use tufaceous_artifact::OsBoard; use tufaceous_artifact::OsPhase1Tags; @@ -59,6 +63,7 @@ pub(crate) struct Unpacked { pub(crate) entries: HashMap, } +#[expect(clippy::too_many_lines)] pub(crate) async fn from_loaded( repo: &tough::Repository, log: &Logger, @@ -71,6 +76,7 @@ pub(crate) async fn from_loaded( let mut artifacts = Artifacts::default(); let mut unpacked = Unpacked { entries: HashMap::new() }; + let mut installinator_document = None; for V1Artifact { version, kind, target } in v1_artifacts { let Some((hash, length)) = sha256_length(repo, log, &target) else { continue; @@ -141,7 +147,11 @@ pub(crate) async fn from_loaded( } V1KnownArtifactKind::InstallinatorDocument => { - KnownArtifactTags::InstallinatorDocument + // Ignore this Installinator document, because it is written for + // the v1 artifacts. We need to generate a new one for the v2 + // artifacts once all of the potential artifacts are extracted. + installinator_document = Some((version, target)); + continue; } V1KnownArtifactKind::ControlPlane => { @@ -161,6 +171,16 @@ pub(crate) async fn from_loaded( let tags = tags.to_tags(); artifacts.insert(Artifact { target_name, version, tags, hash, length }); } + + if let Some((version, target)) = installinator_document { + generate_installinator_document( + &mut artifacts, + &mut unpacked, + version, + target, + ) + .await?; + } Ok(Some((system_version, artifacts, unpacked))) } @@ -172,6 +192,36 @@ pub(crate) struct UnpackedArtifact { } impl UnpackedArtifact { + fn new_blocking( + reader: &mut dyn BufRead, + map_read_err: impl FnOnce(std::io::Error) -> ErrorKind, + ) -> Result { + let mut file = + camino_tempfile::tempfile().map_err(ErrorKind::CreateTempFile)?; + let mut hasher = Sha256::new(); + let mut length = 0u64; + loop { + let buf = match reader.fill_buf() { + Ok(buf) => buf, + Err(error) => return Err(map_read_err(error).into()), + }; + if buf.is_empty() { + break; + } + let len = buf.len(); + file.write_all(buf).map_err(|source| ErrorKind::WriteFile { + source, + path: None, + })?; + hasher.update(buf); + reader.consume(len); + length += u64::try_from(len).expect("usize fits in u64"); + } + let file = Arc::new(file.into()); + let hash = ArtifactHash(hasher.finalize().into()); + Ok(Self { file, hash, length }) + } + pub(crate) fn stream( self, ) -> impl Stream> + 'static { @@ -261,7 +311,7 @@ impl CompositeArtifact { target: target_name.clone(), } })? { - let (mut entry, path) = entry + let (entry, path) = entry .and_then(|entry| { let path = entry.header().path()?.into_owned(); let path = Utf8PathBuf::try_from(path) @@ -272,27 +322,15 @@ impl CompositeArtifact { source, target: target_name.clone(), })?; - let mut file = camino_tempfile::tempfile() - .map_err(ErrorKind::CreateTempFile)?; - let mut hasher = Sha256::new(); - let mut length = 0u64; - let mut buf = [0; 8192]; - while let n = entry.read(&mut buf).map_err(|source| { - ErrorKind::ReadCompositeArtifact { - source, - target: target_name.clone(), - } - })? && n > 0 - { - file.write_all(&buf[..n]).map_err(|source| { - ErrorKind::WriteFile { source, path: None } + let mut entry = BufReader::new(entry); + let unpacked_artifact = + UnpackedArtifact::new_blocking(&mut entry, |source| { + ErrorKind::ReadCompositeArtifact { + source, + target: target_name.clone(), + } })?; - hasher.update(&buf[..n]); - length += u64::try_from(n).expect("usize fits in u64"); - } - let file = Arc::new(file.into()); - let hash = ArtifactHash(hasher.finalize().into()); - entries.insert(path, UnpackedArtifact { file, hash, length }); + entries.insert(path, unpacked_artifact); } Ok(Self { entries, original_target_name: target_name }) }); @@ -480,6 +518,49 @@ impl BufRead for MpscReader { } } +async fn generate_installinator_document( + artifacts: &mut Artifacts, + unpacked: &mut Unpacked, + version: ArtifactVersion, + original_target: String, +) -> Result<(), Error> { + let target_name = format!("{original_target}/v2.json"); + let mut document = InstallinatorDocument::default(); + for artifact in artifacts.iter() { + if let Ok(tags) = KnownArtifactTags::from_tags(&artifact.tags) + && let Some(kind) = tags.to_installinator() + && let Some(file_name) = + Utf8Path::new(&artifact.target_name).file_name() + { + document.artifacts.insert(InstallinatorArtifact { + file_name: file_name.to_owned(), + kind, + sha256: artifact.hash, + }); + } + } + + let mut json = serde_json::to_string_pretty(&document) + .map_err(ErrorKind::SerializeInstallinator)?; + json.push('\n'); + let unpacked_artifact = tokio::task::spawn_blocking(move || { + UnpackedArtifact::new_blocking(&mut json.as_bytes(), |_error| { + unreachable!("Read::read for &[u8] does not return an error") + }) + }) + .await??; + + artifacts.insert(Artifact { + target_name: target_name.clone(), + version, + tags: KnownArtifactTags::InstallinatorDocument.to_tags(), + hash: unpacked_artifact.hash, + length: unpacked_artifact.length, + }); + unpacked.entries.insert(target_name, unpacked_artifact); + Ok(()) +} + #[derive(Debug, Deserialize)] pub(crate) struct V1ArtifactsSchema { system_version: Version, From a23d0fe89a2c432e809278c4808380d9d6bd1c4f Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 10 Feb 2026 15:28:06 -0800 Subject: [PATCH 035/137] disallow creating repos with unexpected tag collisions --- artifact/src/artifact.rs | 14 +++++++++++++- artifact/src/tags.rs | 3 +-- lib/src/edit/editor.rs | 22 ++++++++++++++++++++++ lib/src/error.rs | 13 +++++++++++++ 4 files changed, 49 insertions(+), 3 deletions(-) diff --git a/artifact/src/artifact.rs b/artifact/src/artifact.rs index 27ea028..a4ea590 100644 --- a/artifact/src/artifact.rs +++ b/artifact/src/artifact.rs @@ -39,7 +39,7 @@ impl Artifact { } pub fn display_tags(&self) -> DisplayTags<'_> { - DisplayTags(Cow::Borrowed(&self.tags)) + DisplayTags::from(&self.tags) } } @@ -59,6 +59,18 @@ impl Display for DisplayTags<'_> { } } +impl From> for DisplayTags<'static> { + fn from(tags: BTreeMap) -> Self { + Self(Cow::Owned(tags)) + } +} + +impl<'a> From<&'a BTreeMap> for DisplayTags<'a> { + fn from(tags: &'a BTreeMap) -> Self { + Self(Cow::Borrowed(tags)) + } +} + #[derive(Debug, Clone, Deserialize, Serialize)] #[cfg_attr(any(test, feature = "schemars"), derive(schemars::JsonSchema))] pub struct ArtifactId { diff --git a/artifact/src/tags.rs b/artifact/src/tags.rs index e277482..dda16b8 100644 --- a/artifact/src/tags.rs +++ b/artifact/src/tags.rs @@ -2,7 +2,6 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use std::borrow::Cow; use std::collections::BTreeMap; use serde::Deserialize; @@ -49,7 +48,7 @@ pub enum KnownArtifactTags { impl KnownArtifactTags { pub fn display(&self) -> DisplayTags<'static> { - DisplayTags(Cow::Owned(self.to_tags())) + self.to_tags().into() } pub fn from_tags( diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index 0bc0e7a..7977f6b 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -317,6 +317,28 @@ impl<'a> RepositoryEditor<'a> { }) .collect::, Error>>()?; + // Ensure each set of tags is unique, except for known tag sets that + // we expect to be non-unique. + let mut seen_tags = HashMap::new(); + for (target_name, artifact) in &artifacts { + if let Ok(tags) = KnownArtifactTags::from_tags(&artifact.tags) + && matches!(tags, KnownArtifactTags::MeasurementCorpus) + { + continue; + } + + if let Some(first_target_name) = + seen_tags.insert(&artifact.tags, target_name) + { + return Err(ErrorKind::DisallowedTagCollision { + first_target_name: first_target_name.clone(), + second_target_name: target_name.clone(), + tags: artifact.tags.clone(), + } + .into()); + } + } + // Collect all the sha256 hashes and lengths for each source. For file // and fake sources, we want to calculate the hashes in parallel, so // we spawn their calculation tasks on a JoinSet. Sources from borrowed diff --git a/lib/src/error.rs b/lib/src/error.rs index 811090f..29b8afe 100644 --- a/lib/src/error.rs +++ b/lib/src/error.rs @@ -2,6 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use std::collections::BTreeMap; use std::convert::Infallible; use std::error::Error as _; use std::fmt; @@ -27,6 +28,7 @@ macro_rules! try_path { }; } pub(crate) use try_path; +use tufaceous_artifact::DisplayTags; #[derive(Debug, thiserror::Error)] #[error(transparent)] @@ -175,6 +177,16 @@ pub enum ErrorKind { GuessArtifact { path: Utf8PathBuf }, #[error("target name collision on {target_name}")] TargetNameCollision { target_name: String }, + #[error( + "artifacts {first_target_name} and {second_target_name} + have the same tags {tags}, which is not allowed", + tags = DisplayTags::from(.tags), + )] + DisallowedTagCollision { + first_target_name: String, + second_target_name: String, + tags: BTreeMap, + }, #[error("failed to serialize artifacts document")] SerializeArtifacts(#[source] serde_json::Error), #[error("failed to serialize Installinator document")] @@ -239,6 +251,7 @@ impl ErrorKind { | ErrorKind::Corim { .. } | ErrorKind::GuessArtifact { .. } | ErrorKind::TargetNameCollision { .. } + | ErrorKind::DisallowedTagCollision { .. } | ErrorKind::SerializeArtifacts(_) | ErrorKind::SerializeInstallinator(_) | ErrorKind::NoSigningRoot From 284002cac390ae094e3ffd27df54ae59de5930a1 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 10 Feb 2026 16:08:39 -0800 Subject: [PATCH 036/137] read the artifact version from measurement corpus --- lib/src/edit/editor.rs | 18 +++++++----------- lib/src/edit/guess.rs | 2 -- lib/src/edit/measurement_corpus.rs | 17 ++++++++++++----- lib/src/error.rs | 10 ++++++++-- 4 files changed, 27 insertions(+), 20 deletions(-) diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index 7977f6b..51195fd 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -82,20 +82,16 @@ impl<'a> RepositoryEditor<'a> { path: Utf8PathBuf, ) -> Result { let source = FileSource::open(path).await?; - let input = Input::measurement_corpus( - source, - None, - self.artifact_version.clone()?, - ) - .await?; + let input = Input::measurement_corpus(source, None).await?; Ok(self.insert_input(input)) } - pub fn fake_measurement_corpus(self, hashes: usize) -> Result { - let input = Input::fake_measurement_corpus( - hashes, - self.artifact_version.clone()?, - )?; + pub fn fake_measurement_corpus( + self, + hashes: usize, + version: ArtifactVersion, + ) -> Result { + let input = Input::fake_measurement_corpus(hashes, version)?; Ok(self.insert_input(input)) } diff --git a/lib/src/edit/guess.rs b/lib/src/edit/guess.rs index eeafbf7..51ea674 100644 --- a/lib/src/edit/guess.rs +++ b/lib/src/edit/guess.rs @@ -21,7 +21,6 @@ use crate::error::try_path; pub(crate) struct GuessInput { pub(crate) file_start: Bytes, pub(crate) source: FileSource, - pub(crate) version: ArtifactVersion, } pub(crate) type GuessResult = @@ -48,7 +47,6 @@ impl Input> { let guess_input = GuessInput { file_start: buf.freeze(), source: FileSource::from_file(file, path.clone()), - version, }; let guess_input = match Self::guess_measurement_corpus(guess_input).await? { diff --git a/lib/src/edit/measurement_corpus.rs b/lib/src/edit/measurement_corpus.rs index a4bd133..01e383d 100644 --- a/lib/src/edit/measurement_corpus.rs +++ b/lib/src/edit/measurement_corpus.rs @@ -27,18 +27,24 @@ impl Input> { pub(crate) async fn measurement_corpus( mut source: FileSource, corim: Option, - version: ArtifactVersion, ) -> Result { - let Corim { id, .. } = if let Some(corim) = corim { + let corim = if let Some(corim) = corim { corim } else { let v = source.read_to_end().await?; - try_path!(ciborium::from_reader(v.as_slice()), Corim, &source.path) + try_path!( + ciborium::from_reader(v.as_slice()), + DeserializeCorim, + &source.path + ) }; let sha256 = source.sha256().await?; + let version = + try_path!(corim.get_version(), CorimVersion, &source.path) + .parse()?; Ok(Self::MeasurementCorpus { source: source.into(), - corim_id: id, + corim_id: corim.id, sha256, version, }) @@ -63,7 +69,7 @@ impl Input> { } Err(_) => return Ok(ControlFlow::Continue(input)), }; - Self::measurement_corpus(input.source, corim, input.version) + Self::measurement_corpus(input.source, corim) .await .map(ControlFlow::Break) } @@ -78,6 +84,7 @@ impl Input { builder.vendor("fake-vendor".to_string()); builder.id("fake-measurement-id".to_string()); builder.tag_id("fake-tag-id".to_string()); + builder.version(version.to_string()); for i in 0..hashes { builder.add_hash(format!("layer{i}"), 10, vec![0; 32]); } diff --git a/lib/src/error.rs b/lib/src/error.rs index 29b8afe..2bed542 100644 --- a/lib/src/error.rs +++ b/lib/src/error.rs @@ -172,7 +172,12 @@ pub enum ErrorKind { #[error(transparent)] ArtifactVersion(#[from] tufaceous_artifact::ArtifactVersionError), #[error("failed to read {path} as CoRIM")] - Corim { source: ciborium::de::Error, path: Utf8PathBuf }, + DeserializeCorim { + source: ciborium::de::Error, + path: Utf8PathBuf, + }, + #[error("failed to read version from CoRIM document {path}")] + CorimVersion { source: rats_corim::Error, path: Utf8PathBuf }, #[error("failed to guess what kind of artifact {path} is")] GuessArtifact { path: Utf8PathBuf }, #[error("target name collision on {target_name}")] @@ -248,7 +253,8 @@ impl ErrorKind { | ErrorKind::Ed25519Generate | ErrorKind::KeyId(_) | ErrorKind::RoleVerify(_) - | ErrorKind::Corim { .. } + | ErrorKind::DeserializeCorim { .. } + | ErrorKind::CorimVersion { .. } | ErrorKind::GuessArtifact { .. } | ErrorKind::TargetNameCollision { .. } | ErrorKind::DisallowedTagCollision { .. } From dae6f915d3eb67f32bc4e2638bd5411102724a84 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 10 Feb 2026 16:31:17 -0800 Subject: [PATCH 037/137] better corim errors / partial guess test --- lib/src/edit/measurement_corpus.rs | 68 +++++++++++++++++------ lib/src/error.rs | 16 ++---- lib/tests/data/corim-all-sp-v1.0.55.cbor | Bin 0 -> 829 bytes 3 files changed, 55 insertions(+), 29 deletions(-) create mode 100644 lib/tests/data/corim-all-sp-v1.0.55.cbor diff --git a/lib/src/edit/measurement_corpus.rs b/lib/src/edit/measurement_corpus.rs index 01e383d..b1eeb9b 100644 --- a/lib/src/edit/measurement_corpus.rs +++ b/lib/src/edit/measurement_corpus.rs @@ -4,8 +4,6 @@ use std::ops::ControlFlow; -use bytes::BufMut; -use bytes::BytesMut; use rats_corim::Corim; use rats_corim::CorimBuilder; use sha2::Digest; @@ -32,16 +30,11 @@ impl Input> { corim } else { let v = source.read_to_end().await?; - try_path!( - ciborium::from_reader(v.as_slice()), - DeserializeCorim, - &source.path - ) + try_path!(Corim::from_bytes(v.as_slice()), ReadCorim, &source.path) }; let sha256 = source.sha256().await?; let version = - try_path!(corim.get_version(), CorimVersion, &source.path) - .parse()?; + try_path!(corim.get_version(), ReadCorim, &source.path).parse()?; Ok(Self::MeasurementCorpus { source: source.into(), corim_id: corim.id, @@ -62,12 +55,15 @@ impl Input> { Err(ciborium::de::Error::Io(err)) if err.kind() == std::io::ErrorKind::UnexpectedEof => { - // This was plausibly a CoRIM manifest until we hit the end of - // the buffer, indicating a very high likelihood that if we read - // the entire thing it'd still be a CoRIM manifest. + // This was plausibly a CoRIM manifest until we hit the end + // of the buffer, indicating a very high likelihood that if + // we read the entire thing it'd still be a CoRIM manifest. None } - Err(_) => return Ok(ControlFlow::Continue(input)), + Err(error) => { + eprintln!("{error:?}"); + return Ok(ControlFlow::Continue(input)); + } }; Self::measurement_corpus(input.source, corim) .await @@ -91,11 +87,9 @@ impl Input { let corim = builder .build() .map_err(ErrorKind::GenerateFakeMeasurementCorpus)?; - - let mut writer = BytesMut::new().writer(); - ciborium::into_writer(&corim, &mut writer) + let bytes = corim + .to_vec() .map_err(ErrorKind::SerializeFakeMeasurementCorpus)?; - let bytes = writer.into_inner().freeze(); Ok(Input::MeasurementCorpus { corim_id: corim.id, sha256: ArtifactHash(Sha256::digest(&bytes).into()), @@ -104,3 +98,43 @@ impl Input { }) } } + +#[cfg(test)] +mod tests { + use camino::Utf8Path; + use futures_util::TryStreamExt; + use futures_util::pin_mut; + + use crate::edit::guess::GuessInput; + use crate::edit::input::Input; + use crate::edit::source::FileSource; + use crate::error::Error; + + #[tokio::test] + async fn guess_partial_input() -> Result<(), Error> { + const TRUNCATE_LEN: usize = 16; + + let path = Utf8Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/data/corim-all-sp-v1.0.55.cbor"); + let metadata = tokio::fs::metadata(&path).await.unwrap(); + assert!(usize::try_from(metadata.len()).unwrap() > TRUNCATE_LEN); + + let mut source = FileSource::open(path).await?; + // Read the first chunk from the stream, but truncate it to ensure + // we hit the match arm where we correctly guess that this was an + // incomplete read. + let file_start = { + let stream = source.stream(); + pin_mut!(stream); + stream.try_next().await?.unwrap().split_to(TRUNCATE_LEN) + }; + let input = + Input::guess_measurement_corpus(GuessInput { file_start, source }) + .await? + .break_value() + .unwrap(); + assert!(matches!(input, Input::MeasurementCorpus { .. })); + + Ok(()) + } +} diff --git a/lib/src/error.rs b/lib/src/error.rs index 2bed542..7d4e26c 100644 --- a/lib/src/error.rs +++ b/lib/src/error.rs @@ -140,12 +140,12 @@ pub enum ErrorKind { }, #[error("failed to generate fake hubris archive")] GenerateFakeHubrisArchive(#[source] hubtools::Error), + #[error("failed to read CoRIM manifest {path}")] + ReadCorim { source: rats_corim::Error, path: Utf8PathBuf }, #[error("failed to generate fake measurement corpus")] GenerateFakeMeasurementCorpus(#[source] rats_corim::Error), #[error("failed to serialize fake measurement corpus")] - SerializeFakeMeasurementCorpus( - #[source] ciborium::ser::Error, - ), + SerializeFakeMeasurementCorpus(#[source] rats_corim::Error), #[error("failed to generate fake zone image")] GenerateFakeZoneImage(#[source] std::io::Error), @@ -171,13 +171,6 @@ pub enum ErrorKind { #[error(transparent)] ArtifactVersion(#[from] tufaceous_artifact::ArtifactVersionError), - #[error("failed to read {path} as CoRIM")] - DeserializeCorim { - source: ciborium::de::Error, - path: Utf8PathBuf, - }, - #[error("failed to read version from CoRIM document {path}")] - CorimVersion { source: rats_corim::Error, path: Utf8PathBuf }, #[error("failed to guess what kind of artifact {path} is")] GuessArtifact { path: Utf8PathBuf }, #[error("target name collision on {target_name}")] @@ -248,13 +241,12 @@ impl ErrorKind { | ErrorKind::ReadStream(_) | ErrorKind::GenerateFakeHubrisArchive(_) | ErrorKind::GenerateFakeMeasurementCorpus(_) + | ErrorKind::ReadCorim { .. } | ErrorKind::SerializeFakeMeasurementCorpus(_) | ErrorKind::GenerateFakeZoneImage(_) | ErrorKind::Ed25519Generate | ErrorKind::KeyId(_) | ErrorKind::RoleVerify(_) - | ErrorKind::DeserializeCorim { .. } - | ErrorKind::CorimVersion { .. } | ErrorKind::GuessArtifact { .. } | ErrorKind::TargetNameCollision { .. } | ErrorKind::DisallowedTagCollision { .. } diff --git a/lib/tests/data/corim-all-sp-v1.0.55.cbor b/lib/tests/data/corim-all-sp-v1.0.55.cbor new file mode 100644 index 0000000000000000000000000000000000000000..3a9c348403af538fcce58c9c616c2651b8225356 GIT binary patch literal 829 zcmZ3)P?DTql$ooWn3JPhT%cQKsAr&OYRcFc$*g#j@z)~8MGSdpQfUw+EDISLn-(%G zWGwTq$V^F9aL&&yC@o1X0y7fxD*F~Oq$lSW=jQ7sGA?9lY~qSgxN-I9p^oIzol3cn zzwdsuOl!N%$+B0%pI+=y4b7APt+;0qLpD^0ZcbtnOqX?M_B@ZGXJLQJiav@@UB~`U z?!w7`*PmW`+oq?u|I+^3aC_i7Y$o5~IjyL4+~wFuAC8c;H!%X63tlbmEw~@wyw+*^ z*AlorNyzpnx4Z3ja#s3Q{a5g|ig(oayT7XM1mriYX3>%r*!$yB$|8o0^vv9x)Dm5| zMK_tIh45ZoplW*L?X-6puh!J&sa)c4ULzA&+hA;G{|9JM4onNOO^(+)7BIYcW%YC2 z0+Nl|{ptjWg z#oy-VYMWJl#FcCX48NtWd|*nJX&E@n)CbHg3x4U=O1DFua;ds?6CaozD)?5 zQs6fA?Y3X;w(H5Bgqo9+8|-!bCM&zoZFbi%xHj*Lj+o61giR^PHvN%Uo2$E%@3NS$ zaQK%fw@=0$#g;PLD@}46dZOdfjy5A~N`={U$#cQw+e~*1q>@h*Mp|lJ5iGwn<=yj` z_Q-{$Gp5xSy++uSifq$^0G^Xg*Ja*(a8ni&2)gwuYs(=Y-}-jlW@ffK;SW>;5jLg4 zY?_|gw}0v8hgOc)BMu&Yv~IJY{eJP8KK}A5%~4ZqSSDRX*p!BBQ`G5QW Date: Tue, 10 Feb 2026 16:35:41 -0800 Subject: [PATCH 038/137] installinator document must contain the artifact version --- artifact/src/installinator.rs | 12 +++++++++++- lib/src/edit/editor.rs | 2 +- lib/src/repo/v1.rs | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/artifact/src/installinator.rs b/artifact/src/installinator.rs index 88bdf17..e340b9e 100644 --- a/artifact/src/installinator.rs +++ b/artifact/src/installinator.rs @@ -9,6 +9,7 @@ use serde::Deserialize; use serde::Serialize; use crate::ArtifactHash; +use crate::ArtifactVersion; /// Artifact-specific information used by Installinator. /// @@ -19,11 +20,20 @@ use crate::ArtifactHash; /// There are no backwards compatibility constraints for this document. The /// version of Installinator that processes this document is the same as the /// version of tufaceous that creates it. -#[derive(Debug, Clone, Default, Deserialize, Serialize)] +#[derive(Debug, Clone, Deserialize, Serialize)] pub struct InstallinatorDocument { + pub system_version: ArtifactVersion, pub artifacts: BTreeSet, } +impl InstallinatorDocument { + /// Creates an Installinator document with the provided system version and + /// an empty list of artifacts. + pub fn empty(system_version: ArtifactVersion) -> Self { + Self { system_version, artifacts: BTreeSet::new() } + } +} + /// Describes an artifact available to Installinator. #[derive( Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize, diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index 51195fd..bc21253 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -446,7 +446,7 @@ pub(crate) fn generate_installinator_document( version: ArtifactVersion, ) -> Result, Error> { let target_name = format!("installinator_document-{version}.json"); - let mut document = InstallinatorDocument::default(); + let mut document = InstallinatorDocument::empty(version.clone()); for (artifact, sha256) in artifacts { let artifact = artifact.as_ref(); if let Ok(tags) = KnownArtifactTags::from_tags(&artifact.tags) diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs index 581d230..20a1a9e 100644 --- a/lib/src/repo/v1.rs +++ b/lib/src/repo/v1.rs @@ -525,7 +525,7 @@ async fn generate_installinator_document( original_target: String, ) -> Result<(), Error> { let target_name = format!("{original_target}/v2.json"); - let mut document = InstallinatorDocument::default(); + let mut document = InstallinatorDocument::empty(version.clone()); for artifact in artifacts.iter() { if let Ok(tags) = KnownArtifactTags::from_tags(&artifact.tags) && let Some(kind) = tags.to_installinator() From 38e45c7350270940bf7377a79c201034f1cb9f8f Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 12 Feb 2026 08:56:31 -0800 Subject: [PATCH 039/137] impl Display for Root --- lib/src/edit/root.rs | 52 +++++++++++++++++++++++++++++++++++++++----- lib/src/error.rs | 3 +++ 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/lib/src/edit/root.rs b/lib/src/edit/root.rs index ed5dee0..da35189 100644 --- a/lib/src/edit/root.rs +++ b/lib/src/edit/root.rs @@ -3,6 +3,7 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use std::collections::HashMap; +use std::fmt::Display; use std::num::NonZero; use aws_lc_rs::rand::SystemRandom; @@ -21,22 +22,63 @@ use tough::schema::Signed; use crate::error::Error; use crate::error::ErrorKind; -#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] -#[serde(transparent)] -pub struct Root(Signed); +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Root { + inner: Signed, + buffer: String, +} impl Root { pub async fn generate( keys: &[Box], expires: DateTime, ) -> Result { - Ok(Self(generate_root(keys, expires).await?.signed().clone())) + let inner = generate_root(keys, expires).await?.signed().clone(); + let mut buffer = serde_json::to_string_pretty(&inner) + .map_err(ErrorKind::SerializeRoot)?; + buffer.push('\n'); + Ok(Self { inner, buffer }) } pub fn verify_self_signed(&self) -> Result<(), Error> { - self.0.signed.verify_role(&self.0).map_err(ErrorKind::RoleVerify)?; + self.inner + .signed + .verify_role(&self.inner) + .map_err(ErrorKind::RoleVerify)?; Ok(()) } + + pub fn as_str(&self) -> &str { + &self.buffer + } +} + +impl Display for Root { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.buffer) + } +} + +impl Serialize for Root { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + self.inner.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for Root { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let inner = >::deserialize(deserializer)?; + let mut buffer = serde_json::to_string_pretty(&inner) + .map_err(serde::de::Error::custom)?; + buffer.push('\n'); + Ok(Self { inner, buffer }) + } } pub(crate) async fn generate_root( diff --git a/lib/src/error.rs b/lib/src/error.rs index 7d4e26c..3f436ec 100644 --- a/lib/src/error.rs +++ b/lib/src/error.rs @@ -153,6 +153,8 @@ pub enum ErrorKind { Ed25519Generate, #[error("failed to calculate TUF key ID")] KeyId(#[source] tough::schema::Error), + #[error("failed to serialize root role")] + SerializeRoot(#[source] serde_json::Error), #[error("role verification failed")] RoleVerify(#[source] tough::schema::Error), @@ -246,6 +248,7 @@ impl ErrorKind { | ErrorKind::GenerateFakeZoneImage(_) | ErrorKind::Ed25519Generate | ErrorKind::KeyId(_) + | ErrorKind::SerializeRoot(_) | ErrorKind::RoleVerify(_) | ErrorKind::GuessArtifact { .. } | ErrorKind::TargetNameCollision { .. } From cf894c5f171424e0190063c882eecb0b46b17f82 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 12 Feb 2026 13:01:14 -0800 Subject: [PATCH 040/137] impl Clone for RepositoryEditor --- Cargo.lock | 1 - Cargo.toml | 1 - lib/Cargo.toml | 1 - lib/src/edit/editor.rs | 2 +- lib/src/edit/guess.rs | 2 +- lib/src/edit/hubris_archive.rs | 21 ++--- lib/src/edit/measurement_corpus.rs | 6 +- lib/src/edit/os_images.rs | 3 +- lib/src/edit/source.rs | 120 +++++++++++++++++++---------- lib/src/edit/zone_image.rs | 2 +- lib/src/schema.rs | 2 +- 11 files changed, 101 insertions(+), 60 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ae5c828..5a16eb0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2853,7 +2853,6 @@ dependencies = [ "thiserror 2.0.12", "tokio", "tokio-test", - "tokio-util", "tough", "tufaceous-artifact", "tufaceous-brand-metadata", diff --git a/Cargo.toml b/Cargo.toml index 6506aea..fed19f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,7 +50,6 @@ test-strategy = "0.4.0" thiserror = "2.0.12" tokio = "1.47.0" tokio-test = "0.4.5" -tokio-util = "0.7.15" # https://github.com/awslabs/tough/pull/917 tough = { git = "https://github.com/iliana/tough.git", branch = "transport-is-sync" } tufaceous.path = "lib" diff --git a/lib/Cargo.toml b/lib/Cargo.toml index adf94e7..b33cb0e 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -27,7 +27,6 @@ slog.workspace = true tar.workspace = true thiserror.workspace = true tokio.workspace = true -tokio-util.workspace = true tough.workspace = true tufaceous-artifact = { workspace = true, features = ["hubtools"] } tufaceous-brand-metadata.workspace = true diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index bc21253..451b3c8 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -37,7 +37,7 @@ use crate::error::ErrorKind; use crate::schema::ArtifactSchema; use crate::schema::ArtifactsSchema; -#[derive(Debug)] +#[derive(Debug, Clone)] #[must_use] pub struct RepositoryEditor<'a> { system_version: Version, diff --git a/lib/src/edit/guess.rs b/lib/src/edit/guess.rs index 51ea674..0d4ba5f 100644 --- a/lib/src/edit/guess.rs +++ b/lib/src/edit/guess.rs @@ -46,7 +46,7 @@ impl Input> { let guess_input = GuessInput { file_start: buf.freeze(), - source: FileSource::from_file(file, path.clone()), + source: FileSource::from_file(file.into_std().await, path.clone()), }; let guess_input = match Self::guess_measurement_corpus(guess_input).await? { diff --git a/lib/src/edit/hubris_archive.rs b/lib/src/edit/hubris_archive.rs index b93601c..6f9497a 100644 --- a/lib/src/edit/hubris_archive.rs +++ b/lib/src/edit/hubris_archive.rs @@ -39,12 +39,12 @@ impl Input> { let tags = try_path!( RotTags::from_caboose(&caboose, slot), ReadCaboose, - source.path + source.path() ); let version = ArtifactVersion::new(try_path!( read_version(&caboose), ReadCaboose, - source.path + source.path() ))?; Ok(Self::Rot { source: source.into(), tags, version }) } @@ -60,12 +60,12 @@ impl Input> { let tags = try_path!( RotBootloaderTags::from_caboose(&caboose), ReadCaboose, - source.path + source.path() ); let version = ArtifactVersion::new(try_path!( read_version(&caboose), ReadCaboose, - source.path + source.path() ))?; Ok(Self::RotBootloader { source: source.into(), tags, version }) } @@ -78,14 +78,17 @@ impl Input> { Some(caboose) => caboose, None => source.read_hubris_caboose().await?, }; - let tags = - try_path!(SpTags::from_caboose(&caboose), ReadCaboose, source.path); - let name = - try_path!(read_name(&caboose), ReadCaboose, source.path).to_owned(); + let tags = try_path!( + SpTags::from_caboose(&caboose), + ReadCaboose, + source.path() + ); + let name = try_path!(read_name(&caboose), ReadCaboose, source.path()) + .to_owned(); let version = ArtifactVersion::new(try_path!( read_version(&caboose), ReadCaboose, - source.path + source.path() ))?; Ok(Self::Sp { source: source.into(), tags, name, version }) } diff --git a/lib/src/edit/measurement_corpus.rs b/lib/src/edit/measurement_corpus.rs index b1eeb9b..105a0e2 100644 --- a/lib/src/edit/measurement_corpus.rs +++ b/lib/src/edit/measurement_corpus.rs @@ -30,11 +30,11 @@ impl Input> { corim } else { let v = source.read_to_end().await?; - try_path!(Corim::from_bytes(v.as_slice()), ReadCorim, &source.path) + try_path!(Corim::from_bytes(v.as_slice()), ReadCorim, source.path()) }; let sha256 = source.sha256().await?; let version = - try_path!(corim.get_version(), ReadCorim, &source.path).parse()?; + try_path!(corim.get_version(), ReadCorim, source.path()).parse()?; Ok(Self::MeasurementCorpus { source: source.into(), corim_id: corim.id, @@ -119,7 +119,7 @@ mod tests { let metadata = tokio::fs::metadata(&path).await.unwrap(); assert!(usize::try_from(metadata.len()).unwrap() > TRUNCATE_LEN); - let mut source = FileSource::open(path).await?; + let source = FileSource::open(path).await?; // Read the first chunk from the stream, but truncate it to ensure // we hit the match arm where we correctly guess that this was an // incomplete read. diff --git a/lib/src/edit/os_images.rs b/lib/src/edit/os_images.rs index 2fd0913..aee6139 100644 --- a/lib/src/edit/os_images.rs +++ b/lib/src/edit/os_images.rs @@ -120,7 +120,8 @@ impl Input> { OsVariant::Host }; - let phase_2 = FileSource::from_file(file, phase_2_path); + let phase_2 = + FileSource::from_file(file.into_std().await, phase_2_path); Some( Self::os_images(variant, path, Some(phase_2), version.clone()) .await, diff --git a/lib/src/edit/source.rs b/lib/src/edit/source.rs index e8a8b7d..cc5213c 100644 --- a/lib/src/edit/source.rs +++ b/lib/src/edit/source.rs @@ -4,8 +4,12 @@ use std::convert::Infallible; use std::pin::Pin; +use std::sync::Arc; +use std::sync::Mutex; use bytes::Bytes; +use bytes::BytesMut; +use camino::Utf8Path; use camino::Utf8PathBuf; use futures_util::Stream; use futures_util::StreamExt; @@ -13,12 +17,12 @@ use futures_util::TryStreamExt; use futures_util::stream; use hubtools::Caboose; use hubtools::RawHubrisArchive; +use rawzip::FileReader; +use rawzip::ReaderAt; use serde::Serialize; use sha2::Digest; use sha2::Sha256; use tokio::fs::File; -use tokio::io::AsyncSeekExt; -use tokio_util::io::ReaderStream; use tufaceous_artifact::ArtifactHash; use crate::Repository; @@ -33,7 +37,7 @@ pub(crate) struct Target<'a> { pub(crate) source: TargetSource<'a>, } -#[derive(Debug)] +#[derive(Debug, Clone)] pub(crate) enum TargetSource<'a> { Bytes(BytesSource), File(FileSource), @@ -51,7 +55,7 @@ impl TargetSource<'_> { } pub(crate) fn stream( - &mut self, + &self, ) -> Pin> + '_>> { match self { TargetSource::Bytes(source) => { @@ -162,27 +166,46 @@ impl BytesSource { } } -#[derive(Debug)] +#[derive(Debug, Clone)] pub(crate) struct FileSource { - file: Box, - pub(crate) path: Utf8PathBuf, - length_sha256: Option<(u64, ArtifactHash)>, + inner: Arc, +} + +#[derive(Debug)] +struct FileSourceInner { + file: FileReader, + path: Utf8PathBuf, + length_sha256: Mutex>, } impl FileSource { pub(crate) async fn open(path: Utf8PathBuf) -> Result { let file = try_path!(File::open(&path).await, OpenFile, path); - Ok(Self::from_file(file, path)) + Ok(Self::from_file(file.into_std().await, path)) + } + + pub(crate) fn from_file(file: std::fs::File, path: Utf8PathBuf) -> Self { + Self { + inner: Arc::new(FileSourceInner { + file: file.into(), + path, + length_sha256: Mutex::new(None), + }), + } } - pub(crate) fn from_file(file: File, path: Utf8PathBuf) -> Self { - Self { file: Box::new(file), path, length_sha256: None } + pub(crate) fn path(&self) -> &Utf8Path { + &self.inner.path } pub(crate) async fn into_target( mut self, ) -> Result, Error> { - let (length, sha256) = match self.length_sha256 { + let inner = { + let guard = self.inner.length_sha256.lock().expect("poisoned"); + *guard + }; + let (length, sha256) = match inner { Some(inner) => inner, None => self.read_impl(None).await?, }; @@ -209,11 +232,18 @@ impl FileSource { ) .await?; let sha256 = ArtifactHash(hasher.finalize().into()); - Ok(*self.length_sha256.insert((length, sha256))) + Ok({ + let mut guard = self.inner.length_sha256.lock().expect("poisoned"); + *guard.insert((length, sha256)) + }) } pub(crate) async fn sha256(&mut self) -> Result { - let (_, sha256) = match self.length_sha256 { + let inner = { + let guard = self.inner.length_sha256.lock().expect("poisoned"); + *guard + }; + let (_, sha256) = match inner { Some(inner) => inner, None => self.read_impl(None).await?, }; @@ -229,42 +259,52 @@ impl FileSource { pub(crate) async fn read_hubris_archive( &mut self, ) -> Result { - RawHubrisArchive::from_vec(self.read_to_end().await?).map_err( - |source| { - ErrorKind::ReadHubrisArchive { source, path: self.path.clone() } - .into() - }, - ) + Ok(try_path!( + RawHubrisArchive::from_vec(self.read_to_end().await?), + ReadHubrisArchive, + &self.inner.path + )) } pub(crate) async fn read_hubris_caboose( &mut self, ) -> Result { - self.read_hubris_archive().await?.read_caboose().map_err(|source| { - ErrorKind::ReadHubrisArchive { source, path: self.path.clone() } - .into() - }) + Ok(try_path!( + self.read_hubris_archive().await?.read_caboose(), + ReadHubrisArchive, + &self.inner.path + )) } - pub(crate) fn stream( - &mut self, - ) -> impl Stream> { - stream::once(async { - try_path!(self.file.rewind().await, SeekFile, self.path.clone()); - Ok::<_, Error>( - ReaderStream::new(&mut self.file) - .map_err(|source| ErrorKind::ReadFile { - source, - path: Some(self.path.clone()), - }) - .err_into::(), - ) - }) - .try_flatten() + pub(crate) fn stream(&self) -> impl Stream> { + let inner = self.inner.clone(); + stream::try_unfold( + (inner, BytesMut::new(), 0), + async |(inner, mut buf, mut offset)| { + tokio::task::spawn_blocking(move || { + if buf.capacity() == 0 { + buf.reserve(8192); + } + buf.resize(buf.capacity(), 0); + let n = try_path!( + inner.file.read_at(&mut buf, offset), + ReadFile, + inner.path.clone() + ); + if n == 0 { + return Ok(None); + } + offset += u64::try_from(n).expect("usize fits in u64"); + buf.truncate(n); + Ok(Some((buf.split().freeze(), (inner, buf, offset)))) + }) + .await? + }, + ) } } -#[derive(Debug)] +#[derive(Debug, Clone)] pub(crate) struct RepositorySource<'a> { pub(crate) repo: &'a Repository, pub(crate) target_name: String, diff --git a/lib/src/edit/zone_image.rs b/lib/src/edit/zone_image.rs index 7b8c97a..cfeda7c 100644 --- a/lib/src/edit/zone_image.rs +++ b/lib/src/edit/zone_image.rs @@ -35,7 +35,7 @@ impl Input> { path.clone(), ) .await?; - let source = FileSource::from_file(file.into(), path); + let source = FileSource::from_file(file, path); Ok(Self::Zone { source: source.into(), tags: ZoneTags { zone_name: layer_info.pkg }, diff --git a/lib/src/schema.rs b/lib/src/schema.rs index 604c117..068b910 100644 --- a/lib/src/schema.rs +++ b/lib/src/schema.rs @@ -22,7 +22,7 @@ impl ArtifactsSchema { pub(crate) const TARGET_NAME: &str = "artifacts-v2.json"; } -#[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, Hash)] pub(crate) struct ArtifactSchema { pub(crate) target_name: String, pub(crate) version: ArtifactVersion, From 3e892c667680d2a1238c22f2fb42f819543b6494 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 12 Feb 2026 15:03:19 -0800 Subject: [PATCH 041/137] nit --- lib/src/loader.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/src/loader.rs b/lib/src/loader.rs index 57ad22e..161d076 100644 --- a/lib/src/loader.rs +++ b/lib/src/loader.rs @@ -3,6 +3,7 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use std::fmt::Debug; +use std::io::Seek; use bytes::Bytes; use camino::Utf8PathBuf; @@ -187,13 +188,14 @@ impl RepositoryLoader { log: &Logger, ) -> Result { let (file, sha256) = if self.compute_archive_sha256 { - let path = archive_path.clone(); + let archive_path = archive_path.clone(); tokio::task::spawn_blocking(move || { let mut hasher = Sha256::new(); try_path!( - std::io::copy(&mut file, &mut hasher), + file.rewind() + .and_then(|()| std::io::copy(&mut file, &mut hasher)), ReadFile, - path + archive_path ); Ok::<_, Error>((file, Some(hasher.finalize().into()))) }) From 3f01e5c1c075eb2421033735f0935f29f86ea212 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 12 Feb 2026 16:29:25 -0800 Subject: [PATCH 042/137] rewrite ZipTransport::stream --- lib/src/zip_transport.rs | 95 ++++++++++++++++++++++++++-------------- 1 file changed, 63 insertions(+), 32 deletions(-) diff --git a/lib/src/zip_transport.rs b/lib/src/zip_transport.rs index 546260b..17368bb 100644 --- a/lib/src/zip_transport.rs +++ b/lib/src/zip_transport.rs @@ -64,9 +64,8 @@ use bytes::Bytes; use bytes::BytesMut; use camino::Utf8PathBuf; use flate2::read::DeflateDecoder; -use futures_util::FutureExt; use futures_util::Stream; -use futures_util::StreamExt; +use futures_util::TryFutureExt; use futures_util::stream; use rawzip::CompressionMethod; use rawzip::FileReader; @@ -78,6 +77,7 @@ use rawzip::path::ZipFilePath; use slog::Logger; use slog::warn; use tokio::sync::mpsc; +use tokio::sync::oneshot; use tough::Transport; use tough::TransportError; use tough::TransportErrorKind; @@ -278,47 +278,78 @@ impl ZipTransport { entry_data: EntryData, url: Url, ) -> impl Stream> { - let (tx, mut rx) = mpsc::channel(1); + // This could be a lot nicer but the ZIP entry reader can't cross in/out + // of `spawn_blocking` because the type has an associated lifetime. + let (outer_tx, mut outer_rx) = mpsc::channel::< + oneshot::Sender>, + >(1); let task = tokio::task::spawn_blocking(move || { - let entry = self.inner.archive.get_entry(entry_data.wayfinder)?; - let mut reader = match entry_data.compression_method { - CompressionMethod::Store => { - let reader = entry.reader(); - Box::new(entry.verifying_reader(reader)) as Box - } - CompressionMethod::Deflate => { - let reader = DeflateDecoder::new(entry.reader()); - Box::new(entry.verifying_reader(reader)) as Box - } - other => { - return Err(ZipTransportError::CompressionMethod(other)); + let Some(inner_tx) = outer_rx.blocking_recv() else { return }; + let mut reader = match self + .inner + .archive + .get_entry(entry_data.wayfinder) + .map_err(ZipTransportError::from) + .and_then(|entry| match entry_data.compression_method { + CompressionMethod::Store => Ok(entry.verifying_reader( + Box::new(entry.reader()) as Box, + )), + CompressionMethod::Deflate => Ok(entry.verifying_reader( + Box::new(DeflateDecoder::new(entry.reader())), + )), + other => Err(ZipTransportError::CompressionMethod(other)), + }) { + Ok(reader) => reader, + Err(error) => { + inner_tx.send(Err(error)).ok(); + return; } }; - let mut buf = BytesMut::zeroed(8192); - while let n = reader.read(&mut buf)? - && n > 0 + let mut buf = BytesMut::new(); + let mut first_tx = Some(inner_tx); + while let Some(inner_tx) = + first_tx.take().or_else(|| outer_rx.blocking_recv()) { - buf.truncate(n); - let Ok(()) = tx.blocking_send(buf.split().freeze()) else { - break; - }; if buf.capacity() == 0 { buf.reserve(8192); } buf.resize(buf.capacity(), 0); + let result = match reader.read(&mut buf) { + Ok(0) => break, + Ok(n) => { + buf.truncate(n); + Ok(buf.split().freeze()) + } + Err(error) => Err(error.into()), + }; + let Ok(()) = inner_tx.send(result) else { break }; } - Ok::<_, ZipTransportError>(()) }); - stream::poll_fn(move |cx| rx.poll_recv(cx)).map(Ok).chain( - task.into_stream().filter_map(move |result| { - let error = match result { - Ok(Ok(())) => return std::future::ready(None), - Ok(Err(error)) => error, - Err(join_error) => join_error.into(), - }; - std::future::ready(Some(Err(error.upgrade(url.clone())))) - }), + stream::try_unfold( + (outer_tx, task, url), + async |(outer_tx, task, url)| { + let (inner_tx, inner_rx) = oneshot::channel(); + match outer_tx + .send(inner_tx) + .map_err(|_send_error| ()) + .and_then(|()| inner_rx.map_err(|_recv_error| ())) + .await + { + Ok(Ok(bytes)) => Ok(Some((bytes, (outer_tx, task, url)))), + Ok(Err(error)) => Err(error.upgrade(url)), + Err(()) => { + // The task hung up. If we're still here, then the task + // reached EOF. await the task and return. + match task.await { + Ok(()) => Ok(None), + Err(error) => { + Err(ZipTransportError::from(error).upgrade(url)) + } + } + } + } + }, ) } } From 865bc277fa5ad71b3d66d5d9e7e9fb76a0f619b0 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 23 Feb 2026 15:52:24 -0800 Subject: [PATCH 043/137] add RepositoryEditor::inconsistent_fake this is useful for the cases where ManifestTweaks was used in v1 --- lib/src/edit/editor.rs | 42 ++++++++++++++---- lib/src/edit/fake.rs | 29 +++++++++--- lib/src/edit/hubris_archive.rs | 9 ++-- lib/src/edit/measurement_corpus.rs | 3 +- lib/src/edit/os_images.rs | 18 +++++--- lib/src/edit/zone_image.rs | 3 +- lib/tests/tests.rs | 71 +++++++++++++++++++++++++++--- 7 files changed, 144 insertions(+), 31 deletions(-) diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index 451b3c8..43fca4e 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -91,7 +91,7 @@ impl<'a> RepositoryEditor<'a> { hashes: usize, version: ArtifactVersion, ) -> Result { - let input = Input::fake_measurement_corpus(hashes, version)?; + let input = Input::fake_measurement_corpus(hashes, version, None)?; Ok(self.insert_input(input)) } @@ -116,8 +116,11 @@ impl<'a> RepositoryEditor<'a> { } pub fn fake_os_image(self, variant: OsVariant) -> Result { - let input = - Input::fake_os_images(variant, self.artifact_version.clone()?); + let input = Input::fake_os_images( + variant, + self.artifact_version.clone()?, + None, + ); Ok(self.insert_input(input)) } @@ -145,8 +148,11 @@ impl<'a> RepositoryEditor<'a> { } pub fn fake_rot_archive(self, tags: RotTags) -> Result { - let input = - Input::fake_rot_archive(tags, self.artifact_version.clone()?)?; + let input = Input::fake_rot_archive( + tags, + self.artifact_version.clone()?, + None, + )?; Ok(self.insert_input(input)) } @@ -157,13 +163,14 @@ impl<'a> RepositoryEditor<'a> { let input = Input::fake_rot_bootloader_archive( tags, self.artifact_version.clone()?, + None, )?; Ok(self.insert_input(input)) } pub fn fake_sp_archive(self, tags: SpTags) -> Result { let input = - Input::fake_sp_archive(tags, self.artifact_version.clone()?)?; + Input::fake_sp_archive(tags, self.artifact_version.clone()?, None)?; Ok(self.insert_input(input)) } @@ -173,7 +180,7 @@ impl<'a> RepositoryEditor<'a> { pub fn fake_zone_image(self, name: String) -> Result { let input = - Input::fake_zone_image(name, self.artifact_version.clone()?)?; + Input::fake_zone_image(name, self.artifact_version.clone()?, None)?; Ok(self.insert_input(input)) } @@ -253,10 +260,29 @@ impl<'a> RepositoryEditor<'a> { self } + /// Create a fake repository for testing purposes. pub fn fake(system_version: Version) -> Result { let mut editor = Self::new(system_version); let version = editor.artifact_version.clone()?; - for input in Input::fake(&version)? { + for input in Input::fake(&version, None)? { + editor = editor.insert_input(input); + } + Ok(editor) + } + + /// Create a fake repository for testing purposes where the system version, + /// artifact versions, and the versions interior to the artifact data are + /// potentially different. + /// + /// The Installinator document, if generated, always uses `system_version` + /// for the artifact and interior versions. + pub fn inconsistent_fake( + system_version: Version, + artifact_version: &ArtifactVersion, + interior_version: &ArtifactVersion, + ) -> Result { + let mut editor = Self::new(system_version); + for input in Input::fake(artifact_version, Some(interior_version))? { editor = editor.insert_input(input); } Ok(editor) diff --git a/lib/src/edit/fake.rs b/lib/src/edit/fake.rs index b17e851..c7562e3 100644 --- a/lib/src/edit/fake.rs +++ b/lib/src/edit/fake.rs @@ -38,7 +38,7 @@ pub trait ArtifactsExt: Sized { impl ArtifactsExt for Artifacts { fn fake(version: ArtifactVersion) -> Result { let mut artifacts = Artifacts::default(); - for input in Input::fake(&version)? { + for input in Input::fake(&version, None)? { for output in input.outputs() { artifacts.extend(output.into_artifact()); } @@ -64,14 +64,24 @@ impl ArtifactsExt for Artifacts { } impl Input { - pub(crate) fn fake(version: &ArtifactVersion) -> Result, Error> { + pub(crate) fn fake( + version: &ArtifactVersion, + interior_version: Option<&ArtifactVersion>, + ) -> Result, Error> { let mut inputs = Vec::new(); for hashes in [4, 16] { - inputs - .push(Self::fake_measurement_corpus(hashes, version.clone())?); + inputs.push(Self::fake_measurement_corpus( + hashes, + version.clone(), + interior_version, + )?); } for variant in [OsVariant::Host, OsVariant::Recovery] { - inputs.push(Self::fake_os_images(variant, version.clone())); + inputs.push(Self::fake_os_images( + variant, + version.clone(), + interior_version, + )); } for slot in [RotSlot::A, RotSlot::B] { inputs.push(Self::fake_rot_archive( @@ -81,6 +91,7 @@ impl Input { rot_slot: slot, }, version.clone(), + interior_version, )?); } inputs.push(Self::fake_rot_bootloader_archive( @@ -89,15 +100,21 @@ impl Input { rot_sign: Sign::UNSIGNED, }, version.clone(), + interior_version, )?); for board in ["fake-gimlet", "fake-cosmo", "fake-sidecar", "fake-psc"] { inputs.push(Self::fake_sp_archive( SpTags { sp_board: board.into() }, version.clone(), + interior_version, )?); } for name in FAKE_ZONES { - inputs.push(Self::fake_zone_image(name.into(), version.clone())?); + inputs.push(Self::fake_zone_image( + name.into(), + version.clone(), + interior_version.cloned(), + )?); } Ok(inputs) } diff --git a/lib/src/edit/hubris_archive.rs b/lib/src/edit/hubris_archive.rs index 6f9497a..ff81aa2 100644 --- a/lib/src/edit/hubris_archive.rs +++ b/lib/src/edit/hubris_archive.rs @@ -133,12 +133,13 @@ impl Input { pub(crate) fn fake_rot_archive( tags: RotTags, version: ArtifactVersion, + interior_version: Option<&ArtifactVersion>, ) -> Result { let data = CabooseData { board: &tags.rot_board, sign: &tags.rot_sign, commit: "this-is-a-fake-rot", - version: &version, + version: interior_version.unwrap_or(&version), }; let source = data.generate_fake_archive()?; Ok(Input::Rot { source, tags, version }) @@ -147,12 +148,13 @@ impl Input { pub(crate) fn fake_rot_bootloader_archive( tags: RotBootloaderTags, version: ArtifactVersion, + interior_version: Option<&ArtifactVersion>, ) -> Result { let data = CabooseData { board: &tags.rot_board, sign: &tags.rot_sign, commit: "this-is-a-fake-rot-bootloader", - version: &version, + version: interior_version.unwrap_or(&version), }; let source = data.generate_fake_archive()?; Ok(Input::RotBootloader { source, tags, version }) @@ -161,12 +163,13 @@ impl Input { pub(crate) fn fake_sp_archive( tags: SpTags, version: ArtifactVersion, + interior_version: Option<&ArtifactVersion>, ) -> Result { let data = CabooseData { board: &tags.sp_board, sign: &Sign::UNSIGNED, commit: "this-is-a-fake-sp", - version: &version, + version: interior_version.unwrap_or(&version), }; let source = data.generate_fake_archive()?; Ok(Input::Sp { source, name: tags.sp_board.clone(), tags, version }) diff --git a/lib/src/edit/measurement_corpus.rs b/lib/src/edit/measurement_corpus.rs index 105a0e2..4193df5 100644 --- a/lib/src/edit/measurement_corpus.rs +++ b/lib/src/edit/measurement_corpus.rs @@ -75,12 +75,13 @@ impl Input { pub(crate) fn fake_measurement_corpus( hashes: usize, version: ArtifactVersion, + interior_version: Option<&ArtifactVersion>, ) -> Result { let mut builder = CorimBuilder::new(); builder.vendor("fake-vendor".to_string()); builder.id("fake-measurement-id".to_string()); builder.tag_id("fake-tag-id".to_string()); - builder.version(version.to_string()); + builder.version(interior_version.unwrap_or(&version).to_string()); for i in 0..hashes { builder.add_hash(format!("layer{i}"), 10, vec![0; 32]); } diff --git a/lib/src/edit/os_images.rs b/lib/src/edit/os_images.rs index aee6139..3766ece 100644 --- a/lib/src/edit/os_images.rs +++ b/lib/src/edit/os_images.rs @@ -178,17 +178,25 @@ impl Input { pub(crate) fn fake_os_images( os_variant: OsVariant, version: ArtifactVersion, + interior_version: Option<&ArtifactVersion>, ) -> Self { + let interior_version = interior_version.unwrap_or(&version); let cosmo_phase_1 = BytesSource::fake_padded( - format!("cosmo {os_variant} OS phase 1 image version {version}\n"), + format!( + "cosmo {os_variant} OS phase 1 image version {interior_version}\n" + ), MIB, ); let gimlet_phase_1 = BytesSource::fake_padded( - format!("gimlet {os_variant} OS phase 1 image version {version}\n"), + format!( + "gimlet {os_variant} OS phase 1 image version {interior_version}\n" + ), MIB, ); let phase_2 = BytesSource::fake_padded( - format!("{os_variant} OS phase 2 image version {version}\n"), + format!( + "{os_variant} OS phase 2 image version {interior_version}\n" + ), 4 * MIB, ); @@ -196,14 +204,14 @@ impl Input { extra_targets.insert( String::from("unix.z"), BytesSource::fake_padded( - format!("{os_variant} OS unix.z version {version}\n"), + format!("{os_variant} OS unix.z version {interior_version}\n"), 64 * KIB, ), ); extra_targets.insert( String::from("cpio.z"), BytesSource::fake_padded( - format!("{os_variant} OS cpio.z version {version}\n"), + format!("{os_variant} OS cpio.z version {interior_version}\n"), 256 * KIB, ), ); diff --git a/lib/src/edit/zone_image.rs b/lib/src/edit/zone_image.rs index cfeda7c..7289595 100644 --- a/lib/src/edit/zone_image.rs +++ b/lib/src/edit/zone_image.rs @@ -66,6 +66,7 @@ impl Input { pub(crate) fn fake_zone_image( zone_name: String, version: ArtifactVersion, + interior_version: Option, ) -> Result { let mut archive = tar::Builder::new(GzEncoder::new( BytesMut::new().writer(), @@ -73,7 +74,7 @@ impl Input { )); let metadata = Metadata::new(ArchiveType::Layer(LayerInfo { pkg: zone_name.clone(), - version: version.clone(), + version: interior_version.unwrap_or_else(|| version.clone()), })); metadata .append_to_tar(&mut archive, 0) diff --git a/lib/tests/tests.rs b/lib/tests/tests.rs index f4db75b..39b2c22 100644 --- a/lib/tests/tests.rs +++ b/lib/tests/tests.rs @@ -16,13 +16,15 @@ use tufaceous::RepositoryLoader; use tufaceous::TrustStoreBehavior; use tufaceous::edit::RepositoryEditor; use tufaceous::error::Error; +use tufaceous_artifact::KnownArtifactTags; -const VERSION: Version = Version::new(1, 0, 0); +const V1: Version = Version::new(1, 0, 0); +const V2: Version = Version::new(2, 0, 0); #[tokio::test] async fn it_works() -> Result<(), Error> { let log = slog::Logger::root(slog::Discard, slog::o!()); - let zip = RepositoryEditor::fake(VERSION)? + let zip = RepositoryEditor::fake(V1)? .finish() .await? .generate_root() @@ -47,7 +49,7 @@ async fn it_works() -> Result<(), Error> { #[tokio::test] async fn verify_targets() -> Result<(), Error> { let log = slog::Logger::root(slog::Discard, slog::o!()); - let mut zip = RepositoryEditor::fake(VERSION)? + let mut zip = RepositoryEditor::fake(V1)? .finish() .await? .generate_root() @@ -82,7 +84,7 @@ async fn verify_targets() -> Result<(), Error> { #[tokio::test] async fn no_artifacts() -> Result<(), Error> { let log = slog::Logger::root(slog::Discard, slog::o!()); - let zip = RepositoryEditor::new(VERSION) + let zip = RepositoryEditor::new(V1) .generate_installinator_document(false) .finish() .await? @@ -103,12 +105,12 @@ async fn no_artifacts() -> Result<(), Error> { #[tokio::test] async fn empty_artifact() -> Result<(), Error> { let log = slog::Logger::root(slog::Discard, slog::o!()); - let zip = RepositoryEditor::new(VERSION) + let zip = RepositoryEditor::new(V1) .generate_installinator_document(false) .fake_artifact( "empty.img".to_owned(), "1.0.0".parse().unwrap(), - &tufaceous_artifact::KnownArtifactTags::InstallinatorDocument, + &KnownArtifactTags::InstallinatorDocument, 0, ) .finish() @@ -134,10 +136,65 @@ async fn empty_artifact() -> Result<(), Error> { Ok(()) } +#[tokio::test] +async fn inconsistent_fake() -> Result<(), Error> { + let log = slog::Logger::root(slog::Discard, slog::o!()); + let zip = RepositoryEditor::fake(V1)? + .finish() + .await? + .generate_root() + .sign() + .await? + .write_zip(Vec::new(), Utc::now()) + .await?; + let repo1 = RepositoryLoader::new() + .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .load_zip_buffer(zip, &log) + .await?; + + let artifact_v1 = V1.to_string().parse()?; + let artifact_v2 = V2.to_string().parse()?; + let zip = + RepositoryEditor::inconsistent_fake(V2, &artifact_v2, &artifact_v1)? + .finish() + .await? + .generate_root() + .sign() + .await? + .write_zip(Vec::new(), Utc::now()) + .await?; + let repo2 = RepositoryLoader::new() + .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .load_zip_buffer(zip, &log) + .await?; + + for (first, second) in + repo1.artifacts().iter().zip(repo2.artifacts().iter()) + { + // Each artifact should have the same tags. + assert_eq!(first.tags, second.tags); + // If this is the Installinator document, skip the rest of the checks. + // The Installinator document's version is always the system version. + if first.known_tags() == Some(KnownArtifactTags::InstallinatorDocument) + { + continue; + } + // The first artifact should be 1.0.0, and the second should be 2.0.0. + assert_eq!(first.version, artifact_v1); + assert_eq!(second.version, artifact_v2); + // The artifacts should have the same contents (hash and length), as they + // should both have an interior version of 1.0.0. + assert_eq!(first.hash, second.hash); + assert_eq!(first.length, second.length); + } + + Ok(()) +} + #[tokio::test] async fn compute_archive_sha256() -> Result<(), Error> { let log = slog::Logger::root(slog::Discard, slog::o!()); - let zip = RepositoryEditor::fake(VERSION)? + let zip = RepositoryEditor::fake(V1)? .finish() .await? .generate_root() From 7cb0befa5727447c3261dc1bcb23c50a63d3e283 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 23 Feb 2026 23:22:26 -0800 Subject: [PATCH 044/137] deal with weird zone file names --- lib/src/edit/editor.rs | 14 +++++++++++--- lib/src/edit/fake.rs | 29 +++++++++++++++-------------- lib/src/edit/input.rs | 6 +++--- lib/src/edit/zone_image.rs | 12 ++++++++++++ 4 files changed, 41 insertions(+), 20 deletions(-) diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index 43fca4e..cd6b2ad 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -178,9 +178,17 @@ impl<'a> RepositoryEditor<'a> { Ok(self.insert_input(Input::zone_image(path).await?)) } - pub fn fake_zone_image(self, name: String) -> Result { - let input = - Input::fake_zone_image(name, self.artifact_version.clone()?, None)?; + pub fn fake_zone_image( + self, + zone_name: String, + file_name: String, + ) -> Result { + let input = Input::fake_zone_image( + zone_name, + file_name, + self.artifact_version.clone()?, + None, + )?; Ok(self.insert_input(input)) } diff --git a/lib/src/edit/fake.rs b/lib/src/edit/fake.rs index c7562e3..3b578c6 100644 --- a/lib/src/edit/fake.rs +++ b/lib/src/edit/fake.rs @@ -17,18 +17,18 @@ use crate::edit::source::BytesSource; use crate::error::Error; use crate::schema::ArtifactSchema; -const FAKE_ZONES: [&str; 11] = [ - "clickhouse", - "clickhouse_keeper", - "clickhouse_server", - "cockroachdb", - "crucible-zone", - "crucible-pantry-zone", - "external-dns", - "internal-dns", - "ntp", - "nexus", - "oximeter", +const FAKE_ZONES: [(&str, &str); 11] = [ + ("clickhouse", "clickhouse.tar.gz"), + ("clickhouse_keeper", "clickhouse_keeper.tar.gz"), + ("clickhouse_server", "clickhouse_server.tar.gz"), + ("cockroachdb", "cockroachdb.tar.gz"), + ("crucible-zone", "crucible.tar.gz"), + ("crucible-pantry-zone", "crucible_pantry.tar.gz"), + ("external-dns", "external_dns.tar.gz"), + ("internal-dns", "internal_dns.tar.gz"), + ("ntp", "ntp.tar.gz"), + ("nexus", "nexus.tar.gz"), + ("oximeter", "oximeter.tar.gz"), ]; pub trait ArtifactsExt: Sized { @@ -109,9 +109,10 @@ impl Input { interior_version, )?); } - for name in FAKE_ZONES { + for (zone_name, file_name) in FAKE_ZONES { inputs.push(Self::fake_zone_image( - name.into(), + zone_name.into(), + file_name.into(), version.clone(), interior_version.cloned(), )?); diff --git a/lib/src/edit/input.rs b/lib/src/edit/input.rs index 0565362..8802e22 100644 --- a/lib/src/edit/input.rs +++ b/lib/src/edit/input.rs @@ -60,6 +60,7 @@ pub(crate) enum Input { }, Zone { source: Source, + file_name: String, tags: ZoneTags, version: ArtifactVersion, }, @@ -145,9 +146,8 @@ impl Input { vec![Output::extra(target_name, source)] } } - Input::Zone { source, tags, version } => { - let target_name = - format!("zones/{name}.tar.gz", name = tags.zone_name); + Input::Zone { source, file_name, tags, version } => { + let target_name = format!("zones/{file_name}"); vec![Output::new(target_name, version, tags.into(), source)] } } diff --git a/lib/src/edit/zone_image.rs b/lib/src/edit/zone_image.rs index 7289595..f0ba316 100644 --- a/lib/src/edit/zone_image.rs +++ b/lib/src/edit/zone_image.rs @@ -30,6 +30,10 @@ impl Input> { pub(crate) async fn zone_image(path: Utf8PathBuf) -> Result { let file = try_path!(tokio::fs::File::open(&path).await, OpenFile, path); + let file_name = path + .file_name() + .expect("a path to an opened file must have a file name") + .to_string(); let (file, layer_info) = crate::util::read_zone_layer_info( file.into_std().await, path.clone(), @@ -38,6 +42,7 @@ impl Input> { let source = FileSource::from_file(file, path); Ok(Self::Zone { source: source.into(), + file_name, tags: ZoneTags { zone_name: layer_info.pkg }, version: layer_info.version, }) @@ -54,8 +59,13 @@ impl Input> { else { return Ok(ControlFlow::Continue(input)); }; + let Some(file_name) = input.source.path().file_name() else { + return Ok(ControlFlow::Continue(input)); + }; + let file_name = file_name.to_string(); Ok(ControlFlow::Break(Self::Zone { source: input.source.into(), + file_name, tags: ZoneTags { zone_name: layer_info.pkg }, version: layer_info.version, })) @@ -65,6 +75,7 @@ impl Input> { impl Input { pub(crate) fn fake_zone_image( zone_name: String, + file_name: String, version: ArtifactVersion, interior_version: Option, ) -> Result { @@ -87,6 +98,7 @@ impl Input { .freeze(); Ok(Input::Zone { source: BytesSource::new(bytes), + file_name, tags: ZoneTags { zone_name }, version, }) From e478d7728f4b0ff6897fc2ddad1ab46514bb70b9 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 23 Feb 2026 23:49:41 -0800 Subject: [PATCH 045/137] update fake sp/rot tags for what omicron tests expect --- artifact/src/sign.rs | 4 ++-- lib/src/edit/fake.rs | 13 ++++++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/artifact/src/sign.rs b/artifact/src/sign.rs index 34726b5..0cf8c19 100644 --- a/artifact/src/sign.rs +++ b/artifact/src/sign.rs @@ -61,8 +61,8 @@ pub struct Sign(pub Option); impl Sign { pub const UNSIGNED: Sign = Sign(None); - pub fn new(sign: Option) -> Self { - Self(sign) + pub fn signed(sign: String) -> Self { + Self(Some(sign)) } pub fn is_signed(&self) -> bool { diff --git a/lib/src/edit/fake.rs b/lib/src/edit/fake.rs index 3b578c6..0372b28 100644 --- a/lib/src/edit/fake.rs +++ b/lib/src/edit/fake.rs @@ -17,6 +17,9 @@ use crate::edit::source::BytesSource; use crate::error::Error; use crate::schema::ArtifactSchema; +const FAKE_SIGN: &str = + "0000000000000000000000000000000000000000000000000000000000000000"; + const FAKE_ZONES: [(&str, &str); 11] = [ ("clickhouse", "clickhouse.tar.gz"), ("clickhouse_keeper", "clickhouse_keeper.tar.gz"), @@ -86,8 +89,8 @@ impl Input { for slot in [RotSlot::A, RotSlot::B] { inputs.push(Self::fake_rot_archive( RotTags { - rot_board: "fake-rot".into(), - rot_sign: Sign::UNSIGNED, + rot_board: "SimRot".into(), + rot_sign: Sign::signed(FAKE_SIGN.into()), rot_slot: slot, }, version.clone(), @@ -96,13 +99,13 @@ impl Input { } inputs.push(Self::fake_rot_bootloader_archive( RotBootloaderTags { - rot_board: "fake-rot".into(), - rot_sign: Sign::UNSIGNED, + rot_board: "SimRot".into(), + rot_sign: Sign::signed(FAKE_SIGN.into()), }, version.clone(), interior_version, )?); - for board in ["fake-gimlet", "fake-cosmo", "fake-sidecar", "fake-psc"] { + for board in ["SimGimletSp", "SimCosmoSp", "SimSidecarSp", "SimPscSp"] { inputs.push(Self::fake_sp_archive( SpTags { sp_board: board.into() }, version.clone(), From be9c1e7e2981a2c92f91c1c72692653d29164923 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 5 Mar 2026 17:52:43 -0800 Subject: [PATCH 046/137] implement strongly-typed repo metadata --- artifact/src/artifact.rs | 4 +- artifact/src/lib.rs | 3 ++ artifact/src/map.rs | 85 ++++++++++++++++++++++++++++++++++++++++ artifact/src/metadata.rs | 71 +++++++++++++++++++++++++++++++++ artifact/src/tags.rs | 60 +++++++++++++++++----------- lib/src/edit/editor.rs | 15 +++++-- lib/src/repo.rs | 5 +++ lib/src/repo/v1.rs | 2 +- lib/src/schema.rs | 7 ++++ 9 files changed, 224 insertions(+), 28 deletions(-) create mode 100644 artifact/src/map.rs create mode 100644 artifact/src/metadata.rs diff --git a/artifact/src/artifact.rs b/artifact/src/artifact.rs index a4ea590..73463ce 100644 --- a/artifact/src/artifact.rs +++ b/artifact/src/artifact.rs @@ -35,7 +35,7 @@ impl Artifact { } pub fn known_tags(&self) -> Option { - KnownArtifactTags::from_tags(&self.tags).ok() + KnownArtifactTags::from_tags(self.tags.clone()).ok() } pub fn display_tags(&self) -> DisplayTags<'_> { @@ -80,7 +80,7 @@ pub struct ArtifactId { impl ArtifactId { pub fn known_tags(&self) -> Option { - KnownArtifactTags::from_tags(&self.tags).ok() + KnownArtifactTags::from_tags(self.tags.clone()).ok() } pub fn display_tags(&self) -> DisplayTags<'_> { diff --git a/artifact/src/lib.rs b/artifact/src/lib.rs index 1a97cf7..71ea29e 100644 --- a/artifact/src/lib.rs +++ b/artifact/src/lib.rs @@ -6,6 +6,8 @@ mod artifact; mod hash; pub mod hubris; mod installinator; +mod map; +mod metadata; mod sign; mod tags; mod version; @@ -13,6 +15,7 @@ mod version; pub use artifact::*; pub use hash::*; pub use installinator::*; +pub use metadata::*; pub use sign::*; pub use tags::*; pub use version::*; diff --git a/artifact/src/map.rs b/artifact/src/map.rs new file mode 100644 index 0000000..2869d54 --- /dev/null +++ b/artifact/src/map.rs @@ -0,0 +1,85 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::BTreeMap; + +use serde::Serialize; +use serde::de::DeserializeOwned; + +pub(crate) fn from_map( + map: BTreeMap, +) -> Result { + serde_json::from_value(map.into_iter().collect()) +} + +pub(crate) fn to_map(s: &S) -> BTreeMap { + // Call `to_map_impl`. On error, panic if debug assertions are enabled, + // otherwise return `BTreeMap::default()`. + match to_map_impl(s) { + Ok(map) => map, + Err(err) => { + if cfg!(debug_assertions) { + panic!( + "serializing {} to map should always succeed: {err:?}", + std::any::type_name::() + ); + } else { + BTreeMap::default() + } + } + } +} + +fn to_map_impl( + s: &S, +) -> Result, ToMapError> { + let value = serde_json::to_value(s)?; + let serde_json::Value::Object(map) = value else { + return Err(ToMapError::StructNotObject(Box::new(value))); + }; + map.into_iter() + .map(|(key, value)| match value { + serde_json::Value::String(value) => Ok((key, value)), + _ => { + Err(ToMapError::ValueNotString { key, value: Box::new(value) }) + } + }) + .collect() +} + +#[derive(Debug, thiserror::Error)] +enum ToMapError { + #[error("failed to serialize struct")] + Serialize(#[from] serde_json::Error), + #[error("struct serialized to {0:?}, not an object")] + StructNotObject(Box), + #[error("value for {key:?} serialized to {value:?}, not a string")] + ValueNotString { key: String, value: Box }, +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use serde::Deserialize; + + #[test] + fn from_optional_string() { + #[derive(Debug, PartialEq, Deserialize)] + struct Map { + optional: Option, + } + + let mut tags = BTreeMap::new(); + assert_eq!( + super::from_map::(tags.clone()).unwrap(), + Map { optional: None } + ); + tags.insert("optional".to_owned(), "value".to_owned()); + assert_eq!( + super::from_map::(tags).unwrap(), + Map { optional: Some("value".to_owned()) } + ); + } +} diff --git a/artifact/src/metadata.rs b/artifact/src/metadata.rs new file mode 100644 index 0000000..baf82a6 --- /dev/null +++ b/artifact/src/metadata.rs @@ -0,0 +1,71 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::BTreeMap; + +use serde::Deserialize; +use serde::Serialize; + +/// Structured repository-level metadata stored in `artifacts-v2.json`. +/// +/// Similar to [`KnownArtifactTags`], this struct must serialize to and +/// deserialize from a mapping of string keys to string values. Additionally, it +/// is a requirement that all fields are optional; it must be possible to +/// deserialize this struct from older metadata, and the oldest metadata +/// is no metadata at all. +/// +/// As such, a couple of recommendations for adding new fields: +/// 1. Wrap all fields in [`Option`] and mark with +/// `#[serde(skip_serializing_if = "Option::is_none")]`. +/// 2. Mark nested structs with `#[serde(flatten)]`. +/// +/// [`KnownArtifactTags`]: crate::KnownArtifactTags +#[derive(Debug, Default, Clone, PartialEq, Eq, Deserialize, Serialize)] +#[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] +pub struct Metadata { + // We don't actually have any metadata yet, but we wanted to set up the + // scaffolding to make it possible in the future. +} + +impl Metadata { + /// Deserialize structured metadata from a string mapping. + pub fn from_map( + map: BTreeMap, + ) -> Result { + crate::map::from_map(map) + } + + /// Serialize structured metadata into a string mapping. + pub fn to_map(&self) -> BTreeMap { + crate::map::to_map(self) + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use test_strategy::proptest; + + use crate::Metadata; + + /// **Do not change this test when adding new metadata.** This structure + /// must be capable of deserializing older metadata, and the oldest metadata + /// is no metadata at all. + #[test] + fn deserialize_from_empty() { + Metadata::from_map(BTreeMap::new()).unwrap(); + } + + #[test] + fn default_is_empty() { + assert!(Metadata::default().to_map().is_empty()); + } + + #[proptest] + fn metadata_roundtrip(metadata: Metadata) { + let map = metadata.to_map(); + assert_eq!(Metadata::from_map(map).unwrap(), metadata); + } +} diff --git a/artifact/src/tags.rs b/artifact/src/tags.rs index dda16b8..b4a3547 100644 --- a/artifact/src/tags.rs +++ b/artifact/src/tags.rs @@ -6,7 +6,6 @@ use std::collections::BTreeMap; use serde::Deserialize; use serde::Serialize; -use serde::de::value::MapDeserializer; use crate::DisplayTags; use crate::InstallinatorArtifactKind; @@ -52,26 +51,13 @@ impl KnownArtifactTags { } pub fn from_tags( - tags: &BTreeMap, - ) -> Result { - Self::deserialize(MapDeserializer::new( - tags.iter().map(|(k, v)| (k.as_str(), v.as_str())), - )) + tags: BTreeMap, + ) -> Result { + crate::map::from_map(tags) } pub fn to_tags(&self) -> BTreeMap { - self.to_tags_impl().unwrap_or_default() - } - - fn to_tags_impl(&self) -> Option> { - let value = serde_json::to_value(self).ok()?; - let serde_json::Value::Object(map) = value else { return None }; - map.into_iter() - .map(|(k, v)| match v { - serde_json::Value::String(v) => Some((k, v)), - _ => None, - }) - .collect() + crate::map::to_map(self) } pub fn to_installinator(&self) -> Option { @@ -276,15 +262,45 @@ impl From for KnownArtifactTags { #[cfg(test)] mod tests { + use std::collections::BTreeMap; + use test_strategy::proptest; use crate::KnownArtifactTags; + use crate::RotSlot; + use crate::RotTags; + use crate::Sign; #[proptest] fn tags_roundtrip(tags: KnownArtifactTags) { - let tag_map = tags - .to_tags_impl() - .expect("serialized value trivially converts to tag map"); - assert_eq!(KnownArtifactTags::from_tags(&tag_map).unwrap(), tags); + let map = tags.to_tags(); + assert_eq!(KnownArtifactTags::from_tags(map).unwrap(), tags); + } + + #[test] + fn rot_sign() { + let mut tags = BTreeMap::from([ + ("kind".to_owned(), "rot".to_owned()), + ("rot_board".to_owned(), "oxide-rot-1".to_owned()), + // rot_sign not included + ("rot_slot".to_owned(), "a".to_owned()), + ]); + assert_eq!( + KnownArtifactTags::from_tags(tags.clone()).unwrap(), + KnownArtifactTags::Rot(RotTags { + rot_board: "oxide-rot-1".to_owned(), + rot_sign: Sign::UNSIGNED, + rot_slot: RotSlot::A + }) + ); + tags.insert("rot_sign".to_owned(), "meow".to_owned()); + assert_eq!( + KnownArtifactTags::from_tags(tags).unwrap(), + KnownArtifactTags::Rot(RotTags { + rot_board: "oxide-rot-1".to_owned(), + rot_sign: Sign(Some("meow".to_owned())), + rot_slot: RotSlot::A + }) + ); } } diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index cd6b2ad..451f628 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -17,6 +17,7 @@ use tufaceous_artifact::ArtifactVersionError; use tufaceous_artifact::InstallinatorArtifact; use tufaceous_artifact::InstallinatorDocument; use tufaceous_artifact::KnownArtifactTags; +use tufaceous_artifact::Metadata; use tufaceous_artifact::OsVariant; use tufaceous_artifact::RotBootloaderTags; use tufaceous_artifact::RotSlot; @@ -45,6 +46,7 @@ pub struct RepositoryEditor<'a> { generate_installinator_document: bool, targets: HashMap>>, artifacts: HashMap>, + metadata: BTreeMap, } impl<'a> RepositoryEditor<'a> { @@ -56,6 +58,7 @@ impl<'a> RepositoryEditor<'a> { generate_installinator_document: true, targets: HashMap::new(), artifacts: HashMap::new(), + metadata: BTreeMap::new(), } } @@ -296,6 +299,11 @@ impl<'a> RepositoryEditor<'a> { Ok(editor) } + pub fn metadata(mut self, metadata: &Metadata) -> Self { + self.metadata = metadata.to_map(); + self + } + pub fn from_repo(repo: &'a Repository) -> Result { Self::new(repo.system_version().clone()).import_repo(repo) } @@ -329,6 +337,7 @@ impl<'a> RepositoryEditor<'a> { tags: artifact.tags.clone(), }); } + self.metadata = repo.metadata().clone(); Ok(self) } @@ -351,7 +360,7 @@ impl<'a> RepositoryEditor<'a> { // we expect to be non-unique. let mut seen_tags = HashMap::new(); for (target_name, artifact) in &artifacts { - if let Ok(tags) = KnownArtifactTags::from_tags(&artifact.tags) + if let Some(tags) = artifact.known_tags() && matches!(tags, KnownArtifactTags::MeasurementCorpus) { continue; @@ -437,7 +446,7 @@ impl<'a> RepositoryEditor<'a> { let document = ArtifactsSchema { system_version: self.system_version, artifacts: artifacts.into_values().collect(), - metadata: BTreeMap::new(), + metadata: self.metadata, }; let target = BytesSource::json(&document) .map_err(ErrorKind::SerializeArtifacts)? @@ -483,7 +492,7 @@ pub(crate) fn generate_installinator_document( let mut document = InstallinatorDocument::empty(version.clone()); for (artifact, sha256) in artifacts { let artifact = artifact.as_ref(); - if let Ok(tags) = KnownArtifactTags::from_tags(&artifact.tags) + if let Some(tags) = artifact.known_tags() && let Some(kind) = tags.to_installinator() && let Some(file_name) = Utf8Path::new(&artifact.target_name).file_name() diff --git a/lib/src/repo.rs b/lib/src/repo.rs index 007abea..92f5c90 100644 --- a/lib/src/repo.rs +++ b/lib/src/repo.rs @@ -29,6 +29,7 @@ use tufaceous_artifact::ArtifactHash; use tufaceous_artifact::Artifacts; use tufaceous_artifact::GetError; use tufaceous_artifact::KnownArtifactTags; +use tufaceous_artifact::Metadata; use crate::RepositoryLoader; use crate::error::Error; @@ -188,6 +189,10 @@ impl Repository { &self.metadata } + pub fn structured_metadata(&self) -> Option { + Metadata::from_map(self.metadata.clone()).ok() + } + pub fn is_v1(&self) -> bool { self.v1_unpacked.is_some() } diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs index 20a1a9e..032408a 100644 --- a/lib/src/repo/v1.rs +++ b/lib/src/repo/v1.rs @@ -527,7 +527,7 @@ async fn generate_installinator_document( let target_name = format!("{original_target}/v2.json"); let mut document = InstallinatorDocument::empty(version.clone()); for artifact in artifacts.iter() { - if let Ok(tags) = KnownArtifactTags::from_tags(&artifact.tags) + if let Some(tags) = artifact.known_tags() && let Some(kind) = tags.to_installinator() && let Some(file_name) = Utf8Path::new(&artifact.target_name).file_name() diff --git a/lib/src/schema.rs b/lib/src/schema.rs index 068b910..62a33d8 100644 --- a/lib/src/schema.rs +++ b/lib/src/schema.rs @@ -8,6 +8,7 @@ use semver::Version; use serde::Deserialize; use serde::Serialize; use tufaceous_artifact::ArtifactVersion; +use tufaceous_artifact::KnownArtifactTags; #[derive(Debug, Deserialize, Serialize)] pub(crate) struct ArtifactsSchema { @@ -29,6 +30,12 @@ pub(crate) struct ArtifactSchema { pub(crate) tags: BTreeMap, } +impl ArtifactSchema { + pub(crate) fn known_tags(&self) -> Option { + KnownArtifactTags::from_tags(self.tags.clone()).ok() + } +} + impl AsRef for ArtifactSchema { fn as_ref(&self) -> &ArtifactSchema { self From c6b269ebcc8663d8c831946d31a49d5d37ac376c Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 5 Mar 2026 19:37:16 -0800 Subject: [PATCH 047/137] rework common to_map function --- artifact/src/map.rs | 72 +++++++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 35 deletions(-) diff --git a/artifact/src/map.rs b/artifact/src/map.rs index 2869d54..de4d52b 100644 --- a/artifact/src/map.rs +++ b/artifact/src/map.rs @@ -13,49 +13,51 @@ pub(crate) fn from_map( serde_json::from_value(map.into_iter().collect()) } +/// This should always succeed, and callers must use proptests to ensure that +/// it does. +/// +/// # Panics +/// +/// When debug assertions are enabled, this function panics if +/// `serde_json::to_value` returns a type other than `Value::Object` or an +/// error, or if any of the values in the object are not `Value::String`. +/// +/// When debug assertions are disabled, this function returns an empty map +/// if any of these conditions are hit. pub(crate) fn to_map(s: &S) -> BTreeMap { - // Call `to_map_impl`. On error, panic if debug assertions are enabled, - // otherwise return `BTreeMap::default()`. - match to_map_impl(s) { - Ok(map) => map, - Err(err) => { + macro_rules! debug_panic { + ($($tt:tt)*) => { if cfg!(debug_assertions) { - panic!( - "serializing {} to map should always succeed: {err:?}", - std::any::type_name::() - ); + panic!($($tt)*); } else { - BTreeMap::default() + return BTreeMap::default() } } } -} -fn to_map_impl( - s: &S, -) -> Result, ToMapError> { - let value = serde_json::to_value(s)?; - let serde_json::Value::Object(map) = value else { - return Err(ToMapError::StructNotObject(Box::new(value))); + let value_map = match serde_json::to_value(s) { + Ok(serde_json::Value::Object(map)) => map, + Ok(not_string) => debug_panic!( + "{} serialized to {not_string:?}, not an object", + std::any::type_name::() + ), + Err(err) => debug_panic!( + "failed to serialize {}: {err:?}", + std::any::type_name::() + ), }; - map.into_iter() - .map(|(key, value)| match value { - serde_json::Value::String(value) => Ok((key, value)), - _ => { - Err(ToMapError::ValueNotString { key, value: Box::new(value) }) - } - }) - .collect() -} - -#[derive(Debug, thiserror::Error)] -enum ToMapError { - #[error("failed to serialize struct")] - Serialize(#[from] serde_json::Error), - #[error("struct serialized to {0:?}, not an object")] - StructNotObject(Box), - #[error("value for {key:?} serialized to {value:?}, not a string")] - ValueNotString { key: String, value: Box }, + let mut map = BTreeMap::new(); + for (key, value) in value_map { + if let serde_json::Value::String(value) = value { + map.insert(key, value); + } else { + debug_panic!( + "{key:?} in {} serialized to {value:?}, not a string", + std::any::type_name::() + ); + } + } + map } #[cfg(test)] From 2b44f1757cb19e3e64bbade6071a571cf72e82de Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 24 Mar 2026 09:35:56 -0700 Subject: [PATCH 048/137] bump tar --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5a16eb0..b174985 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2487,9 +2487,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tar" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" +checksum = "22692a6476a21fa75fdfc11d452fda482af402c008cdbaf3476414e122040973" dependencies = [ "filetime", "libc", diff --git a/Cargo.toml b/Cargo.toml index fed19f5..77a6d91 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,7 +45,7 @@ slog = "2.7.0" slog-async = "2.8.0" slog-envlogger = "2.2.0" slog-term = "2.9.2" -tar = "0.4.44" +tar = "0.4.45" test-strategy = "0.4.0" thiserror = "2.0.12" tokio = "1.47.0" From b3415b74e150b980c869731a454f0b7b35ff0f6a Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 7 Apr 2026 09:37:42 -0700 Subject: [PATCH 049/137] bump rsa --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b174985..1483372 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1918,9 +1918,9 @@ dependencies = [ [[package]] name = "rsa" -version = "0.9.6" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d0e5124fcb30e76a7e79bfee683a2746db83784b86289f6251b54b7950a0dfc" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ "const-oid", "digest", From dbd0cf7b0a7689ec8e2eadb76e9e171745ec0b7a Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 7 Apr 2026 12:14:01 -0700 Subject: [PATCH 050/137] move DisplayTags into tags.rs --- artifact/src/artifact.rs | 30 +----------------------------- artifact/src/tags.rs | 31 ++++++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 30 deletions(-) diff --git a/artifact/src/artifact.rs b/artifact/src/artifact.rs index 73463ce..5d02e27 100644 --- a/artifact/src/artifact.rs +++ b/artifact/src/artifact.rs @@ -7,7 +7,6 @@ use std::collections::BTreeMap; use std::collections::BTreeSet; use std::collections::btree_map::IntoValues; use std::collections::btree_map::Values; -use std::fmt::Display; use std::iter::Flatten; use serde::Deserialize; @@ -15,6 +14,7 @@ use serde::Serialize; use crate::ArtifactHash; use crate::ArtifactVersion; +use crate::DisplayTags; use crate::KnownArtifactTags; #[derive( @@ -43,34 +43,6 @@ impl Artifact { } } -#[derive(Debug, Clone)] -pub struct DisplayTags<'a>(pub(crate) Cow<'a, BTreeMap>); - -impl Display for DisplayTags<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut comma = ""; - let kind = self.0.get_key_value("kind").into_iter(); - let remainder = self.0.iter().filter(|(k, _)| *k != "kind"); - for (key, value) in kind.chain(remainder) { - write!(f, "{comma}{key}={value}")?; - comma = ","; - } - Ok(()) - } -} - -impl From> for DisplayTags<'static> { - fn from(tags: BTreeMap) -> Self { - Self(Cow::Owned(tags)) - } -} - -impl<'a> From<&'a BTreeMap> for DisplayTags<'a> { - fn from(tags: &'a BTreeMap) -> Self { - Self(Cow::Borrowed(tags)) - } -} - #[derive(Debug, Clone, Deserialize, Serialize)] #[cfg_attr(any(test, feature = "schemars"), derive(schemars::JsonSchema))] pub struct ArtifactId { diff --git a/artifact/src/tags.rs b/artifact/src/tags.rs index b4a3547..a34e59f 100644 --- a/artifact/src/tags.rs +++ b/artifact/src/tags.rs @@ -2,12 +2,13 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use std::borrow::Cow; use std::collections::BTreeMap; +use std::fmt::Display; use serde::Deserialize; use serde::Serialize; -use crate::DisplayTags; use crate::InstallinatorArtifactKind; use crate::Sign; @@ -260,6 +261,34 @@ impl From for KnownArtifactTags { } } +#[derive(Debug, Clone)] +pub struct DisplayTags<'a>(pub(crate) Cow<'a, BTreeMap>); + +impl Display for DisplayTags<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut comma = ""; + let kind = self.0.get_key_value("kind").into_iter(); + let remainder = self.0.iter().filter(|(k, _)| *k != "kind"); + for (key, value) in kind.chain(remainder) { + write!(f, "{comma}{key}={value}")?; + comma = ","; + } + Ok(()) + } +} + +impl From> for DisplayTags<'static> { + fn from(tags: BTreeMap) -> Self { + Self(Cow::Owned(tags)) + } +} + +impl<'a> From<&'a BTreeMap> for DisplayTags<'a> { + fn from(tags: &'a BTreeMap) -> Self { + Self(Cow::Borrowed(tags)) + } +} + #[cfg(test)] mod tests { use std::collections::BTreeMap; From b099e5865ed975b5a235f8faa11ac2605d795c89 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 7 Apr 2026 12:14:40 -0700 Subject: [PATCH 051/137] remove ArtifactId only used within wicket, easy to move there --- artifact/src/artifact.rs | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/artifact/src/artifact.rs b/artifact/src/artifact.rs index 5d02e27..37516a2 100644 --- a/artifact/src/artifact.rs +++ b/artifact/src/artifact.rs @@ -2,7 +2,6 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use std::borrow::Cow; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::collections::btree_map::IntoValues; @@ -29,11 +28,6 @@ pub struct Artifact { } impl Artifact { - /// Clones this artifact's `version` and `tags` into an [`ArtifactId`]. - pub fn id(&self) -> ArtifactId { - ArtifactId { version: self.version.clone(), tags: self.tags.clone() } - } - pub fn known_tags(&self) -> Option { KnownArtifactTags::from_tags(self.tags.clone()).ok() } @@ -43,29 +37,6 @@ impl Artifact { } } -#[derive(Debug, Clone, Deserialize, Serialize)] -#[cfg_attr(any(test, feature = "schemars"), derive(schemars::JsonSchema))] -pub struct ArtifactId { - pub version: ArtifactVersion, - pub tags: BTreeMap, -} - -impl ArtifactId { - pub fn known_tags(&self) -> Option { - KnownArtifactTags::from_tags(self.tags.clone()).ok() - } - - pub fn display_tags(&self) -> DisplayTags<'_> { - DisplayTags(Cow::Borrowed(&self.tags)) - } -} - -impl From for ArtifactId { - fn from(artifact: Artifact) -> Self { - ArtifactId { version: artifact.version, tags: artifact.tags } - } -} - #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct Artifacts { inner: BTreeMap, BTreeSet>, From 370f3807d45b8fbf80ed4a0134cf89ff213e93f2 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 7 Apr 2026 13:43:12 -0700 Subject: [PATCH 052/137] consistently use re-export of ExpirationEnforcement --- bin/src/edit.rs | 2 +- bin/src/list_targets.rs | 2 +- bin/src/show_target.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/src/edit.rs b/bin/src/edit.rs index 57121f9..09baa25 100644 --- a/bin/src/edit.rs +++ b/bin/src/edit.rs @@ -7,7 +7,7 @@ use camino::Utf8PathBuf; use chrono::Utc; use clap::Parser; use semver::Version; -use tough::ExpirationEnforcement; +use tufaceous::ExpirationEnforcement; use tufaceous::RepositoryLoader; use tufaceous::TrustStoreBehavior; use tufaceous::edit::RepositoryEditor; diff --git a/bin/src/list_targets.rs b/bin/src/list_targets.rs index d9421a1..0d25368 100644 --- a/bin/src/list_targets.rs +++ b/bin/src/list_targets.rs @@ -7,7 +7,7 @@ use std::collections::BTreeSet; use anyhow::Result; use camino::Utf8PathBuf; use clap::Parser; -use tough::ExpirationEnforcement; +use tufaceous::ExpirationEnforcement; use tufaceous::RepositoryLoader; use tufaceous::TrustStoreBehavior; diff --git a/bin/src/show_target.rs b/bin/src/show_target.rs index 2823f92..689e2f3 100644 --- a/bin/src/show_target.rs +++ b/bin/src/show_target.rs @@ -9,7 +9,7 @@ use anyhow::Result; use camino::Utf8PathBuf; use clap::Parser; use futures_util::TryStreamExt; -use tough::ExpirationEnforcement; +use tufaceous::ExpirationEnforcement; use tufaceous::RepositoryLoader; use tufaceous::TrustStoreBehavior; From def2f94f0d20453f04234135b07fc55c5fb1cc43 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 7 Apr 2026 14:26:16 -0700 Subject: [PATCH 053/137] clean up composite archive types from brand-metadata --- brand-metadata/src/lib.rs | 26 +++++--------------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/brand-metadata/src/lib.rs b/brand-metadata/src/lib.rs index fd0a48b..0ae24f2 100644 --- a/brand-metadata/src/lib.rs +++ b/brand-metadata/src/lib.rs @@ -4,10 +4,10 @@ //! Handling of `oxide.json` metadata files in tarballs. //! -//! `oxide.json` is originally defined by the omicron1(7) zone brand, which -//! lives at . tufaceous -//! extended this format with additional archive types for identifying other -//! types of tarballs. +//! `oxide.json` is defined by the omicron1(7) zone brand, which lives +//! at . tufaceous +//! previously extended this format with additional archive types for +//! identifying composite artifacts. use std::io::Error; use std::io::ErrorKind; @@ -32,17 +32,13 @@ pub struct Metadata { t: ArchiveType, } +/// Archive types defined in helios-build-utils (part of helios-omicron-brand). #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "snake_case", tag = "t")] pub enum ArchiveType { - // Originally defined in helios-build-utils (part of helios-omicron-brand): Baseline, Layer(LayerInfo), Os, - - // tufaceous extensions: - Rot, - ControlPlane, } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -130,14 +126,6 @@ impl Metadata { pub fn is_os(&self) -> bool { matches!(&self.t, ArchiveType::Os) } - - pub fn is_rot(&self) -> bool { - matches!(&self.t, ArchiveType::Rot) - } - - pub fn is_control_plane(&self) -> bool { - matches!(&self.t, ArchiveType::ControlPlane) - } } #[cfg(test)] @@ -159,9 +147,5 @@ mod tests { r#"{"v":"1","t":"os","i":{"checksum":"42eda100ee0e3bf44b9d0bb6a836046fa3133c378cd9d3a4ba338c3ba9e56eb7","name":"ci 3a2ed5e/9d37813 2024-12-20 08:54"}}"#, ).unwrap(); assert!(metadata.is_os()); - - let metadata: Metadata = - serde_json::from_str(r#"{"v":"1","t":"control_plane"}"#).unwrap(); - assert!(metadata.is_control_plane()); } } From bc4fd3ddb671044b7c39aef555013a63f2d5ec43 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Wed, 8 Apr 2026 21:31:40 -0700 Subject: [PATCH 054/137] refactor / docs pass on tufaceous-artifact --- artifact/src/artifact.rs | 251 ++++++++++++------------------- artifact/src/hash.rs | 102 ------------- artifact/src/hubris.rs | 61 ++++++-- artifact/src/installinator.rs | 13 +- artifact/src/lib.rs | 20 ++- artifact/src/set.rs | 262 +++++++++++++++++++++++++++++++++ artifact/src/sign.rs | 46 +++--- artifact/src/tags.rs | 41 ++++-- bin/src/edit.rs | 2 +- lib/src/edit/editor.rs | 19 +-- lib/src/edit/fake.rs | 20 +-- lib/src/edit/hubris_archive.rs | 57 ++++--- lib/src/error.rs | 2 +- lib/src/repo.rs | 20 +-- lib/src/repo/v1.rs | 30 ++-- lib/src/schema.rs | 4 +- lib/tests/v1_compatibility.rs | 12 +- 17 files changed, 572 insertions(+), 390 deletions(-) delete mode 100644 artifact/src/hash.rs create mode 100644 artifact/src/set.rs diff --git a/artifact/src/artifact.rs b/artifact/src/artifact.rs index 37516a2..84be4ed 100644 --- a/artifact/src/artifact.rs +++ b/artifact/src/artifact.rs @@ -3,206 +3,145 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use std::collections::BTreeMap; -use std::collections::BTreeSet; -use std::collections::btree_map::IntoValues; -use std::collections::btree_map::Values; -use std::iter::Flatten; +use std::fmt::Debug; +use std::fmt::Display; +use std::str::FromStr; +use daft::Diffable; +use hex::FromHexError; use serde::Deserialize; use serde::Serialize; -use crate::ArtifactHash; use crate::ArtifactVersion; use crate::DisplayTags; use crate::KnownArtifactTags; +/// An artifact in a repository. #[derive( Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, )] pub struct Artifact { + /// The TUF target name, if this value is associated with a loaded + /// repository. + /// + /// If you are instantiating this struct from other records (such as a + /// database), you can use `String::new()` here. pub target_name: String, + + /// The version of the artifact. pub version: ArtifactVersion, + + /// The artifact's tags. + /// + /// Tags describe how an artifact is to be used by the control plane. + /// In this form, they are an arbitrary mapping of string keys to string + /// values. Using [`Artifact::known_tags`], they can be (fallibly) converted + /// into a strongly-typed description of the artifact. + /// + /// When recording artifacts for later use, the control plane must always + /// record these tags as-is, even if `Artifact::known_tags` returns `None`. pub tags: BTreeMap, + + /// The SHA-256 checksum of the artifact. pub hash: ArtifactHash, + + /// The length of the artifact in bytes. pub length: u64, } impl Artifact { + /// Resolves [`Artifact::tags`] into [`KnownArtifactTags`]. + /// + /// Returns `None` if the tags do not resolve to any known artifact. pub fn known_tags(&self) -> Option { KnownArtifactTags::from_tags(self.tags.clone()).ok() } + /// Returns an adapter for displaying [`Artifact::tags`] as a human-readable + /// string. pub fn display_tags(&self) -> DisplayTags<'_> { DisplayTags::from(&self.tags) } } -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct Artifacts { - inner: BTreeMap, BTreeSet>, -} - -impl Artifacts { - pub fn new(iter: impl IntoIterator) -> Self { - Self::from_iter(iter) - } - - pub fn len(&self) -> usize { - self.inner.values().map(BTreeSet::len).sum() - } - - pub fn is_empty(&self) -> bool { - debug_assert_eq!(self.len(), 0); - self.inner.is_empty() - } - - pub fn insert(&mut self, artifact: Artifact) { - self.inner.entry(artifact.known_tags()).or_default().insert(artifact); - } - - pub fn get(&self, tags: KnownArtifactTags) -> Result<&Artifact, GetError> { - let vec = self.inner.get(&Some(tags)).ok_or(GetError::NotFound)?; - if vec.len() == 1 - && let Some(artifact) = vec.first() - { - Ok(artifact) - } else { - Err(GetError::TooMany) - } - } - - pub fn get_all( - &self, - tags: KnownArtifactTags, - ) -> impl Iterator { - self.inner.get(&Some(tags)).map(BTreeSet::iter).unwrap_or_default() - } - - pub fn contains(&self, artifact: &Artifact) -> bool { - self.inner - .get(&artifact.known_tags()) - .is_some_and(|set| set.contains(artifact)) - } - - pub fn filter_tags( - &self, - mut predicate: impl FnMut(&KnownArtifactTags) -> bool, - ) -> impl Iterator { - self.inner - .iter() - .filter_map(move |(tags, artifacts)| { - predicate(tags.as_ref()?).then_some(artifacts) - }) - .flatten() - } - - pub fn iter(&self) -> Iter<'_> { - Iter { inner: self.inner.values().flatten() } - } -} - -impl Extend for Artifacts { - fn extend>(&mut self, iter: T) { - for artifact in iter { - self.insert(artifact); - } - } -} - -impl FromIterator for Artifacts { - fn from_iter>(iter: T) -> Self { - let mut artifacts = Self::default(); - artifacts.extend(iter); - artifacts - } -} - -impl IntoIterator for Artifacts { - type Item = Artifact; - type IntoIter = IntoIter; - - fn into_iter(self) -> Self::IntoIter { - IntoIter { inner: self.inner.into_values().flatten() } - } -} - -impl<'a> IntoIterator for &'a Artifacts { - type Item = &'a Artifact; - type IntoIter = Iter<'a>; - - fn into_iter(self) -> Self::IntoIter { - self.iter() +/// The SHA-256 checksum of an artifact. +#[derive( + Copy, + Clone, + Diffable, + Eq, + PartialEq, + Ord, + PartialOrd, + Hash, + Serialize, + Deserialize, +)] +#[daft(leaf)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +#[cfg_attr(feature = "proptest", derive(test_strategy::Arbitrary))] +#[serde(transparent)] +pub struct ArtifactHash( + #[serde(with = "serde_human_bytes::hex_array")] + #[cfg_attr( + feature = "schemars", + schemars(schema_with = "hex_schema::<32>") + )] + pub [u8; 32], +); + +impl AsRef<[u8]> for ArtifactHash { + fn as_ref(&self) -> &[u8] { + &self.0 } } -impl<'de> Deserialize<'de> for Artifacts { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - struct Visitor; - - impl<'de> serde::de::Visitor<'de> for Visitor { - type Value = Artifacts; - - fn expecting( - &self, - f: &mut std::fmt::Formatter, - ) -> std::fmt::Result { - write!(f, "a sequence") - } - - fn visit_seq(self, mut seq: A) -> Result - where - A: serde::de::SeqAccess<'de>, - { - std::iter::from_fn(|| seq.next_element().transpose()).collect() - } - } - - deserializer.deserialize_seq(Visitor) +impl Debug for ArtifactHash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_tuple("ArtifactHash").field(&hex::encode(self.0)).finish() } } -impl Serialize for Artifacts { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - serializer.collect_seq(self.iter()) +impl Display for ArtifactHash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + Display::fmt(&hex::encode(self.0), f) } } -#[derive(Debug)] -pub struct IntoIter { - inner: Flatten, BTreeSet>>, -} - -impl Iterator for IntoIter { - type Item = Artifact; +impl FromStr for ArtifactHash { + type Err = FromHexError; - fn next(&mut self) -> Option { - self.inner.next() + fn from_str(s: &str) -> std::result::Result { + let mut out = [0u8; 32]; + hex::decode_to_slice(s, &mut out)?; + Ok(Self(out)) } } -#[derive(Debug, Clone)] -pub struct Iter<'a> { - inner: Flatten, BTreeSet>>, +/// Produce an OpenAPI schema describing a hex array of a specific length (e.g., +/// a hash digest). +#[cfg(feature = "schemars")] +fn hex_schema( + generator: &mut schemars::SchemaGenerator, +) -> schemars::schema::Schema { + use schemars::JsonSchema; + + let mut schema: schemars::schema::SchemaObject = + ::json_schema(generator).into(); + schema.format = Some(format!("hex string ({N} bytes)")); + schema.into() } -impl<'a> Iterator for Iter<'a> { - type Item = &'a Artifact; +#[cfg(test)] +mod tests { + use super::*; - fn next(&mut self) -> Option { - self.inner.next() + #[test] + fn display_respects_padding() { + let h = ArtifactHash([0; 32]); + assert_eq!( + format!("{h:x>100}"), + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx0000000000000000000000000000000000000000000000000000000000000000" + ); } } - -#[derive(Debug, thiserror::Error)] -pub enum GetError { - #[error("artifact not found")] - NotFound, - #[error("more than one artifact found")] - TooMany, -} diff --git a/artifact/src/hash.rs b/artifact/src/hash.rs deleted file mode 100644 index 19ef43d..0000000 --- a/artifact/src/hash.rs +++ /dev/null @@ -1,102 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use std::array::TryFromSliceError; -use std::fmt; -use std::str::FromStr; - -use daft::Diffable; -use hex::FromHexError; -use serde::Deserialize; -use serde::Serialize; - -/// The hash of an artifact. -#[derive( - Copy, - Clone, - Diffable, - Eq, - PartialEq, - Ord, - PartialOrd, - Hash, - Serialize, - Deserialize, -)] -#[daft(leaf)] -#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] -#[cfg_attr(feature = "proptest", derive(test_strategy::Arbitrary))] -#[serde(transparent)] -pub struct ArtifactHash( - #[serde(with = "serde_human_bytes::hex_array")] - #[cfg_attr( - feature = "schemars", - schemars(schema_with = "hex_schema::<32>") - )] - pub [u8; 32], -); - -impl AsRef<[u8]> for ArtifactHash { - fn as_ref(&self) -> &[u8] { - &self.0 - } -} - -impl fmt::Debug for ArtifactHash { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_tuple("ArtifactHash").field(&hex::encode(self.0)).finish() - } -} - -impl fmt::Display for ArtifactHash { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - hex::encode(self.0).fmt(f) - } -} - -impl FromStr for ArtifactHash { - type Err = FromHexError; - - fn from_str(s: &str) -> std::result::Result { - let mut out = [0u8; 32]; - hex::decode_to_slice(s, &mut out)?; - Ok(Self(out)) - } -} - -impl TryFrom<&[u8]> for ArtifactHash { - type Error = TryFromSliceError; - - fn try_from(value: &[u8]) -> Result { - value.try_into().map(Self) - } -} - -/// Produce an OpenAPI schema describing a hex array of a specific length (e.g., -/// a hash digest). -#[cfg(feature = "schemars")] -fn hex_schema( - generator: &mut schemars::SchemaGenerator, -) -> schemars::schema::Schema { - use schemars::JsonSchema; - - let mut schema: schemars::schema::SchemaObject = - ::json_schema(generator).into(); - schema.format = Some(format!("hex string ({N} bytes)")); - schema.into() -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn display_respects_padding() { - let h = ArtifactHash([0; 32]); - assert_eq!( - format!("{h:x>100}"), - "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx0000000000000000000000000000000000000000000000000000000000000000" - ); - } -} diff --git a/artifact/src/hubris.rs b/artifact/src/hubris.rs index 7d45da3..84f1df8 100644 --- a/artifact/src/hubris.rs +++ b/artifact/src/hubris.rs @@ -9,40 +9,70 @@ use hubtools::CabooseError; use crate::KnownArtifactTags; use crate::RotBootloaderTags; +use crate::RotSign; use crate::RotSlot; use crate::RotTags; -use crate::Sign; use crate::SpTags; impl RotTags { + /// Attempts to read the values of `RotTags` from a Hubris caboose. + /// + /// `slot` is not part of the caboose and must be provided. + /// + /// # Errors + /// + /// Returns an error if the caboose is not valid, if `BORD` is not present, + /// or if `BORD` or `SIGN` are not valid UTF-8 strings. pub fn from_caboose( caboose: &Caboose, slot: RotSlot, ) -> Result { Ok(Self { rot_board: read_board(caboose)?, - rot_sign: Sign(read_sign(caboose)?), + rot_sign: RotSign(read_sign(caboose)?), rot_slot: slot, }) } } impl RotBootloaderTags { + /// Attempts to read the values of `RotBootloaderTags` from a Hubris + /// caboose. + /// + /// # Errors + /// + /// Returns an error if the caboose is not valid, if `BORD` is not present, + /// or if `BORD` or `SIGN` are not valid UTF-8 strings. pub fn from_caboose(caboose: &Caboose) -> Result { Ok(Self { rot_board: read_board(caboose)?, - rot_sign: Sign(read_sign(caboose)?), + rot_sign: RotSign(read_sign(caboose)?), }) } } impl SpTags { + /// Attempts to read the values of `SpTags` from a Hubris caboose. + /// + /// # Errors + /// + /// Returns an error if the caboose is not valid, or if `BORD` is not + /// present or is not not a valid UTF-8 string. pub fn from_caboose(caboose: &Caboose) -> Result { Ok(Self { sp_board: read_board(caboose)? }) } } impl KnownArtifactTags { + /// Attempts to read the values of [`KnownArtifactTags::Rot`] from + /// a Hubris caboose. + /// + /// `slot` is not part of the caboose and must be provided. + /// + /// # Errors + /// + /// Returns an error if the caboose is not valid, if `BORD` is not present, + /// or if `BORD` or `SIGN` are not valid UTF-8 strings. pub fn from_rot_caboose( caboose: &Caboose, slot: RotSlot, @@ -50,6 +80,13 @@ impl KnownArtifactTags { RotTags::from_caboose(caboose, slot).map(KnownArtifactTags::Rot) } + /// Attempts to read the values of [`KnownArtifactTags::RotBootloader`] from + /// a Hubris caboose. + /// + /// # Errors + /// + /// Returns an error if the caboose is not valid, if `BORD` is not present, + /// or if `BORD` or `SIGN` are not valid UTF-8 strings. pub fn from_rot_bootloader_caboose( caboose: &Caboose, ) -> Result { @@ -57,6 +94,13 @@ impl KnownArtifactTags { .map(KnownArtifactTags::RotBootloader) } + /// Attempts to read the values of [`KnownArtifactTags::Sp`] from a Hubris + /// caboose. + /// + /// # Errors + /// + /// Returns an error if the caboose is not valid, or if `BORD` is not + /// present or is not not a valid UTF-8 string. pub fn from_sp_caboose( caboose: &Caboose, ) -> Result { @@ -84,18 +128,13 @@ fn read_sign(caboose: &Caboose) -> Result, ReadCabooseError> { } } -pub fn read_name(caboose: &Caboose) -> Result<&str, ReadCabooseError> { - utf8(caboose.name()?, "NAME") -} - -pub fn read_version(caboose: &Caboose) -> Result<&str, ReadCabooseError> { - utf8(caboose.version()?, "VERS") -} - +/// An error that occurred while reading a Hubris archive. #[derive(Debug, thiserror::Error)] pub enum ReadCabooseError { + /// The caboose was not valid, or the required tag was missing. #[error(transparent)] Caboose(#[from] hubtools::CabooseError), + /// The tag value was not valid UTF-8. #[error("{tag} is not valid UTF-8")] Utf8 { tag: &'static str, source: std::str::Utf8Error }, } diff --git a/artifact/src/installinator.rs b/artifact/src/installinator.rs index e340b9e..8c639df 100644 --- a/artifact/src/installinator.rs +++ b/artifact/src/installinator.rs @@ -19,7 +19,7 @@ use crate::ArtifactVersion; /// /// There are no backwards compatibility constraints for this document. The /// version of Installinator that processes this document is the same as the -/// version of tufaceous that creates it. +/// version of Tufaceous that creates it. #[derive(Debug, Clone, Deserialize, Serialize)] pub struct InstallinatorDocument { pub system_version: ArtifactVersion, @@ -41,18 +41,15 @@ impl InstallinatorDocument { pub struct InstallinatorArtifact { #[serde(flatten)] pub kind: InstallinatorArtifactKind, - pub sha256: ArtifactHash, + pub hash: ArtifactHash, /// A file name without directory separators; not necessarily the target /// name. pub file_name: String, } impl InstallinatorArtifact { - pub fn downgrade(&self) -> InstallinatorArtifactId { - InstallinatorArtifactId { - kind: self.kind.downgrade(), - hash: self.sha256, - } + pub fn to_id(&self) -> InstallinatorArtifactId { + InstallinatorArtifactId { kind: self.kind.to_id(), hash: self.hash } } } @@ -71,7 +68,7 @@ pub enum InstallinatorArtifactKind { } impl InstallinatorArtifactKind { - pub fn downgrade(&self) -> InstallinatorArtifactKindId { + pub fn to_id(&self) -> InstallinatorArtifactKindId { InstallinatorArtifactKindId(match self { InstallinatorArtifactKind::MeasurementCorpus => { Cow::Borrowed("measurement_corpus") diff --git a/artifact/src/lib.rs b/artifact/src/lib.rs index 71ea29e..7dfe108 100644 --- a/artifact/src/lib.rs +++ b/artifact/src/lib.rs @@ -2,20 +2,30 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +//! tufaceous-artifact defines the core types of tufaceous so that they can be +//! used without the full tufaceous library. + mod artifact; -mod hash; -pub mod hubris; -mod installinator; +mod hubris; +pub mod installinator; mod map; mod metadata; +mod set; mod sign; mod tags; mod version; pub use artifact::*; -pub use hash::*; -pub use installinator::*; +pub use hubris::ReadCabooseError; pub use metadata::*; +pub use set::ArtifactSet; pub use sign::*; pub use tags::*; pub use version::*; + +/// Types related to [`ArtifactSet`]. +pub mod artifact_set { + pub use crate::set::GetError; + pub use crate::set::IntoIter; + pub use crate::set::Iter; +} diff --git a/artifact/src/set.rs b/artifact/src/set.rs new file mode 100644 index 0000000..bc8ce96 --- /dev/null +++ b/artifact/src/set.rs @@ -0,0 +1,262 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::BTreeMap; +use std::collections::BTreeSet; +use std::collections::btree_map::IntoValues; +use std::collections::btree_map::Values; +use std::collections::btree_set; +use std::iter::Chain; +use std::iter::Flatten; + +use serde::Deserialize; +use serde::Serialize; + +use crate::Artifact; +use crate::KnownArtifactTags; + +/// A set of [`Artifact`]s. +/// +/// This type is logically equivalent to `BTreeSet`, but has +/// the additional ability to look up specific artifacts based on their +/// [`KnownArtifactTags`]. (Because of this, artifacts are first sorted by the +/// `Ord` implementation of `KnownArtifactTags`.) +/// +/// Artifacts are consistently ordered within this struct, so two sets +/// containing the same artifacts are equal to each other. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ArtifactSet { + known: BTreeMap>, + unknown: BTreeSet, +} + +impl ArtifactSet { + /// Makes a new, empty `ArtifactSet`. + /// + /// Does not allocate anything on its own. + pub const fn new() -> Self { + Self { known: BTreeMap::new(), unknown: BTreeSet::new() } + } + + /// Adds an artifact to the set. + /// + /// Returns whether the artifact was newly inserted. That is: + /// + /// * If the set did not previously contain an equal artifact, `true` is + /// returned. + /// * If the set already contained an equal value, `false` is returned, and + /// the entry is not updated. + pub fn insert(&mut self, artifact: Artifact) -> bool { + match artifact.known_tags() { + Some(tags) => self.known.entry(tags).or_default().insert(artifact), + None => self.unknown.insert(artifact), + } + } + + /// Returns the number of artifacts in the set. + pub fn len(&self) -> usize { + self.known.values().map(BTreeSet::len).sum::() + + self.unknown.len() + } + + /// Returns `true` if the set contains no artifacts. + pub fn is_empty(&self) -> bool { + self.known.is_empty() && self.unknown.is_empty() + } + + /// Returns `true` if the set contains an artifact equal to `artifact`. + pub fn contains(&self, artifact: &Artifact) -> bool { + if let Some(tags) = artifact.known_tags() { + self.known.get(&tags).is_some_and(|set| set.contains(artifact)) + } else { + self.unknown.contains(artifact) + } + } + + /// Returns the single artifact matching `tags`. + /// + /// # Errors + /// + /// Returns an error if there is not exactly one artifact matching `tags`. + pub fn get(&self, tags: &KnownArtifactTags) -> Result<&Artifact, GetError> { + let set = self.known.get(tags).ok_or(GetError::NotFound)?; + if set.len() == 1 + && let Some(artifact) = set.first() + { + Ok(artifact) + } else { + Err(GetError::TooMany) + } + } + + /// Returns the set of all artifacts matching `tags`. + /// + /// If you are not certain that you want all matching artifacts, prefer + /// [`ArtifactSet::get`] instead. The control plane must not randomly choose + /// from matching artifacts when it expects only one. + pub fn get_all(&self, tags: &KnownArtifactTags) -> &BTreeSet { + static EMPTY: BTreeSet = BTreeSet::new(); + self.known.get(tags).unwrap_or(&EMPTY) + } + + /// Returns an iterator of artifacts where the tags match `predicate`. + /// + /// # Example + /// + /// ``` + /// # use tufaceous_artifact::ArtifactSet; + /// # use tufaceous_artifact::KnownArtifactTags; + /// # let set = ArtifactSet::new(); + /// for zone in set + /// .filter_tags(|tags| matches!(tags, KnownArtifactTags::Zone { .. })) + /// { + /// // ... + /// } + /// ``` + pub fn filter_tags( + &self, + mut predicate: impl FnMut(&KnownArtifactTags) -> bool, + ) -> impl Iterator { + self.known + .iter() + .filter_map(move |(tags, artifacts)| { + predicate(tags).then_some(artifacts) + }) + .flatten() + } + + /// Returns an iterator over the artifacts in the set. + pub fn iter(&self) -> Iter<'_> { + Iter { inner: self.known.values().flatten().chain(&self.unknown) } + } +} + +impl Extend for ArtifactSet { + fn extend>(&mut self, iter: T) { + for artifact in iter { + self.insert(artifact); + } + } +} + +impl From<[Artifact; N]> for ArtifactSet { + fn from(arr: [Artifact; N]) -> Self { + if N == 0 { ArtifactSet::new() } else { arr.into_iter().collect() } + } +} + +impl FromIterator for ArtifactSet { + fn from_iter>(iter: T) -> Self { + let mut artifacts = Self::default(); + artifacts.extend(iter); + artifacts + } +} + +impl IntoIterator for ArtifactSet { + type Item = Artifact; + type IntoIter = IntoIter; + + fn into_iter(self) -> Self::IntoIter { + IntoIter { + inner: self.known.into_values().flatten().chain(self.unknown), + } + } +} + +impl<'a> IntoIterator for &'a ArtifactSet { + type Item = &'a Artifact; + type IntoIter = Iter<'a>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +impl<'de> Deserialize<'de> for ArtifactSet { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct Visitor; + + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = ArtifactSet; + + fn expecting( + &self, + f: &mut std::fmt::Formatter, + ) -> std::fmt::Result { + write!(f, "a sequence") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + std::iter::from_fn(|| seq.next_element().transpose()).collect() + } + } + + deserializer.deserialize_seq(Visitor) + } +} + +impl Serialize for ArtifactSet { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.collect_seq(self.iter()) + } +} + +/// An iterator that moves values out of an [`ArtifactSet`]. +/// +/// This struct is created by [`ArtifactSet::into_iter`] (provided by the +/// [`IntoIterator`] trait). +#[derive(Debug)] +pub struct IntoIter { + inner: Chain< + Flatten>>, + btree_set::IntoIter, + >, +} + +impl Iterator for IntoIter { + type Item = Artifact; + + fn next(&mut self) -> Option { + self.inner.next() + } +} + +/// An iterator that borrows values from an [`ArtifactSet`]. +/// +/// This struct is created by [`ArtifactSet::iter`]. +#[derive(Debug, Clone)] +pub struct Iter<'a> { + inner: Chain< + Flatten>>, + btree_set::Iter<'a, Artifact>, + >, +} + +impl<'a> Iterator for Iter<'a> { + type Item = &'a Artifact; + + fn next(&mut self) -> Option { + self.inner.next() + } +} + +/// Returned by [`ArtifactSet::get`] when there is not exactly one matching +/// artifact. +#[derive(Debug, thiserror::Error)] +pub enum GetError { + #[error("artifact not found")] + NotFound, + #[error("more than one artifact found")] + TooMany, +} diff --git a/artifact/src/sign.rs b/artifact/src/sign.rs index 0cf8c19..136a0fa 100644 --- a/artifact/src/sign.rs +++ b/artifact/src/sign.rs @@ -51,41 +51,42 @@ const CA_LIST: [(&str, &str); 9] = [ static CA_MAP: LazyLock> = LazyLock::new(|| HashMap::from(CA_LIST)); +/// Value of the `SIGN` field of a Hubris caboose. +/// +/// Used in [`RotTags`] and [`RotBootloaderTags`]. +/// +/// This is a lowercase hexadecimal string of the ROT Key Table Hash (RKTH) that +/// is used to identify the CA. +/// +/// [`RotTags`]: crate::RotTags +/// [`RotBootloaderTags`]: crate::RotBootloaderTags #[derive( Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, )] #[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] #[serde(transparent)] -pub struct Sign(pub Option); - -impl Sign { - pub const UNSIGNED: Sign = Sign(None); - - pub fn signed(sign: String) -> Self { - Self(Some(sign)) - } - - pub fn is_signed(&self) -> bool { - self.0.is_some() - } - - pub fn is_unsigned(&self) -> bool { - self.0.is_none() - } +pub struct RotSign(pub Option); +impl RotSign { + /// Returns a friendly name for the CA this RKTH represents, if one is + /// known. Returns `None` otherwise. pub fn friendly_ca_name(&self) -> Option<&'static str> { let inner = self.0.as_deref()?; CA_MAP.get(inner).copied() } + + pub(crate) fn is_none(&self) -> bool { + self.0.is_none() + } } -impl Debug for Sign { +impl Debug for RotSign { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { Debug::fmt(&self.0, f) } } -impl Display for Sign { +impl Display for RotSign { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self.0.as_deref() { Some(inner) => { @@ -97,15 +98,6 @@ impl Display for Sign { } } -impl From for Sign -where - Option: From, -{ - fn from(sign: T) -> Self { - Self(sign.into()) - } -} - #[cfg(test)] mod tests { use std::collections::HashSet; diff --git a/artifact/src/tags.rs b/artifact/src/tags.rs index a34e59f..7c61b43 100644 --- a/artifact/src/tags.rs +++ b/artifact/src/tags.rs @@ -9,8 +9,8 @@ use std::fmt::Display; use serde::Deserialize; use serde::Serialize; -use crate::InstallinatorArtifactKind; -use crate::Sign; +use crate::RotSign; +use crate::installinator::InstallinatorArtifactKind; /// Sets of artifact tags known to the control plane. #[derive( @@ -160,6 +160,7 @@ pub enum OsBoard { } display_serialize!(OsBoard); +/// The inner value of [`KnownArtifactTags::Rot`]. #[derive( Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, )] @@ -173,8 +174,8 @@ pub struct RotTags { /// For unsigned images this will not be present; this will generally /// never occur in release repos but can be useful on hardware that has /// not fully made it through manufacturing yet. - #[serde(skip_serializing_if = "Sign::is_unsigned")] - pub rot_sign: Sign, + #[serde(skip_serializing_if = "RotSign::is_none")] + pub rot_sign: RotSign, /// ROT images are compiled for two different locations in flash; this /// identifies which slot this image belongs to. pub rot_slot: RotSlot, @@ -207,6 +208,7 @@ pub enum RotSlot { } display_serialize!(RotSlot); +/// The inner value of [`KnownArtifactTags::RotBootloader`]. #[derive( Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, )] @@ -220,8 +222,8 @@ pub struct RotBootloaderTags { /// For unsigned images this will not be present; this will generally /// never occur in release repos but can be useful on hardware that has /// not fully made it through manufacturing yet. - #[serde(skip_serializing_if = "Sign::is_unsigned")] - pub rot_sign: Sign, + #[serde(skip_serializing_if = "RotSign::is_none")] + pub rot_sign: RotSign, } impl From for KnownArtifactTags { @@ -230,6 +232,7 @@ impl From for KnownArtifactTags { } } +/// The inner value of [`KnownArtifactTags::Sp`]. #[derive( Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, )] @@ -245,6 +248,7 @@ impl From for KnownArtifactTags { } } +/// The inner value of [`KnownArtifactTags::Zone`]. #[derive( Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, )] @@ -261,6 +265,25 @@ impl From for KnownArtifactTags { } } +/// An adapter that implements [`Display`] for a set of tags. +/// +/// This is intended for error and log messages and is not a portable format. +/// +/// # Example +/// +/// ``` +/// # use std::collections::BTreeMap; +/// # use tufaceous_artifact::DisplayTags; +/// let tags = BTreeMap::from([ +/// ("foo".to_string(), "yes".to_string()), +/// ("bar".to_string(), "definitely".to_string()), +/// ("kind".to_string(), "thing".to_string()), +/// ]); +/// assert_eq!( +/// DisplayTags::from(&tags).to_string(), +/// "kind=thing,bar=definitely,foo=yes" +/// ); +/// ``` #[derive(Debug, Clone)] pub struct DisplayTags<'a>(pub(crate) Cow<'a, BTreeMap>); @@ -296,9 +319,9 @@ mod tests { use test_strategy::proptest; use crate::KnownArtifactTags; + use crate::RotSign; use crate::RotSlot; use crate::RotTags; - use crate::Sign; #[proptest] fn tags_roundtrip(tags: KnownArtifactTags) { @@ -318,7 +341,7 @@ mod tests { KnownArtifactTags::from_tags(tags.clone()).unwrap(), KnownArtifactTags::Rot(RotTags { rot_board: "oxide-rot-1".to_owned(), - rot_sign: Sign::UNSIGNED, + rot_sign: RotSign(None), rot_slot: RotSlot::A }) ); @@ -327,7 +350,7 @@ mod tests { KnownArtifactTags::from_tags(tags).unwrap(), KnownArtifactTags::Rot(RotTags { rot_board: "oxide-rot-1".to_owned(), - rot_sign: Sign(Some("meow".to_owned())), + rot_sign: RotSign(Some("meow".to_owned())), rot_slot: RotSlot::A }) ); diff --git a/bin/src/edit.rs b/bin/src/edit.rs index 09baa25..b608042 100644 --- a/bin/src/edit.rs +++ b/bin/src/edit.rs @@ -44,7 +44,7 @@ impl Args { // always remove an existing installinator document for artifact in - repo.artifacts().get_all(KnownArtifactTags::InstallinatorDocument) + repo.artifacts().get_all(&KnownArtifactTags::InstallinatorDocument) { editor = editor.remove_target(&artifact.target_name); } diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index 451f628..7363041 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -12,10 +12,9 @@ use futures_util::FutureExt; use futures_util::TryFutureExt; use semver::Version; use tokio::task::JoinSet; +use tufaceous_artifact::ArtifactHash; use tufaceous_artifact::ArtifactVersion; use tufaceous_artifact::ArtifactVersionError; -use tufaceous_artifact::InstallinatorArtifact; -use tufaceous_artifact::InstallinatorDocument; use tufaceous_artifact::KnownArtifactTags; use tufaceous_artifact::Metadata; use tufaceous_artifact::OsVariant; @@ -23,6 +22,8 @@ use tufaceous_artifact::RotBootloaderTags; use tufaceous_artifact::RotSlot; use tufaceous_artifact::RotTags; use tufaceous_artifact::SpTags; +use tufaceous_artifact::installinator::InstallinatorArtifact; +use tufaceous_artifact::installinator::InstallinatorDocument; use crate::Repository; use crate::edit::UnsignedRepository; @@ -36,7 +37,7 @@ use crate::edit::source::TargetSource; use crate::error::Error; use crate::error::ErrorKind; use crate::schema::ArtifactSchema; -use crate::schema::ArtifactsSchema; +use crate::schema::ArtifactSetSchema; #[derive(Debug, Clone)] #[must_use] @@ -314,7 +315,7 @@ impl<'a> RepositoryEditor<'a> { } for (target_name, target) in repo.targets() { - if target_name.raw() == ArtifactsSchema::TARGET_NAME { + if target_name.raw() == ArtifactSetSchema::TARGET_NAME { continue; } self.targets.entry(target_name.raw().to_owned()).or_default().push( @@ -443,7 +444,7 @@ impl<'a> RepositoryEditor<'a> { )?; } - let document = ArtifactsSchema { + let document = ArtifactSetSchema { system_version: self.system_version, artifacts: artifacts.into_values().collect(), metadata: self.metadata, @@ -452,7 +453,7 @@ impl<'a> RepositoryEditor<'a> { .map_err(ErrorKind::SerializeArtifacts)? .into_target() .await; - targets.insert(ArtifactsSchema::TARGET_NAME.to_owned(), target)?; + targets.insert(ArtifactSetSchema::TARGET_NAME.to_owned(), target)?; Ok(UnsignedRepository::from_targets(targets.0)) } } @@ -490,18 +491,18 @@ pub(crate) fn generate_installinator_document( ) -> Result, Error> { let target_name = format!("installinator_document-{version}.json"); let mut document = InstallinatorDocument::empty(version.clone()); - for (artifact, sha256) in artifacts { + for (artifact, hash) in artifacts { let artifact = artifact.as_ref(); if let Some(tags) = artifact.known_tags() && let Some(kind) = tags.to_installinator() && let Some(file_name) = Utf8Path::new(&artifact.target_name).file_name() - && let Ok(sha256) = sha256.as_ref().try_into() + && let Ok(hash) = hash.as_ref().try_into().map(ArtifactHash) { document.artifacts.insert(InstallinatorArtifact { file_name: file_name.to_owned(), kind, - sha256, + hash, }); } } diff --git a/lib/src/edit/fake.rs b/lib/src/edit/fake.rs index 0372b28..bc4e591 100644 --- a/lib/src/edit/fake.rs +++ b/lib/src/edit/fake.rs @@ -2,13 +2,13 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use tufaceous_artifact::ArtifactSet; use tufaceous_artifact::ArtifactVersion; -use tufaceous_artifact::Artifacts; use tufaceous_artifact::OsVariant; use tufaceous_artifact::RotBootloaderTags; +use tufaceous_artifact::RotSign; use tufaceous_artifact::RotSlot; use tufaceous_artifact::RotTags; -use tufaceous_artifact::Sign; use tufaceous_artifact::SpTags; use crate::edit::generate_installinator_document; @@ -34,13 +34,13 @@ const FAKE_ZONES: [(&str, &str); 11] = [ ("oximeter", "oximeter.tar.gz"), ]; -pub trait ArtifactsExt: Sized { +pub trait ArtifactSetExt: Sized { fn fake(version: ArtifactVersion) -> Result; } -impl ArtifactsExt for Artifacts { +impl ArtifactSetExt for ArtifactSet { fn fake(version: ArtifactVersion) -> Result { - let mut artifacts = Artifacts::default(); + let mut artifacts = ArtifactSet::default(); for input in Input::fake(&version, None)? { for output in input.outputs() { artifacts.extend(output.into_artifact()); @@ -90,7 +90,7 @@ impl Input { inputs.push(Self::fake_rot_archive( RotTags { rot_board: "SimRot".into(), - rot_sign: Sign::signed(FAKE_SIGN.into()), + rot_sign: RotSign(Some(FAKE_SIGN.into())), rot_slot: slot, }, version.clone(), @@ -100,7 +100,7 @@ impl Input { inputs.push(Self::fake_rot_bootloader_archive( RotBootloaderTags { rot_board: "SimRot".into(), - rot_sign: Sign::signed(FAKE_SIGN.into()), + rot_sign: RotSign(Some(FAKE_SIGN.into())), }, version.clone(), interior_version, @@ -129,11 +129,11 @@ mod tests { use std::time::Duration; use semver::Version; + use tufaceous_artifact::ArtifactSet; use tufaceous_artifact::ArtifactVersion; - use tufaceous_artifact::Artifacts; use crate::Repository; - use crate::edit::ArtifactsExt; + use crate::edit::ArtifactSetExt; #[tokio::test] async fn fake_artifacts_equals_fake_repo() { @@ -141,7 +141,7 @@ mod tests { let system_version = Version::new(1, 0, 0); let version = ArtifactVersion::new(system_version.to_string()).unwrap(); - let artifacts = Artifacts::fake(version).unwrap(); + let artifacts = ArtifactSet::fake(version).unwrap(); // sleep 1 second to ensure any embedded timestamps would be different. // we could ostensibly use `tokio::time::Instant` throughout the code // base but that wouldn't take into account third-party libraries we use diff --git a/lib/src/edit/hubris_archive.rs b/lib/src/edit/hubris_archive.rs index ff81aa2..d340eeb 100644 --- a/lib/src/edit/hubris_archive.rs +++ b/lib/src/edit/hubris_archive.rs @@ -8,13 +8,12 @@ use hubtools::Caboose; use hubtools::CabooseBuilder; use hubtools::HubrisArchiveBuilder; use tufaceous_artifact::ArtifactVersion; +use tufaceous_artifact::ReadCabooseError; use tufaceous_artifact::RotBootloaderTags; +use tufaceous_artifact::RotSign; use tufaceous_artifact::RotSlot; use tufaceous_artifact::RotTags; -use tufaceous_artifact::Sign; use tufaceous_artifact::SpTags; -use tufaceous_artifact::hubris::read_name; -use tufaceous_artifact::hubris::read_version; use crate::edit::guess::GuessInput; use crate::edit::guess::GuessResult; @@ -41,11 +40,12 @@ impl Input> { ReadCaboose, source.path() ); - let version = ArtifactVersion::new(try_path!( - read_version(&caboose), + let version = try_path!( + tag_helper(caboose.version(), "VERS"), ReadCaboose, source.path() - ))?; + ) + .parse()?; Ok(Self::Rot { source: source.into(), tags, version }) } @@ -62,12 +62,16 @@ impl Input> { ReadCaboose, source.path() ); - let version = ArtifactVersion::new(try_path!( - read_version(&caboose), + let version = try_path!( + tag_helper(caboose.version(), "VERS"), ReadCaboose, source.path() - ))?; - Ok(Self::RotBootloader { source: source.into(), tags, version }) + ); + Ok(Self::RotBootloader { + source: source.into(), + tags, + version: version.parse()?, + }) } pub(crate) async fn sp_archive( @@ -83,14 +87,22 @@ impl Input> { ReadCaboose, source.path() ); - let name = try_path!(read_name(&caboose), ReadCaboose, source.path()) - .to_owned(); - let version = ArtifactVersion::new(try_path!( - read_version(&caboose), + let name = try_path!( + tag_helper(caboose.name(), "NAME"), + ReadCaboose, + source.path() + ); + let version = try_path!( + tag_helper(caboose.version(), "VERS"), ReadCaboose, source.path() - ))?; - Ok(Self::Sp { source: source.into(), tags, name, version }) + ); + Ok(Self::Sp { + source: source.into(), + tags, + name, + version: version.parse()?, + }) } pub(crate) async fn guess_hubris_archive( @@ -167,7 +179,7 @@ impl Input { ) -> Result { let data = CabooseData { board: &tags.sp_board, - sign: &Sign::UNSIGNED, + sign: &RotSign(None), commit: "this-is-a-fake-sp", version: interior_version.unwrap_or(&version), }; @@ -178,7 +190,7 @@ impl Input { struct CabooseData<'a> { board: &'a str, - sign: &'a Sign, + sign: &'a RotSign, commit: &'static str, version: &'a ArtifactVersion, } @@ -205,3 +217,12 @@ impl CabooseData<'_> { Ok(BytesSource::new(vec)) } } + +fn tag_helper( + value: Result<&[u8], hubtools::CabooseError>, + tag: &'static str, +) -> Result { + let s = std::str::from_utf8(value?) + .map_err(|source| ReadCabooseError::Utf8 { tag, source })?; + Ok(s.to_owned()) +} diff --git a/lib/src/error.rs b/lib/src/error.rs index 3f436ec..6916e2a 100644 --- a/lib/src/error.rs +++ b/lib/src/error.rs @@ -135,7 +135,7 @@ pub enum ErrorKind { ReadHubrisArchive { source: hubtools::Error, path: Utf8PathBuf }, #[error("failed to read caboose from {path}")] ReadCaboose { - source: tufaceous_artifact::hubris::ReadCabooseError, + source: tufaceous_artifact::ReadCabooseError, path: Utf8PathBuf, }, #[error("failed to generate fake hubris archive")] diff --git a/lib/src/repo.rs b/lib/src/repo.rs index 92f5c90..41db454 100644 --- a/lib/src/repo.rs +++ b/lib/src/repo.rs @@ -26,16 +26,16 @@ use tough::schema::Hashes; use tough::schema::Target; use tufaceous_artifact::Artifact; use tufaceous_artifact::ArtifactHash; -use tufaceous_artifact::Artifacts; -use tufaceous_artifact::GetError; +use tufaceous_artifact::ArtifactSet; use tufaceous_artifact::KnownArtifactTags; use tufaceous_artifact::Metadata; +use tufaceous_artifact::artifact_set::GetError; use crate::RepositoryLoader; use crate::error::Error; use crate::error::ErrorKind; use crate::schema::ArtifactSchema; -use crate::schema::ArtifactsSchema; +use crate::schema::ArtifactSetSchema; pub type TargetStream = Pin> + Send + Sync + 'static>>; @@ -48,7 +48,7 @@ pub struct Repository { trust_root: Vec, pub(crate) archive_path: Option, pub(crate) archive_sha256: Option<[u8; 32]>, - artifacts: Artifacts, + artifacts: ArtifactSet, metadata: BTreeMap, v1_unpacked: Option, } @@ -109,8 +109,8 @@ impl Repository { archive_sha256: Option<[u8; 32]>, v1_compatibility: bool, ) -> Result { - let Some(ArtifactsSchema { system_version, artifacts, metadata }) = - read_target_json(&repo, ArtifactsSchema::TARGET_NAME).await? + let Some(ArtifactSetSchema { system_version, artifacts, metadata }) = + read_target_json(&repo, ArtifactSetSchema::TARGET_NAME).await? else { if v1_compatibility && let Some((system_version, artifacts, v1_unpacked)) = @@ -129,7 +129,7 @@ impl Repository { } return Err(ErrorKind::TargetNotFound { - target_name: ArtifactsSchema::TARGET_NAME.to_owned(), + target_name: ArtifactSetSchema::TARGET_NAME.to_owned(), } .into()); }; @@ -181,7 +181,7 @@ impl Repository { &self.inner.targets().signed.targets } - pub fn artifacts(&self) -> &Artifacts { + pub fn artifacts(&self) -> &ArtifactSet { &self.artifacts } @@ -226,7 +226,7 @@ impl Repository { /// Returns an error if there is not exactly one artifact matching `tags`. pub fn get_handle( self: &Arc, - tags: KnownArtifactTags, + tags: &KnownArtifactTags, ) -> Result { let artifact = self.artifacts.get(tags)?.clone(); Ok(ArtifactHandle { artifact, repo: Arc::clone(self) }) @@ -405,5 +405,5 @@ fn sha256_length( ); }) .ok()?; - Some((sha256, target.length)) + Some((ArtifactHash(sha256), target.length)) } diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs index 032408a..704ccad 100644 --- a/lib/src/repo/v1.rs +++ b/lib/src/repo/v1.rs @@ -34,18 +34,18 @@ use slog::warn; use tokio::sync::mpsc; use tufaceous_artifact::Artifact; use tufaceous_artifact::ArtifactHash; +use tufaceous_artifact::ArtifactSet; use tufaceous_artifact::ArtifactVersion; -use tufaceous_artifact::Artifacts; -use tufaceous_artifact::InstallinatorArtifact; -use tufaceous_artifact::InstallinatorDocument; use tufaceous_artifact::KnownArtifactTags; use tufaceous_artifact::OsBoard; use tufaceous_artifact::OsPhase1Tags; use tufaceous_artifact::OsPhase2Tags; use tufaceous_artifact::OsVariant; +use tufaceous_artifact::ReadCabooseError; use tufaceous_artifact::RotSlot; use tufaceous_artifact::ZoneTags; -use tufaceous_artifact::hubris::ReadCabooseError; +use tufaceous_artifact::installinator::InstallinatorArtifact; +use tufaceous_artifact::installinator::InstallinatorDocument; use crate::COSMO_PHASE_1_PATH; use crate::GIMLET_PHASE_1_PATH; @@ -67,14 +67,14 @@ pub(crate) struct Unpacked { pub(crate) async fn from_loaded( repo: &tough::Repository, log: &Logger, -) -> Result, Error> { - let Some(V1ArtifactsSchema { system_version, artifacts: v1_artifacts }) = - read_target_json(repo, V1ArtifactsSchema::TARGET_NAME).await? +) -> Result, Error> { + let Some(V1ArtifactSetSchema { system_version, artifacts: v1_artifacts }) = + read_target_json(repo, V1ArtifactSetSchema::TARGET_NAME).await? else { return Ok(None); }; - let mut artifacts = Artifacts::default(); + let mut artifacts = ArtifactSet::default(); let mut unpacked = Unpacked { entries: HashMap::new() }; let mut installinator_document = None; for V1Artifact { version, kind, target } in v1_artifacts { @@ -364,7 +364,7 @@ impl CompositeArtifact { async fn read_rot( mut self, - artifacts: &mut Artifacts, + artifacts: &mut ArtifactSet, unpacked: &mut Unpacked, version: ArtifactVersion, ) -> Result<(), Error> { @@ -406,7 +406,7 @@ impl CompositeArtifact { fn read_os_image( mut self, - artifacts: &mut Artifacts, + artifacts: &mut ArtifactSet, unpacked: &mut Unpacked, os_variant: OsVariant, version: &ArtifactVersion, @@ -449,7 +449,7 @@ impl CompositeArtifact { async fn read_control_plane( self, - artifacts: &mut Artifacts, + artifacts: &mut ArtifactSet, unpacked: &mut Unpacked, ) -> Result<(), Error> { for (tar_path, UnpackedArtifact { file, hash, length }) in self.entries @@ -519,7 +519,7 @@ impl BufRead for MpscReader { } async fn generate_installinator_document( - artifacts: &mut Artifacts, + artifacts: &mut ArtifactSet, unpacked: &mut Unpacked, version: ArtifactVersion, original_target: String, @@ -535,7 +535,7 @@ async fn generate_installinator_document( document.artifacts.insert(InstallinatorArtifact { file_name: file_name.to_owned(), kind, - sha256: artifact.hash, + hash: artifact.hash, }); } } @@ -562,12 +562,12 @@ async fn generate_installinator_document( } #[derive(Debug, Deserialize)] -pub(crate) struct V1ArtifactsSchema { +pub(crate) struct V1ArtifactSetSchema { system_version: Version, artifacts: Vec, } -impl V1ArtifactsSchema { +impl V1ArtifactSetSchema { pub(crate) const TARGET_NAME: &str = "artifacts.json"; } diff --git a/lib/src/schema.rs b/lib/src/schema.rs index 62a33d8..b40dd67 100644 --- a/lib/src/schema.rs +++ b/lib/src/schema.rs @@ -11,7 +11,7 @@ use tufaceous_artifact::ArtifactVersion; use tufaceous_artifact::KnownArtifactTags; #[derive(Debug, Deserialize, Serialize)] -pub(crate) struct ArtifactsSchema { +pub(crate) struct ArtifactSetSchema { pub(crate) system_version: Version, #[serde(default)] pub(crate) artifacts: Vec, @@ -19,7 +19,7 @@ pub(crate) struct ArtifactsSchema { pub(crate) metadata: BTreeMap, } -impl ArtifactsSchema { +impl ArtifactSetSchema { pub(crate) const TARGET_NAME: &str = "artifacts-v2.json"; } diff --git a/lib/tests/v1_compatibility.rs b/lib/tests/v1_compatibility.rs index 44d5a1c..67a9182 100644 --- a/lib/tests/v1_compatibility.rs +++ b/lib/tests/v1_compatibility.rs @@ -7,16 +7,16 @@ use tufaceous::ExpirationEnforcement; use tufaceous::RepositoryLoader; use tufaceous::TrustStoreBehavior; use tufaceous::error::Error; -use tufaceous_artifact::Artifacts; +use tufaceous_artifact::ArtifactSet; use tufaceous_artifact::KnownArtifactTags; use tufaceous_artifact::OsBoard; use tufaceous_artifact::OsPhase1Tags; use tufaceous_artifact::OsPhase2Tags; use tufaceous_artifact::OsVariant; use tufaceous_artifact::RotBootloaderTags; +use tufaceous_artifact::RotSign; use tufaceous_artifact::RotSlot; use tufaceous_artifact::RotTags; -use tufaceous_artifact::Sign; use tufaceous_artifact::SpTags; use tufaceous_artifact::ZoneTags; @@ -51,7 +51,7 @@ async fn v1_fake() -> Result<(), Error> { expected.push( RotTags { rot_board: "SimRot".into(), - rot_sign: Sign(Some(rot_sign.into())), + rot_sign: RotSign(Some(rot_sign.into())), rot_slot, } .into(), @@ -60,7 +60,7 @@ async fn v1_fake() -> Result<(), Error> { expected.push( RotBootloaderTags { rot_board: "SimRot".into(), - rot_sign: Sign(Some(rot_sign.into())), + rot_sign: RotSign(Some(rot_sign.into())), } .into(), ); @@ -72,9 +72,9 @@ async fn v1_fake() -> Result<(), Error> { expected.push(ZoneTags { zone_name: zone_name.into() }.into()); } let seen = expected - .into_iter() + .iter() .map(|tags| repo.artifacts().get(tags).unwrap().clone()) - .collect::(); + .collect::(); // And there should be no unexpected artifacts: assert_eq!(&seen, repo.artifacts()); From 159e0473f981e3688633055dc6c45350d83ea51e Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Wed, 15 Apr 2026 13:37:07 -0700 Subject: [PATCH 055/137] docs pass, remove InstallinatorArtifactId etc --- artifact/src/hubris.rs | 7 ++- artifact/src/installinator.rs | 81 +++++++++-------------------------- artifact/src/lib.rs | 5 ++- artifact/src/map.rs | 2 + artifact/src/metadata.rs | 27 ++++++------ artifact/src/set.rs | 2 + artifact/src/sign.rs | 5 ++- artifact/src/tags.rs | 29 ++++++++++++- artifact/src/version.rs | 13 +++++- lib/src/edit/editor.rs | 4 +- lib/src/repo/v1.rs | 6 +-- 11 files changed, 96 insertions(+), 85 deletions(-) diff --git a/artifact/src/hubris.rs b/artifact/src/hubris.rs index 84f1df8..4c3f041 100644 --- a/artifact/src/hubris.rs +++ b/artifact/src/hubris.rs @@ -136,5 +136,10 @@ pub enum ReadCabooseError { Caboose(#[from] hubtools::CabooseError), /// The tag value was not valid UTF-8. #[error("{tag} is not valid UTF-8")] - Utf8 { tag: &'static str, source: std::str::Utf8Error }, + Utf8 { + /// The tag that was not valid UTF-8. + tag: &'static str, + /// The underlying error. + source: std::str::Utf8Error, + }, } diff --git a/artifact/src/installinator.rs b/artifact/src/installinator.rs index 8c639df..0015f42 100644 --- a/artifact/src/installinator.rs +++ b/artifact/src/installinator.rs @@ -2,7 +2,6 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use std::borrow::Cow; use std::collections::BTreeSet; use serde::Deserialize; @@ -22,7 +21,9 @@ use crate::ArtifactVersion; /// version of Tufaceous that creates it. #[derive(Debug, Clone, Deserialize, Serialize)] pub struct InstallinatorDocument { + /// The system version of the repository this document is associated with. pub system_version: ArtifactVersion, + /// The list of Installinator artifacts. pub artifacts: BTreeSet, } @@ -39,20 +40,20 @@ impl InstallinatorDocument { Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize, )] pub struct InstallinatorArtifact { + /// The artifact kind. #[serde(flatten)] pub kind: InstallinatorArtifactKind, + /// The SHA256 hash of the artifact. pub hash: ArtifactHash, /// A file name without directory separators; not necessarily the target /// name. + // This alias is present for backwards compatibility with Tufaceous v1 + // repositories uploaded to older versions of Wicket that are still using + // Tufaceous v1. This alias can be removed when the transition is complete. + #[serde(alias = "name")] pub file_name: String, } -impl InstallinatorArtifact { - pub fn to_id(&self) -> InstallinatorArtifactId { - InstallinatorArtifactId { kind: self.kind.to_id(), hash: self.hash } - } -} - /// The artifact kind for an Installinator artifact. #[derive( Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize, @@ -64,57 +65,17 @@ pub enum InstallinatorArtifactKind { /// A measurement corpus. MeasurementCorpus, /// A control plane zone artifact. - Zone { zone_name: String }, -} - -impl InstallinatorArtifactKind { - pub fn to_id(&self) -> InstallinatorArtifactKindId { - InstallinatorArtifactKindId(match self { - InstallinatorArtifactKind::MeasurementCorpus => { - Cow::Borrowed("measurement_corpus") - } - InstallinatorArtifactKind::HostPhase2 => { - Cow::Borrowed("host_phase2") - } - InstallinatorArtifactKind::Zone { zone_name } => { - Cow::Owned(format!("zone-{zone_name}")) - } - }) - } -} - -/// Identifies an artifact that Installinator wants or has used. -/// -/// Historically this was called `ArtifactHashId` and consists of two -/// strings: `kind` and `hash`. It was developed before the present system -/// of tags representing an artifact kind but remains in use for any -/// Installinator-related interfaces (namely Wicket and mupdate overrides). -/// -/// This schema is stored to disk and should not change. -#[derive( - Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize, -)] -#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] -pub struct InstallinatorArtifactId { - pub kind: InstallinatorArtifactKindId, - pub hash: ArtifactHash, -} - -/// Encodes [`InstallinatorArtifactKind`] as a string. -/// -/// Used only in [`InstallinatorArtifactId`]. -#[derive( - Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize, -)] -#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] -#[serde(transparent)] -pub struct InstallinatorArtifactKindId(Cow<'static, str>); - -impl InstallinatorArtifactKindId { - pub const INSTALLINATOR_DOCUMENT: Self = - Self(Cow::Borrowed("installinator_document")); - - pub fn as_str(&self) -> &str { - &self.0 - } + Zone { + /// The zone name, as self-identified in the tarball's `oxide.json` + /// file. This may differ from the file name. + zone_name: String, + }, + /// A tarball of control plane zones. Used for backwards compatibility + /// only. + /// + /// This variant is present for backwards compatibility with Tufaceous + /// v1 repositories uploaded to older versions of Wicket that are still + /// using Tufaceous v1. This variant can be removed when the transition + /// is complete. + ControlPlane, } diff --git a/artifact/src/lib.rs b/artifact/src/lib.rs index 7dfe108..59194c9 100644 --- a/artifact/src/lib.rs +++ b/artifact/src/lib.rs @@ -5,9 +5,11 @@ //! tufaceous-artifact defines the core types of tufaceous so that they can be //! used without the full tufaceous library. +#![warn(missing_docs)] + mod artifact; mod hubris; -pub mod installinator; +mod installinator; mod map; mod metadata; mod set; @@ -17,6 +19,7 @@ mod version; pub use artifact::*; pub use hubris::ReadCabooseError; +pub use installinator::*; pub use metadata::*; pub use set::ArtifactSet; pub use sign::*; diff --git a/artifact/src/map.rs b/artifact/src/map.rs index de4d52b..0a325b6 100644 --- a/artifact/src/map.rs +++ b/artifact/src/map.rs @@ -13,6 +13,8 @@ pub(crate) fn from_map( serde_json::from_value(map.into_iter().collect()) } +/// Internal function to convert a `Serialize` to a `BTreeMap`. +/// /// This should always succeed, and callers must use proptests to ensure that /// it does. /// diff --git a/artifact/src/metadata.rs b/artifact/src/metadata.rs index baf82a6..794d97a 100644 --- a/artifact/src/metadata.rs +++ b/artifact/src/metadata.rs @@ -8,19 +8,20 @@ use serde::Deserialize; use serde::Serialize; /// Structured repository-level metadata stored in `artifacts-v2.json`. -/// -/// Similar to [`KnownArtifactTags`], this struct must serialize to and -/// deserialize from a mapping of string keys to string values. Additionally, it -/// is a requirement that all fields are optional; it must be possible to -/// deserialize this struct from older metadata, and the oldest metadata -/// is no metadata at all. -/// -/// As such, a couple of recommendations for adding new fields: -/// 1. Wrap all fields in [`Option`] and mark with -/// `#[serde(skip_serializing_if = "Option::is_none")]`. -/// 2. Mark nested structs with `#[serde(flatten)]`. -/// -/// [`KnownArtifactTags`]: crate::KnownArtifactTags +// +// NOTE: Similar to KnownArtifactTags, this struct must serialize to +// and deserialize from a mapping of string keys to string values. The +// `metadata_roundtrip` test covers this (crate::map::to_map panics when debug +// assertions are enabled if this does not hold). +// +// Additionally, it is a requirement that all fields are optional; it must +// be possible to deserialize this struct from older metadata, and the oldest +// metadata is no metadata at all. +// +// As such, a couple of recommendations for adding new fields: +// 1. Wrap all fields in [`Option`] and mark with +// `#[serde(skip_serializing_if = "Option::is_none")]`. +// 2. Mark nested structs with `#[serde(flatten)]`. #[derive(Debug, Default, Clone, PartialEq, Eq, Deserialize, Serialize)] #[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] pub struct Metadata { diff --git a/artifact/src/set.rs b/artifact/src/set.rs index bc8ce96..5bb278f 100644 --- a/artifact/src/set.rs +++ b/artifact/src/set.rs @@ -255,8 +255,10 @@ impl<'a> Iterator for Iter<'a> { /// artifact. #[derive(Debug, thiserror::Error)] pub enum GetError { + /// An artifact matching the requested tags was not found. #[error("artifact not found")] NotFound, + /// There were multiple artifacts matching the requested tags. #[error("more than one artifact found")] TooMany, } diff --git a/artifact/src/sign.rs b/artifact/src/sign.rs index 136a0fa..97d08fd 100644 --- a/artifact/src/sign.rs +++ b/artifact/src/sign.rs @@ -55,8 +55,9 @@ static CA_MAP: LazyLock> = /// /// Used in [`RotTags`] and [`RotBootloaderTags`]. /// -/// This is a lowercase hexadecimal string of the ROT Key Table Hash (RKTH) that -/// is used to identify the CA. +/// This is usually a lowercase hexadecimal string of the ROT Key Table Hash +/// (RKTH) that is used to identify the CA, but this is not enforced by the +/// library. /// /// [`RotTags`]: crate::RotTags /// [`RotBootloaderTags`]: crate::RotBootloaderTags diff --git a/artifact/src/tags.rs b/artifact/src/tags.rs index 7c61b43..87d75da 100644 --- a/artifact/src/tags.rs +++ b/artifact/src/tags.rs @@ -13,6 +13,11 @@ use crate::RotSign; use crate::installinator::InstallinatorArtifactKind; /// Sets of artifact tags known to the control plane. +// +// NOTE: This struct must serialize and deserialize from a mapping of +// string keys to string values. The `tags_roundtrip` test covers this +// (crate::map::to_map panics when debug assertions are enabled if this does +// not hold). #[derive( Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, )] @@ -47,20 +52,30 @@ pub enum KnownArtifactTags { } impl KnownArtifactTags { + /// Returns an adapter for displaying the tags as a human-readable string. pub fn display(&self) -> DisplayTags<'static> { self.to_tags().into() } + /// Resolves known tags from a tag mapping. + /// + /// # Errors + /// + /// Returns an error if the `kind` tag is missing, not a known kind, or + /// required tags for that kind are not present. pub fn from_tags( tags: BTreeMap, ) -> Result { crate::map::from_map(tags) } + /// Converts these known tags to a tag mapping. pub fn to_tags(&self) -> BTreeMap { crate::map::to_map(self) } + /// Converts these tags into an [`InstallinatorArtifactKind`] if this + /// artifact kind should be included in the Installinator document. pub fn to_installinator(&self) -> Option { match self { KnownArtifactTags::MeasurementCorpus => { @@ -89,12 +104,15 @@ macro_rules! display_serialize { }; } +/// The inner value of [`KnownArtifactTags::OsPhase1`]. #[derive( Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, )] #[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] pub struct OsPhase1Tags { + /// OS board artifact tag (gimlet or cosmo). pub os_board: OsBoard, + /// OS variant artifact tag (host or recovery). pub os_variant: OsVariant, } @@ -104,11 +122,13 @@ impl From for KnownArtifactTags { } } +/// The inner value of [`KnownArtifactTags::OsPhase2`]. #[derive( Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, )] #[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] pub struct OsPhase2Tags { + /// OS variant artifact tag (host or recovery). pub os_variant: OsVariant, } @@ -134,7 +154,10 @@ impl From for KnownArtifactTags { #[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] #[serde(rename_all = "snake_case")] pub enum OsVariant { + /// The host OS. Host, + /// The recovery OS (sometimes called the trampoline OS), which contains + /// Installinator and is used to install the host OS. Recovery, } display_serialize!(OsVariant); @@ -155,7 +178,9 @@ display_serialize!(OsVariant); #[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] #[serde(rename_all = "snake_case")] pub enum OsBoard { + /// First-generation SP3 compute board. Gimlet, + /// Second-generation SP5 compute board. Cosmo, } display_serialize!(OsBoard); @@ -203,7 +228,9 @@ impl From for KnownArtifactTags { #[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] #[serde(rename_all = "snake_case")] pub enum RotSlot { + /// Slot A. A, + /// Slot B. B, } display_serialize!(RotSlot); @@ -238,7 +265,7 @@ impl From for KnownArtifactTags { )] #[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] pub struct SpTags { - /// The `BORD` field in the caboose (such as `oxide-rot-1`). + /// The `BORD` field in the caboose (such as `cosmo-b`). pub sp_board: String, } diff --git a/artifact/src/version.rs b/artifact/src/version.rs index 4693bee..ae19701 100644 --- a/artifact/src/version.rs +++ b/artifact/src/version.rs @@ -183,19 +183,28 @@ static PROPTEST_REGEX: &str = { #[derive(Debug, Clone, PartialEq, Eq, Error)] #[non_exhaustive] pub enum ArtifactVersionError { + /// The version string was empty. #[error("version is empty")] Empty, + /// The version string was too long. #[error( "version is too long ({len} bytes, max {})", ArtifactVersion::MAX_LEN )] - TooLong { len: usize }, + TooLong { + /// Length of the version string that was too long. + len: usize, + }, #[error( "version contains invalid byte `{}` (allowed: {})", b.escape_ascii(), ArtifactVersion::REGEX )] - InvalidByte { b: u8 }, + /// The version string contained an invalid byte. + InvalidByte { + /// The first invalid byte. + b: u8, + }, } impl ArtifactVersionError { diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index 7363041..c89fa4f 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -15,6 +15,8 @@ use tokio::task::JoinSet; use tufaceous_artifact::ArtifactHash; use tufaceous_artifact::ArtifactVersion; use tufaceous_artifact::ArtifactVersionError; +use tufaceous_artifact::InstallinatorArtifact; +use tufaceous_artifact::InstallinatorDocument; use tufaceous_artifact::KnownArtifactTags; use tufaceous_artifact::Metadata; use tufaceous_artifact::OsVariant; @@ -22,8 +24,6 @@ use tufaceous_artifact::RotBootloaderTags; use tufaceous_artifact::RotSlot; use tufaceous_artifact::RotTags; use tufaceous_artifact::SpTags; -use tufaceous_artifact::installinator::InstallinatorArtifact; -use tufaceous_artifact::installinator::InstallinatorDocument; use crate::Repository; use crate::edit::UnsignedRepository; diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs index 704ccad..7713aba 100644 --- a/lib/src/repo/v1.rs +++ b/lib/src/repo/v1.rs @@ -36,6 +36,8 @@ use tufaceous_artifact::Artifact; use tufaceous_artifact::ArtifactHash; use tufaceous_artifact::ArtifactSet; use tufaceous_artifact::ArtifactVersion; +use tufaceous_artifact::InstallinatorArtifact; +use tufaceous_artifact::InstallinatorDocument; use tufaceous_artifact::KnownArtifactTags; use tufaceous_artifact::OsBoard; use tufaceous_artifact::OsPhase1Tags; @@ -44,8 +46,6 @@ use tufaceous_artifact::OsVariant; use tufaceous_artifact::ReadCabooseError; use tufaceous_artifact::RotSlot; use tufaceous_artifact::ZoneTags; -use tufaceous_artifact::installinator::InstallinatorArtifact; -use tufaceous_artifact::installinator::InstallinatorDocument; use crate::COSMO_PHASE_1_PATH; use crate::GIMLET_PHASE_1_PATH; @@ -454,7 +454,7 @@ impl CompositeArtifact { ) -> Result<(), Error> { for (tar_path, UnpackedArtifact { file, hash, length }) in self.entries { - if !tar_path.starts_with("zones") { + if !tar_path.starts_with("zones/") { continue; } let target_name = From 76b6c4ba6846bd48406b157884e7484de1e88304 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Wed, 15 Apr 2026 15:39:36 -0700 Subject: [PATCH 056/137] oops --- artifact/src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/artifact/src/lib.rs b/artifact/src/lib.rs index 59194c9..9df2069 100644 --- a/artifact/src/lib.rs +++ b/artifact/src/lib.rs @@ -18,7 +18,8 @@ mod tags; mod version; pub use artifact::*; -pub use hubris::ReadCabooseError; +#[cfg(feature = "hubtools")] +pub use hubris::*; pub use installinator::*; pub use metadata::*; pub use set::ArtifactSet; From cce2ef226924cb03bc95334ee000de01b09e90c0 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Fri, 17 Apr 2026 19:58:13 -0700 Subject: [PATCH 057/137] upstream tough PR got merged yay --- Cargo.lock | 4 ++-- Cargo.toml | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1483372..98c6c26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1560,7 +1560,7 @@ dependencies = [ [[package]] name = "olpc-cjson" version = "0.1.4" -source = "git+https://github.com/iliana/tough.git?branch=transport-is-sync#47aff7d174914f46db6166dbade64124849f2468" +source = "git+https://github.com/awslabs/tough.git?branch=develop#c9a9865dc2d64a65c9930ad5daf4930e556662c5" dependencies = [ "serde", "serde_json", @@ -2796,7 +2796,7 @@ dependencies = [ [[package]] name = "tough" version = "0.21.0" -source = "git+https://github.com/iliana/tough.git?branch=transport-is-sync#47aff7d174914f46db6166dbade64124849f2468" +source = "git+https://github.com/awslabs/tough.git?branch=develop#c9a9865dc2d64a65c9930ad5daf4930e556662c5" dependencies = [ "async-recursion", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 77a6d91..797df1e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,8 +50,7 @@ test-strategy = "0.4.0" thiserror = "2.0.12" tokio = "1.47.0" tokio-test = "0.4.5" -# https://github.com/awslabs/tough/pull/917 -tough = { git = "https://github.com/iliana/tough.git", branch = "transport-is-sync" } +tough = { git = "https://github.com/awslabs/tough.git", branch = "develop" } tufaceous.path = "lib" tufaceous-artifact.path = "artifact" tufaceous-brand-metadata.path = "brand-metadata" From 865bf09e94aa25678d0b57ce11de250728458fb9 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 20 Apr 2026 16:07:38 -0700 Subject: [PATCH 058/137] README; problem checker; verify subcommand --- Cargo.lock | 1 + README.md | 66 ++++++++++++++++ artifact/src/artifact.rs | 4 +- artifact/src/set.rs | 9 +++ bin/src/main.rs | 8 ++ bin/src/verify.rs | 65 ++++++++++++++++ lib/Cargo.toml | 1 + lib/src/check.rs | 157 +++++++++++++++++++++++++++++++++++++++ lib/src/edit/source.rs | 3 +- lib/src/error.rs | 2 +- lib/src/lib.rs | 2 + lib/src/repo.rs | 14 ++++ lib/src/repo/v1.rs | 75 +++++++++++++++++-- 13 files changed, 394 insertions(+), 13 deletions(-) create mode 100644 README.md create mode 100644 bin/src/verify.rs create mode 100644 lib/src/check.rs diff --git a/Cargo.lock b/Cargo.lock index 98c6c26..aca43d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2837,6 +2837,7 @@ dependencies = [ "camino-tempfile", "chrono", "ciborium", + "daft", "flate2", "futures-util", "hex", diff --git a/README.md b/README.md new file mode 100644 index 0000000..f9f097b --- /dev/null +++ b/README.md @@ -0,0 +1,66 @@ +# Tufaceous + +Tufaceous is the library and tooling for Oxide update repositories. It is a layer of Oxide-specific tooling on top of [tough](https://github.com/awslabs/tough), a Rust library that implements [TUF](https://theupdateframework.io/). + +This README is intended to provide a high-level overview of the library and instructions for adding to it. More context can be found in [RFD 621](https://rfd.shared.oxide.computer/rfd/0621). + +## Repository format + +We selected ZIP archives as the primary format for our repositories, despite ZIP's many shortcomings. Because Oxide customers are receiving update repositories and uploading them to their systems, we wanted them to be readable for most users so that they could see what is inside. + +The ZIP archive has a top-level `repo` directory. Inside this directory is a `metadata` directory, containing the signed TUF metadata, and a `targets` directory, containing all the targets described by that metadata. + +## Artifacts + +"Artifacts" are our jargon for a file that the Oxide control plane needs in order to perform updates. (All artifacts are "targets", TUF jargon for a file described by the repository metadata, but not all targets are artifacts.) + +Artifacts in the repository are described by the `artifacts-v2.json`. Each artifact has a set of tags that is used by the control plane to decide what to do with the artifact. When creating and using artifacts, these tags are strongly-typed (see `KnownArtifactTags` in `artifact/src/tags.rs`), but they are encoded as an arbitrary key-value mapping. This allows us to potentially release a new type of artifact and use it in the same release, once Nexus has been updated. + +As a general rule it is not possible to use this tooling to create a repository with arbitrary blobs and artifacts in it. The library is very opinionated about how artifacts are named to keep things usable for the CLI and for users of these repositories. + +## Tooling + +Most of the time you're working with repositories generated by the [Omicron release tooling](https://github.com/oxidecomputer/omicron/blob/main/docs/releng.adoc) (`cargo xtask releng` in Omicron). There is also a binary target that is a thin CLI on top of the library, which you can build with `cargo build --release --bin tufaceous`. + +### Verify a repository + +`tufaceous verify` will read a repository, print its SHA256 checksum, and verify the metadata and checksums of all the metadata in the repository. It will also run a series of compatibility checks on the repository and warn you of any detected issues. + +### Unpack a repository + +Scripts can use `tufaceous list-targets` to get a list of targets from the repository, and `tufaceous show-target` to write a target to stdout. + +To unpack an entire repository, use `unzip` or ZIP tooling provided by your OS. + +### Build a repository + +`tufaceous build` has two required flags: + +- `--output file.zip`, which provides the path to write to +- `--version 1.0.0`, which provides the system version + +The arguments are the set of artifacts to add: + +- A measurement corpus is a CoRIM file +- An OS image is a directory containing `cosmo.rom`, `gimlet.rom`, and `zfs.img`, as produced by `helios-build image` +- An RoT, RoT bootloader, or SP image is a Hubris archive (ZIP format) +- A zone image is a tarball that starts with the `oxide.json` Omicron brand metadata + +When building a repository, an Installinator document is automatically created based on the artifacts in the repository. You can avoid this with `--no-installinator-document`. + +### Add/remove artifacts from a repository + +Using `tufaceous edit` you can remove targets from a repository using `--remove-targets` and add new artifacts using `--add-artifacts`. + +The Installinator document is automatically regenerated based on the artifacts in the final repository. You can avoid this with `--no-installinator-document`, which will keep an existing Installinator document if there is one. + +## Adding new artifacts + +In order to add a new type of artifact, you will need to make a series of changes to the library: + +- Modify `KnownArtifactTags` to describe your new artifact +- Add methods for adding an actual artifact and a fake artifact to `RepositoryEditor` +- Add a fake artifact in `Input::fake` (`lib/src/edit/fake.rs`) +- Add code for guessing this type of artifact to `Input::guess` (`lib/src/edit/guess.rs`) + +As a general rule, Tufaceous artifacts are self-describing. The "guess" code is used by the CLI and generates tags from the artifact to avoid needing an unwieldy set of command line options to describe tags. If your artifact is not self-describing, you can staple a header of some kind to the beginning of the artifact that control plane code knows to strip before using the artifact. diff --git a/artifact/src/artifact.rs b/artifact/src/artifact.rs index 84be4ed..50a70b6 100644 --- a/artifact/src/artifact.rs +++ b/artifact/src/artifact.rs @@ -42,7 +42,7 @@ pub struct Artifact { /// record these tags as-is, even if `Artifact::known_tags` returns `None`. pub tags: BTreeMap, - /// The SHA-256 checksum of the artifact. + /// The SHA256 checksum of the artifact. pub hash: ArtifactHash, /// The length of the artifact in bytes. @@ -64,7 +64,7 @@ impl Artifact { } } -/// The SHA-256 checksum of an artifact. +/// The SHA256 checksum of an artifact. #[derive( Copy, Clone, diff --git a/artifact/src/set.rs b/artifact/src/set.rs index 5bb278f..53c0efd 100644 --- a/artifact/src/set.rs +++ b/artifact/src/set.rs @@ -130,6 +130,15 @@ impl ArtifactSet { pub fn iter(&self) -> Iter<'_> { Iter { inner: self.known.values().flatten().chain(&self.unknown) } } + + /// Returns an iterator over the known artifacts in the set. + pub fn known( + &self, + ) -> impl Iterator< + Item = (&KnownArtifactTags, impl ExactSizeIterator), + > { + self.known.iter().map(|(tags, artifacts)| (tags, artifacts.iter())) + } } impl Extend for ArtifactSet { diff --git a/bin/src/main.rs b/bin/src/main.rs index 603cc15..fa07b1b 100644 --- a/bin/src/main.rs +++ b/bin/src/main.rs @@ -7,6 +7,7 @@ mod edit; mod list_targets; mod show_target; mod sign; +mod verify; use std::sync::LazyLock; @@ -33,12 +34,18 @@ static LOG: LazyLock = LazyLock::new(|| { #[derive(Debug, Parser)] enum Command { + /// Build a new repository from a set of artifacts Build(build::Args), + /// Add artifacts to and remove targets from a repository Edit(edit::Args), + /// List the targets in a repository #[clap(alias = "ls")] ListTargets(list_targets::Args), + /// Write a target to stdout #[clap(aliases = ["cat", "show"])] ShowTarget(show_target::Args), + /// Verify the integrity of a repository and check for issues + Verify(verify::Args), } #[tokio::main] @@ -49,5 +56,6 @@ async fn main() -> Result<()> { Command::Edit(args) => args.run().await, Command::ListTargets(args) => args.run().await, Command::ShowTarget(args) => args.run().await, + Command::Verify(args) => args.run().await, } } diff --git a/bin/src/verify.rs b/bin/src/verify.rs new file mode 100644 index 0000000..e05b68a --- /dev/null +++ b/bin/src/verify.rs @@ -0,0 +1,65 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::fmt::Display; + +use anyhow::Result; +use anyhow::ensure; +use camino::Utf8PathBuf; +use clap::Parser; +use tough::ExpirationEnforcement; +use tufaceous::CheckProblem; +use tufaceous::RepositoryLoader; +use tufaceous::TrustStoreBehavior; + +#[derive(Debug, Parser)] +pub struct Args { + /// Path to the repository + repo: Utf8PathBuf, + /// Number of threads to use while verifying targets + #[clap(short = 'j', alias = "jobs", default_value_t = default_threads())] + threads: usize, +} + +fn default_threads() -> usize { + std::thread::available_parallelism() + .map(std::num::NonZero::get) + .unwrap_or(1) +} + +impl Args { + pub async fn run(self) -> Result<()> { + let repo = RepositoryLoader::new() + .expiration_enforcement(ExpirationEnforcement::Unsafe) + .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .v1_compatibility(true) + .load_zip_path(self.repo.clone(), &crate::LOG) + .await?; + + repo.verify_targets(self.threads).await?; + + let problems = repo.check_problems(); + ensure!( + problems.is_empty(), + "found compatibility problems:\n{}", + WriteProblems(&problems) + ); + + eprintln!("{}: OK", self.repo); + Ok(()) + } +} + +struct WriteProblems<'a>(&'a [CheckProblem]); + +impl Display for WriteProblems<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut nl = ""; + for problem in self.0 { + write!(f, "{nl}- {problem}")?; + nl = "\n"; + } + Ok(()) + } +} diff --git a/lib/Cargo.toml b/lib/Cargo.toml index b33cb0e..3ed8a93 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -13,6 +13,7 @@ camino.workspace = true camino-tempfile.workspace = true chrono.workspace = true ciborium.workspace = true +daft.workspace = true flate2.workspace = true futures-util.workspace = true hex.workspace = true diff --git a/lib/src/check.rs b/lib/src/check.rs new file mode 100644 index 0000000..6128c50 --- /dev/null +++ b/lib/src/check.rs @@ -0,0 +1,157 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::BTreeMap; + +use daft::Diffable; +use tufaceous_artifact::DisplayTags; +use tufaceous_artifact::KnownArtifactTags; +use tufaceous_artifact::OsBoard; +use tufaceous_artifact::OsPhase1Tags; +use tufaceous_artifact::OsPhase2Tags; +use tufaceous_artifact::OsVariant; +use tufaceous_artifact::artifact_set::GetError; + +use crate::Repository; + +impl Repository { + /// Check the repository for consistency and other problems. + /// + /// This must *not* be used to determine whether to accept a repository by + /// the control plane; it assumes that the current version of Tufaceous is + /// the same one that was used to generate the repository. + pub fn check_problems(&self) -> Vec { + let mut problems = Vec::new(); + + for artifact in self.artifacts() { + // Check that all targets listed in `artifacts-v2.json` actually + // exist in the repository. + if !self.contains_target(&artifact.target_name) { + problems.push(CheckProblem::MissingTarget { + target_name: artifact.target_name.clone(), + }); + } + // Check that no artifact contains unknown tags. + if artifact.known_tags().is_none() { + problems.push(CheckProblem::UnknownTags { + target_name: artifact.target_name.clone(), + tags: artifact.tags.clone(), + }); + } + } + + match self.structured_metadata() { + Some(structured_metadata) => { + // Serialize the metadata back to a mapping and verify there + // are no unexpected changes. + let map = structured_metadata.to_map(); + if self.metadata() != &map { + let diff = self.metadata().diff(&map); + for (key, leaf) in diff.modified() { + problems.push(CheckProblem::UnexpectedMetadataDiff { + key: key.clone(), + old_value: Some(leaf.before.clone()), + new_value: leaf.after.clone(), + }); + } + for (key, value) in diff.added { + problems.push(CheckProblem::UnexpectedMetadataDiff { + key: key.clone(), + old_value: None, + new_value: value.clone(), + }); + } + for (key, _) in diff.removed { + problems.push(CheckProblem::UnknownMetadataKey( + key.clone(), + )); + } + } + } + None => { + problems.push(CheckProblem::UnknownMetadata( + self.metadata().clone(), + )); + } + } + + // For all tags we expect to see a single artifact, except for + // MeasurementCorpus. + for (tags, iter) in self.artifacts().known() { + if !matches!(tags, KnownArtifactTags::MeasurementCorpus) + && iter.len() > 1 + { + problems.push(CheckProblem::MultipleArtifacts(tags.clone())); + } + } + + // We expect to see artifacts matching these tags for Installinator + // to work. + let mut expected = vec![KnownArtifactTags::InstallinatorDocument]; + for os_variant in [OsVariant::Host, OsVariant::Recovery] { + for os_board in [OsBoard::Cosmo, OsBoard::Gimlet] { + expected.push(KnownArtifactTags::OsPhase1(OsPhase1Tags { + os_board, + os_variant, + })); + } + expected + .push(KnownArtifactTags::OsPhase2(OsPhase2Tags { os_variant })); + } + for tags in expected { + if matches!(self.artifacts().get(&tags), Err(GetError::NotFound)) { + problems.push(CheckProblem::MissingArtifact(tags)); + } + } + + problems + } +} + +/// Possible problems found by [`Repository::check_problems`]. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum CheckProblem { + /// An artifact matching these tags was not found. + #[error("no artifact matching {}", .0.display())] + MissingArtifact(KnownArtifactTags), + + /// An artifact's target name is not in the repository. + #[error( + "artifact with target name {target_name} listed but not present \ + in repository" + )] + MissingTarget { target_name: String }, + + /// Multiple artifacts for these tags were not expected. + #[error("multiple artifacts found matching {}", .0.display())] + MultipleArtifacts(KnownArtifactTags), + + /// The repository metadata is unexpectedly different from the parsed + /// structured metadata. + #[error( + "unexpected diff in structured metadata: \ + {key}={old_value:?}->{new_value:?}" + )] + UnexpectedMetadataDiff { + key: String, + old_value: Option, + new_value: String, + }, + + /// The repository metadata cannot be parsed. + #[error("couldn't parse metadata {0:?}")] + UnknownMetadata(BTreeMap), + + /// The repository metadata contains an unknown key. + #[error("unknown metadata key {0:?}")] + UnknownMetadataKey(String), + + /// An artifact has unknown tags. + #[error( + "artifact {target_name} has unknown tags {}", + DisplayTags::from(.tags) + )] + UnknownTags { target_name: String, tags: BTreeMap }, +} diff --git a/lib/src/edit/source.rs b/lib/src/edit/source.rs index cc5213c..86ba8db 100644 --- a/lib/src/edit/source.rs +++ b/lib/src/edit/source.rs @@ -277,9 +277,8 @@ impl FileSource { } pub(crate) fn stream(&self) -> impl Stream> { - let inner = self.inner.clone(); stream::try_unfold( - (inner, BytesMut::new(), 0), + (self.inner.clone(), BytesMut::new(), 0), async |(inner, mut buf, mut offset)| { tokio::task::spawn_blocking(move || { if buf.capacity() == 0 { diff --git a/lib/src/error.rs b/lib/src/error.rs index 6916e2a..0a90fe8 100644 --- a/lib/src/error.rs +++ b/lib/src/error.rs @@ -350,7 +350,7 @@ impl Display for SpacePath<'_> { } } -struct DebugByteString<'a>(&'a [u8]); +pub(crate) struct DebugByteString<'a>(pub(crate) &'a [u8]); impl Debug for DebugByteString<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 0ca62bf..27a1354 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -2,6 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +mod check; pub mod edit; pub mod error; mod loader; @@ -11,6 +12,7 @@ mod util; mod zip_transport; mod zip_writer; +pub use check::*; pub use loader::*; pub use repo::*; pub use zip_transport::*; diff --git a/lib/src/repo.rs b/lib/src/repo.rs index 41db454..cbc2529 100644 --- a/lib/src/repo.rs +++ b/lib/src/repo.rs @@ -197,6 +197,20 @@ impl Repository { self.v1_unpacked.is_some() } + pub(crate) fn contains_target(&self, target: &str) -> bool { + if let Ok(target_name) = target.parse() + && self.targets().contains_key(&target_name) + { + true + } else if let Some(unpacked) = &self.v1_unpacked + && unpacked.entries.contains_key(target) + { + true + } else { + false + } + } + pub async fn read_target( &self, target: &str, diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs index 7713aba..7de166f 100644 --- a/lib/src/repo/v1.rs +++ b/lib/src/repo/v1.rs @@ -30,6 +30,7 @@ use serde::Deserialize; use sha2::Digest; use sha2::Sha256; use slog::Logger; +use slog::info; use slog::warn; use tokio::sync::mpsc; use tufaceous_artifact::Artifact; @@ -50,6 +51,7 @@ use tufaceous_artifact::ZoneTags; use crate::COSMO_PHASE_1_PATH; use crate::GIMLET_PHASE_1_PATH; use crate::PHASE_2_PATH; +use crate::error::DebugByteString; use crate::error::Error; use crate::error::ErrorKind; use crate::error::try_path; @@ -100,29 +102,66 @@ pub(crate) async fn from_loaded( | V1KnownArtifactKind::SwitchSp => { let image = read_target_vec(repo, &target).await?; let Some(image) = image else { continue }; - caboose_tags( - image, - &target, - KnownArtifactTags::from_sp_caboose, - )? + let mut is_lab_image = false; + let tags = caboose_tags(image, &target, |caboose| { + if let Ok(board) = caboose.board() + && let Ok(name) = caboose.name() + && board != name + { + // This is a lab image. These are stored in the TUF repo + // for manufacturing but are not used in the control + // plane, as they can never be used in an actual rack. + info!( + log, + "skipping lab SP image"; + "board" => ?DebugByteString(board), + "name" => ?DebugByteString(name), + ); + is_lab_image = true; + } + KnownArtifactTags::from_sp_caboose(caboose) + })?; + if is_lab_image { + continue; + } + tags } V1KnownArtifactKind::GimletRotBootloader | V1KnownArtifactKind::PscRotBootloader | V1KnownArtifactKind::SwitchRotBootloader => { let image = read_target_vec(repo, &target).await?; let Some(image) = image else { continue }; - caboose_tags( + let tags = caboose_tags( image, &target, KnownArtifactTags::from_rot_bootloader_caboose, - )? + )?; + if artifacts.get_all(&tags).iter().any(|artifact| { + if let Some(existing) = + sha256_length(repo, log, &artifact.target_name) + && (hash, length) == existing + { + info!( + log, + "skipping duplicate ROT bootloader image"; + "existing" => &artifact.target_name, + "skipped" => &target, + ); + true + } else { + false + } + }) { + continue; + } + tags } V1KnownArtifactKind::GimletRot | V1KnownArtifactKind::PscRot | V1KnownArtifactKind::SwitchRot => { CompositeArtifact::unpack(repo, target) .await? - .read_rot(&mut artifacts, &mut unpacked, version) + .read_rot(log, &mut artifacts, &mut unpacked, version) .await?; continue; } @@ -364,6 +403,7 @@ impl CompositeArtifact { async fn read_rot( mut self, + log: &Logger, artifacts: &mut ArtifactSet, unpacked: &mut Unpacked, version: ArtifactVersion, @@ -390,6 +430,25 @@ impl CompositeArtifact { let tags = caboose_tags(image, &target_name, |caboose| { KnownArtifactTags::from_rot_caboose(caboose, slot) })?; + if artifacts.get_all(&tags).iter().any(|artifact| { + if let Some(existing) = + unpacked.entries.get(&artifact.target_name) + && hash == existing.hash + && length == existing.length + { + info!( + log, + "skipping duplicate ROT image"; + "existing" => &artifact.target_name, + "skipped" => &target_name, + ); + true + } else { + false + } + }) { + continue; + } artifacts.insert(Artifact { target_name: target_name.clone(), version: version.clone(), From 3fdbb39ca66a03f047bacc35cf8721eb4f8f686c Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 21 Apr 2026 09:53:17 -0700 Subject: [PATCH 059/137] further checks and tests for some checks --- bin/src/verify.rs | 8 ++++++- lib/src/check.rs | 37 ++++++++++++++++++++++++++++- lib/src/edit/editor.rs | 18 ++++++-------- lib/src/repo/v1.rs | 18 +++----------- lib/src/util.rs | 27 +++++++++++++++++++++ lib/tests/check.rs | 53 ++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 133 insertions(+), 28 deletions(-) create mode 100644 lib/tests/check.rs diff --git a/bin/src/verify.rs b/bin/src/verify.rs index e05b68a..da5b60b 100644 --- a/bin/src/verify.rs +++ b/bin/src/verify.rs @@ -2,6 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use std::error::Error; use std::fmt::Display; use anyhow::Result; @@ -39,7 +40,7 @@ impl Args { repo.verify_targets(self.threads).await?; - let problems = repo.check_problems(); + let problems = repo.check_problems().await; ensure!( problems.is_empty(), "found compatibility problems:\n{}", @@ -58,6 +59,11 @@ impl Display for WriteProblems<'_> { let mut nl = ""; for problem in self.0 { write!(f, "{nl}- {problem}")?; + let mut source = problem.source(); + while let Some(s) = source { + write!(f, ": {s}")?; + source = s.source(); + } nl = "\n"; } Ok(()) diff --git a/lib/src/check.rs b/lib/src/check.rs index 6128c50..e8a3824 100644 --- a/lib/src/check.rs +++ b/lib/src/check.rs @@ -5,7 +5,10 @@ use std::collections::BTreeMap; use daft::Diffable; +use futures_util::TryStreamExt; +use tufaceous_artifact::ArtifactVersion; use tufaceous_artifact::DisplayTags; +use tufaceous_artifact::InstallinatorDocument; use tufaceous_artifact::KnownArtifactTags; use tufaceous_artifact::OsBoard; use tufaceous_artifact::OsPhase1Tags; @@ -21,7 +24,7 @@ impl Repository { /// This must *not* be used to determine whether to accept a repository by /// the control plane; it assumes that the current version of Tufaceous is /// the same one that was used to generate the repository. - pub fn check_problems(&self) -> Vec { + pub async fn check_problems(&self) -> Vec { let mut problems = Vec::new(); for artifact in self.artifacts() { @@ -105,6 +108,24 @@ impl Repository { } } + if let Ok(artifact) = + self.artifacts().get(&KnownArtifactTags::InstallinatorDocument) + && let Ok(stream) = self.read_target(&artifact.target_name).await + && let Ok(bytes) = stream.map_ok(Vec::from).try_concat().await + && let Ok(doc) = serde_json::from_slice::( + &bytes, + ) + .map_err(|source| { + problems.push(CheckProblem::DeserializeInstallinator(source)); + }) + && doc.system_version.as_str() != self.system_version().to_string() + { + problems.push(CheckProblem::InstallinatorVersion { + doc_version: doc.system_version.clone(), + system_version: self.system_version().clone(), + }); + } + problems } } @@ -113,6 +134,20 @@ impl Repository { #[derive(Debug, thiserror::Error)] #[non_exhaustive] pub enum CheckProblem { + /// The Installinator document could not be parsed. + #[error("failed to deserialize Installinator document")] + DeserializeInstallinator(#[source] serde_json::Error), + + /// The Installinator document has the wrong system version. + #[error( + "Installinator document has version {doc_version} \ + but the system version is {system_version}" + )] + InstallinatorVersion { + doc_version: ArtifactVersion, + system_version: semver::Version, + }, + /// An artifact matching these tags was not found. #[error("no artifact matching {}", .0.display())] MissingArtifact(KnownArtifactTags), diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index c89fa4f..e83f6f2 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -15,7 +15,6 @@ use tokio::task::JoinSet; use tufaceous_artifact::ArtifactHash; use tufaceous_artifact::ArtifactVersion; use tufaceous_artifact::ArtifactVersionError; -use tufaceous_artifact::InstallinatorArtifact; use tufaceous_artifact::InstallinatorDocument; use tufaceous_artifact::KnownArtifactTags; use tufaceous_artifact::Metadata; @@ -493,17 +492,14 @@ pub(crate) fn generate_installinator_document( let mut document = InstallinatorDocument::empty(version.clone()); for (artifact, hash) in artifacts { let artifact = artifact.as_ref(); - if let Some(tags) = artifact.known_tags() - && let Some(kind) = tags.to_installinator() - && let Some(file_name) = - Utf8Path::new(&artifact.target_name).file_name() - && let Ok(hash) = hash.as_ref().try_into().map(ArtifactHash) - { - document.artifacts.insert(InstallinatorArtifact { - file_name: file_name.to_owned(), - kind, + if let Ok(hash) = hash.as_ref().try_into().map(ArtifactHash) + && let Some(artifact) = crate::util::installinator_artifact( + artifact.tags.clone(), hash, - }); + &artifact.target_name, + ) + { + document.artifacts.insert(artifact); } } let source = BytesSource::json(&document) diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs index 7de166f..39d62c0 100644 --- a/lib/src/repo/v1.rs +++ b/lib/src/repo/v1.rs @@ -13,7 +13,6 @@ use bytes::Buf; use bytes::Bytes; use bytes::BytesMut; use camino::FromPathBufError; -use camino::Utf8Path; use camino::Utf8PathBuf; use flate2::bufread::GzDecoder; use futures_util::Stream; @@ -37,7 +36,6 @@ use tufaceous_artifact::Artifact; use tufaceous_artifact::ArtifactHash; use tufaceous_artifact::ArtifactSet; use tufaceous_artifact::ArtifactVersion; -use tufaceous_artifact::InstallinatorArtifact; use tufaceous_artifact::InstallinatorDocument; use tufaceous_artifact::KnownArtifactTags; use tufaceous_artifact::OsBoard; @@ -59,6 +57,7 @@ use crate::repo::read_target; use crate::repo::read_target_json; use crate::repo::read_target_vec; use crate::repo::sha256_length; +use crate::util::ArtifactExt; #[derive(Debug, Clone)] pub(crate) struct Unpacked { @@ -585,19 +584,8 @@ async fn generate_installinator_document( ) -> Result<(), Error> { let target_name = format!("{original_target}/v2.json"); let mut document = InstallinatorDocument::empty(version.clone()); - for artifact in artifacts.iter() { - if let Some(tags) = artifact.known_tags() - && let Some(kind) = tags.to_installinator() - && let Some(file_name) = - Utf8Path::new(&artifact.target_name).file_name() - { - document.artifacts.insert(InstallinatorArtifact { - file_name: file_name.to_owned(), - kind, - hash: artifact.hash, - }); - } - } + document.artifacts = + artifacts.iter().filter_map(Artifact::to_installinator).collect(); let mut json = serde_json::to_string_pretty(&document) .map_err(ErrorKind::SerializeInstallinator)?; diff --git a/lib/src/util.rs b/lib/src/util.rs index 681abbc..4f140f2 100644 --- a/lib/src/util.rs +++ b/lib/src/util.rs @@ -2,6 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use std::collections::BTreeMap; use std::io::Read; use std::pin::Pin; use std::task::Context; @@ -12,6 +13,10 @@ use camino::Utf8Path; use camino::Utf8PathBuf; use flate2::read::GzDecoder; use futures_util::Stream; +use tufaceous_artifact::Artifact; +use tufaceous_artifact::ArtifactHash; +use tufaceous_artifact::InstallinatorArtifact; +use tufaceous_artifact::KnownArtifactTags; use tufaceous_brand_metadata::LayerInfo; use tufaceous_brand_metadata::Metadata; @@ -19,6 +24,28 @@ use crate::error::Error; use crate::error::ErrorKind; use crate::error::try_path; +pub(crate) trait ArtifactExt { + fn to_installinator(&self) -> Option; +} + +impl ArtifactExt for Artifact { + fn to_installinator(&self) -> Option { + installinator_artifact(self.tags.clone(), self.hash, &self.target_name) + } +} + +pub(crate) fn installinator_artifact( + tags: BTreeMap, + hash: ArtifactHash, + target_name: &str, +) -> Option { + Some(InstallinatorArtifact { + kind: KnownArtifactTags::from_tags(tags).ok()?.to_installinator()?, + hash, + file_name: Utf8Path::new(target_name).file_name()?.to_string(), + }) +} + pub(crate) async fn read_zone_layer_info( reader: R, path: Utf8PathBuf, diff --git a/lib/tests/check.rs b/lib/tests/check.rs new file mode 100644 index 0000000..78b4839 --- /dev/null +++ b/lib/tests/check.rs @@ -0,0 +1,53 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use chrono::Utc; +use semver::Version; +use tufaceous::CheckProblem; +use tufaceous::Repository; +use tufaceous::RepositoryLoader; +use tufaceous::TrustStoreBehavior; +use tufaceous::edit::RepositoryEditor; +use tufaceous::error::Error; +use tufaceous_artifact::KnownArtifactTags; + +const V1: Version = Version::new(1, 0, 0); + +#[tokio::test] +async fn fake_checks_out() -> Result<(), Error> { + let log = slog::Logger::root(slog::Discard, slog::o!()); + let repo = Repository::fake(V1, &log).await?; + let problems = repo.check_problems().await; + assert!(problems.is_empty(), "repo has unexpected problems: {problems:?}"); + Ok(()) +} + +#[tokio::test] +async fn missing_installinator() -> Result<(), Error> { + let log = slog::Logger::root(slog::Discard, slog::o!()); + let zip = RepositoryEditor::fake(V1)? + .generate_installinator_document(false) + .finish() + .await? + .generate_root() + .sign() + .await? + .write_zip(Vec::new(), Utc::now()) + .await?; + let repo = RepositoryLoader::new() + .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .load_zip_buffer(zip, &log) + .await?; + let problems = repo.check_problems().await; + assert!( + matches!( + problems.as_slice(), + [CheckProblem::MissingArtifact( + KnownArtifactTags::InstallinatorDocument + )] + ), + "repo has unexpected problems: {problems:?}" + ); + Ok(()) +} From 9f19c9c9fd7c0c32c3186c6d0a8293ee5d4a0320 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 21 Apr 2026 10:32:35 -0700 Subject: [PATCH 060/137] further comment nits --- artifact/src/sign.rs | 2 +- artifact/src/tags.rs | 6 ++--- clippy.toml | 2 +- lib/src/edit/editor.rs | 46 ++++++++++++++++++++++++++++++++-- lib/src/edit/hubris_archive.rs | 4 +-- lib/src/repo/v1.rs | 4 +-- 6 files changed, 53 insertions(+), 11 deletions(-) diff --git a/artifact/src/sign.rs b/artifact/src/sign.rs index 97d08fd..6982ee2 100644 --- a/artifact/src/sign.rs +++ b/artifact/src/sign.rs @@ -55,7 +55,7 @@ static CA_MAP: LazyLock> = /// /// Used in [`RotTags`] and [`RotBootloaderTags`]. /// -/// This is usually a lowercase hexadecimal string of the ROT Key Table Hash +/// This is usually a lowercase hexadecimal string of the RoT Key Table Hash /// (RKTH) that is used to identify the CA, but this is not enforced by the /// library. /// diff --git a/artifact/src/tags.rs b/artifact/src/tags.rs index 87d75da..c0f0ca3 100644 --- a/artifact/src/tags.rs +++ b/artifact/src/tags.rs @@ -28,7 +28,7 @@ pub enum KnownArtifactTags { /// writing during mupdate and sled recovery. InstallinatorDocument, - /// CORIM manifest for remote attestation. + /// CoRIM manifest for remote attestation. MeasurementCorpus, /// Phase 1 OS image, written to flash. Differs based on the target board. @@ -201,7 +201,7 @@ pub struct RotTags { /// not fully made it through manufacturing yet. #[serde(skip_serializing_if = "RotSign::is_none")] pub rot_sign: RotSign, - /// ROT images are compiled for two different locations in flash; this + /// RoT images are compiled for two different locations in flash; this /// identifies which slot this image belongs to. pub rot_slot: RotSlot, } @@ -212,7 +212,7 @@ impl From for KnownArtifactTags { } } -/// ROT slot artifact tag (A or B). +/// RoT slot artifact tag (A or B). #[derive( Debug, Clone, diff --git a/clippy.toml b/clippy.toml index 8a4e1a8..d8d02c4 100644 --- a/clippy.toml +++ b/clippy.toml @@ -1 +1 @@ -doc-valid-idents = ["OpenAPI", ".."] +doc-valid-idents = ["CoRIM", "OpenAPI", "RoT", ".."] diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index e83f6f2..43db147 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -89,6 +89,11 @@ impl<'a> RepositoryEditor<'a> { Ok(self.insert_input(input)) } + /// Add a fake measurement corpus to the repository. + /// + /// `hashes` specifies the number of SHA256 hashes to list in the CoRIM + /// document. This can be used to create different documents at the same + /// version. pub fn fake_measurement_corpus( self, hashes: usize, @@ -118,6 +123,7 @@ impl<'a> RepositoryEditor<'a> { Ok(self.insert_input(input)) } + /// Add a fake OS image to the repository. pub fn fake_os_image(self, variant: OsVariant) -> Result { let input = Input::fake_os_images( variant, @@ -127,15 +133,22 @@ impl<'a> RepositoryEditor<'a> { Ok(self.insert_input(input)) } + /// Add a Root of Trust Hubris archive to the repository. + /// + /// Tags are automatically determined based on the image caboose, except + /// for `slot` which must be specified. pub async fn rot_archive( self, - slot: RotSlot, + rot_slot: RotSlot, path: Utf8PathBuf, ) -> Result { let source = FileSource::open(path).await?; - Ok(self.insert_input(Input::rot_archive(source, None, slot).await?)) + Ok(self.insert_input(Input::rot_archive(source, None, rot_slot).await?)) } + /// Add a Root of Trust Bootloader Hubris archive to the repository. + /// + /// Tags are automatically determined based on the image caboose. pub async fn rot_bootloader_archive( self, path: Utf8PathBuf, @@ -145,11 +158,17 @@ impl<'a> RepositoryEditor<'a> { .insert_input(Input::rot_bootloader_archive(source, None).await?)) } + /// Add a Service Processor Hubris archive to the repository. + /// + /// Tags are automatically determined based on the image caboose. pub async fn sp_archive(self, path: Utf8PathBuf) -> Result { let source = FileSource::open(path).await?; Ok(self.insert_input(Input::sp_archive(source, None).await?)) } + /// Add a fake Root of Trust Hubris archive to the repository. + /// + /// This will generate a fake Hubris archive with the appropriate tags. pub fn fake_rot_archive(self, tags: RotTags) -> Result { let input = Input::fake_rot_archive( tags, @@ -159,6 +178,9 @@ impl<'a> RepositoryEditor<'a> { Ok(self.insert_input(input)) } + /// Add a fake Root of Trust Bootloader Hubris archive to the repository. + /// + /// This will generate a fake Hubris archive with the appropriate tags. pub fn fake_rot_bootloader_archive( self, tags: RotBootloaderTags, @@ -171,16 +193,27 @@ impl<'a> RepositoryEditor<'a> { Ok(self.insert_input(input)) } + /// Add a fake Service Processor Hubris archive to the repository. + /// + /// This will generate a fake Hubris archive with the appropriate tags. pub fn fake_sp_archive(self, tags: SpTags) -> Result { let input = Input::fake_sp_archive(tags, self.artifact_version.clone()?, None)?; Ok(self.insert_input(input)) } + /// Add a zone image to the repository. + /// + /// The `zone_name` tag is automatically determined based on the layer + /// metadata (the `oxide.json` file that starts zone tarballs). pub async fn zone_image(self, path: Utf8PathBuf) -> Result { Ok(self.insert_input(Input::zone_image(path).await?)) } + /// Add a fake zone image to the repository. + /// + /// This will generate a tarball containing a matching `oxide.json` layer + /// metadata file. pub fn fake_zone_image( self, zone_name: String, @@ -245,6 +278,7 @@ impl<'a> RepositoryEditor<'a> { Ok(self) } + /// Remove a target with the given target name. pub fn remove_target(mut self, target_name: &str) -> Self { self.targets.remove(target_name); self.artifacts.remove(target_name); @@ -299,15 +333,22 @@ impl<'a> RepositoryEditor<'a> { Ok(editor) } + /// Set the repository-level metadata. pub fn metadata(mut self, metadata: &Metadata) -> Self { self.metadata = metadata.to_map(); self } + /// Create a `RepositoryEditor` from a loaded repository. + /// + /// This creates an editor with references to all of the artifacts and + /// targets in the original repository. pub fn from_repo(repo: &'a Repository) -> Result { Self::new(repo.system_version().clone()).import_repo(repo) } + /// Import all of the artifacts and targets from a repository into this + /// editor. pub fn import_repo(mut self, repo: &'a Repository) -> Result { if repo.is_v1() { return Err(ErrorKind::ImportV1Repo.into()); @@ -341,6 +382,7 @@ impl<'a> RepositoryEditor<'a> { Ok(self) } + /// Finalize the artifacts and targets, returning an [`UnsignedRepository`]. pub async fn finish(self) -> Result, Error> { // Un-nest `self.artifacts`, returning an error if we have multiple // artifact definitions for a single target name. diff --git a/lib/src/edit/hubris_archive.rs b/lib/src/edit/hubris_archive.rs index d340eeb..512ef10 100644 --- a/lib/src/edit/hubris_archive.rs +++ b/lib/src/edit/hubris_archive.rs @@ -118,8 +118,8 @@ impl Input> { return Ok(ControlFlow::Continue(input)); }; // HACK: We are reading the `image-name` file in the archive, which - // appears to be "a" or "b" if it's an ROT image, "default" if it's - // an SP image, and nonexistent if it's an ROT bootloader image. This + // appears to be "a" or "b" if it's an RoT image, "default" if it's + // an SP image, and nonexistent if it's an RoT bootloader image. This // seems fragile. Ideally this can be in the caboose someday (see // sprot-release#74). match archive.image_name().as_deref() { diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs index 39d62c0..bc3acfc 100644 --- a/lib/src/repo/v1.rs +++ b/lib/src/repo/v1.rs @@ -142,7 +142,7 @@ pub(crate) async fn from_loaded( { info!( log, - "skipping duplicate ROT bootloader image"; + "skipping duplicate RoT bootloader image"; "existing" => &artifact.target_name, "skipped" => &target, ); @@ -437,7 +437,7 @@ impl CompositeArtifact { { info!( log, - "skipping duplicate ROT image"; + "skipping duplicate RoT image"; "existing" => &artifact.target_name, "skipped" => &target_name, ); From 152463b349f6b14d74d9c4c54641888b3667001d Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 28 Apr 2026 10:29:08 -0700 Subject: [PATCH 061/137] rename "sign" -> "RKTH" --- artifact/src/hubris.rs | 6 +++--- artifact/src/lib.rs | 4 ++-- artifact/src/{sign.rs => rkth.rs} | 19 +++++++++---------- artifact/src/tags.rs | 25 ++++++++++++------------- lib/src/edit/fake.rs | 6 +++--- lib/src/edit/hubris_archive.rs | 10 +++++----- lib/src/edit/input.rs | 4 ++-- lib/tests/v1_compatibility.rs | 8 ++++---- 8 files changed, 40 insertions(+), 42 deletions(-) rename artifact/src/{sign.rs => rkth.rs} (89%) diff --git a/artifact/src/hubris.rs b/artifact/src/hubris.rs index 4c3f041..aa549ab 100644 --- a/artifact/src/hubris.rs +++ b/artifact/src/hubris.rs @@ -8,8 +8,8 @@ use hubtools::Caboose; use hubtools::CabooseError; use crate::KnownArtifactTags; +use crate::RotKeyTableHash; use crate::RotBootloaderTags; -use crate::RotSign; use crate::RotSlot; use crate::RotTags; use crate::SpTags; @@ -29,7 +29,7 @@ impl RotTags { ) -> Result { Ok(Self { rot_board: read_board(caboose)?, - rot_sign: RotSign(read_sign(caboose)?), + rot_rkth: RotKeyTableHash(read_sign(caboose)?), rot_slot: slot, }) } @@ -46,7 +46,7 @@ impl RotBootloaderTags { pub fn from_caboose(caboose: &Caboose) -> Result { Ok(Self { rot_board: read_board(caboose)?, - rot_sign: RotSign(read_sign(caboose)?), + rot_rkth: RotKeyTableHash(read_sign(caboose)?), }) } } diff --git a/artifact/src/lib.rs b/artifact/src/lib.rs index 9df2069..b9081e5 100644 --- a/artifact/src/lib.rs +++ b/artifact/src/lib.rs @@ -12,8 +12,8 @@ mod hubris; mod installinator; mod map; mod metadata; +mod rkth; mod set; -mod sign; mod tags; mod version; @@ -22,8 +22,8 @@ pub use artifact::*; pub use hubris::*; pub use installinator::*; pub use metadata::*; +pub use rkth::*; pub use set::ArtifactSet; -pub use sign::*; pub use tags::*; pub use version::*; diff --git a/artifact/src/sign.rs b/artifact/src/rkth.rs similarity index 89% rename from artifact/src/sign.rs rename to artifact/src/rkth.rs index 6982ee2..e35370d 100644 --- a/artifact/src/sign.rs +++ b/artifact/src/rkth.rs @@ -51,13 +51,12 @@ const CA_LIST: [(&str, &str); 9] = [ static CA_MAP: LazyLock> = LazyLock::new(|| HashMap::from(CA_LIST)); -/// Value of the `SIGN` field of a Hubris caboose. +/// The RoT Key Table Hash, which identifies a CA that signed a Hubris image. /// /// Used in [`RotTags`] and [`RotBootloaderTags`]. /// -/// This is usually a lowercase hexadecimal string of the RoT Key Table Hash -/// (RKTH) that is used to identify the CA, but this is not enforced by the -/// library. +/// This is usually a lowercase hexadecimal string, but this is not enforced by +/// the library. /// /// [`RotTags`]: crate::RotTags /// [`RotBootloaderTags`]: crate::RotBootloaderTags @@ -66,9 +65,9 @@ static CA_MAP: LazyLock> = )] #[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] #[serde(transparent)] -pub struct RotSign(pub Option); +pub struct RotKeyTableHash(pub Option); -impl RotSign { +impl RotKeyTableHash { /// Returns a friendly name for the CA this RKTH represents, if one is /// known. Returns `None` otherwise. pub fn friendly_ca_name(&self) -> Option<&'static str> { @@ -81,13 +80,13 @@ impl RotSign { } } -impl Debug for RotSign { +impl Debug for RotKeyTableHash { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { Debug::fmt(&self.0, f) } } -impl Display for RotSign { +impl Display for RotKeyTableHash { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self.0.as_deref() { Some(inner) => { @@ -103,8 +102,8 @@ impl Display for RotSign { mod tests { use std::collections::HashSet; - use crate::sign::CA_LIST; - use crate::sign::CA_MAP; + use crate::rkth::CA_LIST; + use crate::rkth::CA_MAP; #[test] fn ensure_consistency() { diff --git a/artifact/src/tags.rs b/artifact/src/tags.rs index c0f0ca3..391519f 100644 --- a/artifact/src/tags.rs +++ b/artifact/src/tags.rs @@ -9,7 +9,7 @@ use std::fmt::Display; use serde::Deserialize; use serde::Serialize; -use crate::RotSign; +use crate::RotKeyTableHash; use crate::installinator::InstallinatorArtifactKind; /// Sets of artifact tags known to the control plane. @@ -193,14 +193,13 @@ display_serialize!(OsBoard); pub struct RotTags { /// The `BORD` field in the caboose (such as `oxide-rot-1`). pub rot_board: String, - /// The `SIGN` field in the caboose. This is the Root Key Table Hash - /// (RKTH). + /// The RoT Key Table Hash (RKTH). This is the `SIGN` field in the caboose. /// /// For unsigned images this will not be present; this will generally /// never occur in release repos but can be useful on hardware that has /// not fully made it through manufacturing yet. - #[serde(skip_serializing_if = "RotSign::is_none")] - pub rot_sign: RotSign, + #[serde(skip_serializing_if = "RotKeyTableHash::is_none")] + pub rot_rkth: RotKeyTableHash, /// RoT images are compiled for two different locations in flash; this /// identifies which slot this image belongs to. pub rot_slot: RotSlot, @@ -249,8 +248,8 @@ pub struct RotBootloaderTags { /// For unsigned images this will not be present; this will generally /// never occur in release repos but can be useful on hardware that has /// not fully made it through manufacturing yet. - #[serde(skip_serializing_if = "RotSign::is_none")] - pub rot_sign: RotSign, + #[serde(skip_serializing_if = "RotKeyTableHash::is_none")] + pub rot_rkth: RotKeyTableHash, } impl From for KnownArtifactTags { @@ -346,7 +345,7 @@ mod tests { use test_strategy::proptest; use crate::KnownArtifactTags; - use crate::RotSign; + use crate::RotKeyTableHash; use crate::RotSlot; use crate::RotTags; @@ -357,27 +356,27 @@ mod tests { } #[test] - fn rot_sign() { + fn rot_rkth() { let mut tags = BTreeMap::from([ ("kind".to_owned(), "rot".to_owned()), ("rot_board".to_owned(), "oxide-rot-1".to_owned()), - // rot_sign not included + // rot_rkth not included ("rot_slot".to_owned(), "a".to_owned()), ]); assert_eq!( KnownArtifactTags::from_tags(tags.clone()).unwrap(), KnownArtifactTags::Rot(RotTags { rot_board: "oxide-rot-1".to_owned(), - rot_sign: RotSign(None), + rot_rkth: RotKeyTableHash(None), rot_slot: RotSlot::A }) ); - tags.insert("rot_sign".to_owned(), "meow".to_owned()); + tags.insert("rot_rkth".to_owned(), "meow".to_owned()); assert_eq!( KnownArtifactTags::from_tags(tags).unwrap(), KnownArtifactTags::Rot(RotTags { rot_board: "oxide-rot-1".to_owned(), - rot_sign: RotSign(Some("meow".to_owned())), + rot_rkth: RotKeyTableHash(Some("meow".to_owned())), rot_slot: RotSlot::A }) ); diff --git a/lib/src/edit/fake.rs b/lib/src/edit/fake.rs index bc4e591..1ed0a3c 100644 --- a/lib/src/edit/fake.rs +++ b/lib/src/edit/fake.rs @@ -5,8 +5,8 @@ use tufaceous_artifact::ArtifactSet; use tufaceous_artifact::ArtifactVersion; use tufaceous_artifact::OsVariant; +use tufaceous_artifact::RotKeyTableHash; use tufaceous_artifact::RotBootloaderTags; -use tufaceous_artifact::RotSign; use tufaceous_artifact::RotSlot; use tufaceous_artifact::RotTags; use tufaceous_artifact::SpTags; @@ -90,7 +90,7 @@ impl Input { inputs.push(Self::fake_rot_archive( RotTags { rot_board: "SimRot".into(), - rot_sign: RotSign(Some(FAKE_SIGN.into())), + rot_rkth: RotKeyTableHash(Some(FAKE_SIGN.into())), rot_slot: slot, }, version.clone(), @@ -100,7 +100,7 @@ impl Input { inputs.push(Self::fake_rot_bootloader_archive( RotBootloaderTags { rot_board: "SimRot".into(), - rot_sign: RotSign(Some(FAKE_SIGN.into())), + rot_rkth: RotKeyTableHash(Some(FAKE_SIGN.into())), }, version.clone(), interior_version, diff --git a/lib/src/edit/hubris_archive.rs b/lib/src/edit/hubris_archive.rs index 512ef10..8ed6aa9 100644 --- a/lib/src/edit/hubris_archive.rs +++ b/lib/src/edit/hubris_archive.rs @@ -10,7 +10,7 @@ use hubtools::HubrisArchiveBuilder; use tufaceous_artifact::ArtifactVersion; use tufaceous_artifact::ReadCabooseError; use tufaceous_artifact::RotBootloaderTags; -use tufaceous_artifact::RotSign; +use tufaceous_artifact::RotKeyTableHash; use tufaceous_artifact::RotSlot; use tufaceous_artifact::RotTags; use tufaceous_artifact::SpTags; @@ -149,7 +149,7 @@ impl Input { ) -> Result { let data = CabooseData { board: &tags.rot_board, - sign: &tags.rot_sign, + sign: &tags.rot_rkth, commit: "this-is-a-fake-rot", version: interior_version.unwrap_or(&version), }; @@ -164,7 +164,7 @@ impl Input { ) -> Result { let data = CabooseData { board: &tags.rot_board, - sign: &tags.rot_sign, + sign: &tags.rot_rkth, commit: "this-is-a-fake-rot-bootloader", version: interior_version.unwrap_or(&version), }; @@ -179,7 +179,7 @@ impl Input { ) -> Result { let data = CabooseData { board: &tags.sp_board, - sign: &RotSign(None), + sign: &RotKeyTableHash(None), commit: "this-is-a-fake-sp", version: interior_version.unwrap_or(&version), }; @@ -190,7 +190,7 @@ impl Input { struct CabooseData<'a> { board: &'a str, - sign: &'a RotSign, + sign: &'a RotKeyTableHash, commit: &'static str, version: &'a ArtifactVersion, } diff --git a/lib/src/edit/input.rs b/lib/src/edit/input.rs index 8802e22..4f5acd8 100644 --- a/lib/src/edit/input.rs +++ b/lib/src/edit/input.rs @@ -119,7 +119,7 @@ impl Input { let target_name = format!( "rot/{board}-{sign}-{version}-slot-{slot}.zip", board = tags.rot_board, - sign = tags.rot_sign, + sign = tags.rot_rkth, slot = tags.rot_slot ); vec![Output::new(target_name, version, tags.into(), source)] @@ -128,7 +128,7 @@ impl Input { let target_name = format!( "rot-bootloader/{board}-{sign}-{version}.zip", board = tags.rot_board, - sign = tags.rot_sign + sign = tags.rot_rkth ); vec![Output::new(target_name, version, tags.into(), source)] } diff --git a/lib/tests/v1_compatibility.rs b/lib/tests/v1_compatibility.rs index 67a9182..96c4d55 100644 --- a/lib/tests/v1_compatibility.rs +++ b/lib/tests/v1_compatibility.rs @@ -14,7 +14,7 @@ use tufaceous_artifact::OsPhase1Tags; use tufaceous_artifact::OsPhase2Tags; use tufaceous_artifact::OsVariant; use tufaceous_artifact::RotBootloaderTags; -use tufaceous_artifact::RotSign; +use tufaceous_artifact::RotKeyTableHash; use tufaceous_artifact::RotSlot; use tufaceous_artifact::RotTags; use tufaceous_artifact::SpTags; @@ -46,12 +46,12 @@ async fn v1_fake() -> Result<(), Error> { } expected.push(OsPhase2Tags { os_variant }.into()); } - for rot_sign in ["sign-gimlet", "sign-psc", "sign-switch"] { + for rot_rkth in ["sign-gimlet", "sign-psc", "sign-switch"] { for rot_slot in [RotSlot::A, RotSlot::B] { expected.push( RotTags { rot_board: "SimRot".into(), - rot_sign: RotSign(Some(rot_sign.into())), + rot_rkth: RotKeyTableHash(Some(rot_rkth.into())), rot_slot, } .into(), @@ -60,7 +60,7 @@ async fn v1_fake() -> Result<(), Error> { expected.push( RotBootloaderTags { rot_board: "SimRot".into(), - rot_sign: RotSign(Some(rot_sign.into())), + rot_rkth: RotKeyTableHash(Some(rot_rkth.into())), } .into(), ); From 72f9778983754a0997ab3d568524a0237db4a758 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 28 Apr 2026 10:29:57 -0700 Subject: [PATCH 062/137] tough 0.22.0 --- Cargo.lock | 8 +++++--- Cargo.toml | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index aca43d5..3a98516 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1560,7 +1560,8 @@ dependencies = [ [[package]] name = "olpc-cjson" version = "0.1.4" -source = "git+https://github.com/awslabs/tough.git?branch=develop#c9a9865dc2d64a65c9930ad5daf4930e556662c5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "696183c9b5fe81a7715d074fd632e8bd46f4ccc0231a3ed7fc580a80de5f7083" dependencies = [ "serde", "serde_json", @@ -2795,8 +2796,9 @@ dependencies = [ [[package]] name = "tough" -version = "0.21.0" -source = "git+https://github.com/awslabs/tough.git?branch=develop#c9a9865dc2d64a65c9930ad5daf4930e556662c5" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8031cff0872dd1c6312370515a6be8098f6ea5512f1bad725016046fc725f272" dependencies = [ "async-recursion", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 797df1e..9a9311c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,7 +50,7 @@ test-strategy = "0.4.0" thiserror = "2.0.12" tokio = "1.47.0" tokio-test = "0.4.5" -tough = { git = "https://github.com/awslabs/tough.git", branch = "develop" } +tough = "0.22.0" tufaceous.path = "lib" tufaceous-artifact.path = "artifact" tufaceous-brand-metadata.path = "brand-metadata" From 2610c9e11fc009892bba0162db8aab02e7edc1f6 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 28 Apr 2026 10:42:45 -0700 Subject: [PATCH 063/137] os_board should not be an enum --- artifact/src/tags.rs | 34 ++++++++++++++++------------------ lib/src/check.rs | 2 +- lib/src/edit/input.rs | 4 ++-- lib/src/repo/v1.rs | 4 ++-- lib/tests/v1_compatibility.rs | 2 +- 5 files changed, 22 insertions(+), 24 deletions(-) diff --git a/artifact/src/tags.rs b/artifact/src/tags.rs index 391519f..43ced7e 100644 --- a/artifact/src/tags.rs +++ b/artifact/src/tags.rs @@ -162,28 +162,26 @@ pub enum OsVariant { } display_serialize!(OsVariant); -/// OS board artifact tag (gimlet or cosmo). +/// OS board artifact tag (gimlet, cosmo, etc). #[derive( - Debug, - Clone, - Copy, - PartialEq, - Eq, - PartialOrd, - Ord, - Hash, - Deserialize, - Serialize, + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, )] #[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] -#[serde(rename_all = "snake_case")] -pub enum OsBoard { - /// First-generation SP3 compute board. - Gimlet, - /// Second-generation SP5 compute board. - Cosmo, +#[serde(transparent)] +pub struct OsBoard(pub Cow<'static, str>); + +impl OsBoard { + /// First-generation SP3 compute sled. + pub const GIMLET: Self = Self(Cow::Borrowed("gimlet")); + /// Second-generation SP5 compute sled. + pub const COSMO: Self = Self(Cow::Borrowed("cosmo")); +} + +impl Display for OsBoard { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + Display::fmt(&self.0, f) + } } -display_serialize!(OsBoard); /// The inner value of [`KnownArtifactTags::Rot`]. #[derive( diff --git a/lib/src/check.rs b/lib/src/check.rs index e8a3824..1848dbc 100644 --- a/lib/src/check.rs +++ b/lib/src/check.rs @@ -93,7 +93,7 @@ impl Repository { // to work. let mut expected = vec![KnownArtifactTags::InstallinatorDocument]; for os_variant in [OsVariant::Host, OsVariant::Recovery] { - for os_board in [OsBoard::Cosmo, OsBoard::Gimlet] { + for os_board in [OsBoard::COSMO, OsBoard::GIMLET] { expected.push(KnownArtifactTags::OsPhase1(OsPhase1Tags { os_board, os_variant, diff --git a/lib/src/edit/input.rs b/lib/src/edit/input.rs index 4f5acd8..4aac1f2 100644 --- a/lib/src/edit/input.rs +++ b/lib/src/edit/input.rs @@ -90,14 +90,14 @@ impl Input { vec.push(Output::new( base.join(COSMO_PHASE_1_PATH).into(), version.clone(), - OsPhase1Tags { os_board: OsBoard::Cosmo, os_variant } + OsPhase1Tags { os_board: OsBoard::COSMO, os_variant } .into(), cosmo_phase_1, )); vec.push(Output::new( base.join(GIMLET_PHASE_1_PATH).into(), version.clone(), - OsPhase1Tags { os_board: OsBoard::Gimlet, os_variant } + OsPhase1Tags { os_board: OsBoard::GIMLET, os_variant } .into(), gimlet_phase_1, )); diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs index bc3acfc..731c092 100644 --- a/lib/src/repo/v1.rs +++ b/lib/src/repo/v1.rs @@ -474,14 +474,14 @@ impl CompositeArtifact { COSMO_PHASE_1_PATH, KnownArtifactTags::OsPhase1(OsPhase1Tags { os_variant, - os_board: OsBoard::Cosmo, + os_board: OsBoard::COSMO, }), ), ( GIMLET_PHASE_1_PATH, KnownArtifactTags::OsPhase1(OsPhase1Tags { os_variant, - os_board: OsBoard::Gimlet, + os_board: OsBoard::GIMLET, }), ), ( diff --git a/lib/tests/v1_compatibility.rs b/lib/tests/v1_compatibility.rs index 96c4d55..3018401 100644 --- a/lib/tests/v1_compatibility.rs +++ b/lib/tests/v1_compatibility.rs @@ -41,7 +41,7 @@ async fn v1_fake() -> Result<(), Error> { KnownArtifactTags::MeasurementCorpus, ]; for os_variant in [OsVariant::Host, OsVariant::Recovery] { - for os_board in [OsBoard::Cosmo, OsBoard::Gimlet] { + for os_board in [OsBoard::COSMO, OsBoard::GIMLET] { expected.push(OsPhase1Tags { os_board, os_variant }.into()); } expected.push(OsPhase2Tags { os_variant }.into()); From b0fce4fdf170ef79823a51755e3b5a30004b480f Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 28 Apr 2026 10:55:07 -0700 Subject: [PATCH 064/137] add RotKeyTableHash::from_bytes --- artifact/src/rkth.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/artifact/src/rkth.rs b/artifact/src/rkth.rs index e35370d..64d1eef 100644 --- a/artifact/src/rkth.rs +++ b/artifact/src/rkth.rs @@ -68,6 +68,11 @@ static CA_MAP: LazyLock> = pub struct RotKeyTableHash(pub Option); impl RotKeyTableHash { + /// Create a `RotKeyTableHash` from an array of 32 bytes. + pub fn from_bytes(hash: [u8; 32]) -> Self { + Self(Some(hex::encode(hash))) + } + /// Returns a friendly name for the CA this RKTH represents, if one is /// known. Returns `None` otherwise. pub fn friendly_ca_name(&self) -> Option<&'static str> { From f3ae21d0a48b9fc4cc1103f709048105ebf510d5 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 28 Apr 2026 11:15:06 -0700 Subject: [PATCH 065/137] helix pls --- artifact/src/hubris.rs | 2 +- lib/src/edit/fake.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/artifact/src/hubris.rs b/artifact/src/hubris.rs index aa549ab..d49cacb 100644 --- a/artifact/src/hubris.rs +++ b/artifact/src/hubris.rs @@ -8,8 +8,8 @@ use hubtools::Caboose; use hubtools::CabooseError; use crate::KnownArtifactTags; -use crate::RotKeyTableHash; use crate::RotBootloaderTags; +use crate::RotKeyTableHash; use crate::RotSlot; use crate::RotTags; use crate::SpTags; diff --git a/lib/src/edit/fake.rs b/lib/src/edit/fake.rs index 1ed0a3c..4e63981 100644 --- a/lib/src/edit/fake.rs +++ b/lib/src/edit/fake.rs @@ -5,8 +5,8 @@ use tufaceous_artifact::ArtifactSet; use tufaceous_artifact::ArtifactVersion; use tufaceous_artifact::OsVariant; -use tufaceous_artifact::RotKeyTableHash; use tufaceous_artifact::RotBootloaderTags; +use tufaceous_artifact::RotKeyTableHash; use tufaceous_artifact::RotSlot; use tufaceous_artifact::RotTags; use tufaceous_artifact::SpTags; From 268680f3c6bb89a15594bbdb6d602b9aaa8d8c08 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 28 Apr 2026 11:35:45 -0700 Subject: [PATCH 066/137] zip transport error handling nits --- lib/src/error.rs | 8 +++-- lib/src/zip_transport.rs | 70 +++++++++++++++++++--------------------- 2 files changed, 40 insertions(+), 38 deletions(-) diff --git a/lib/src/error.rs b/lib/src/error.rs index 0a90fe8..a725da2 100644 --- a/lib/src/error.rs +++ b/lib/src/error.rs @@ -14,21 +14,25 @@ use std::ops::Range; use crate::ZipTransportError; use camino::Utf8PathBuf; use tough::TransportErrorKind; +use tufaceous_artifact::DisplayTags; macro_rules! try_path { ($result:expr, $kind:ident, $path:expr) => { + $crate::error::try_path!($result, $kind, path: $path) + }; + + ($result:expr, $kind:ident, $field:ident: $path:expr) => { match $result { Ok(value) => value, Err(source) => { return Err( - ErrorKind::$kind { source, path: $path.into() }.into() + ErrorKind::$kind { source, $field: $path.into() }.into() ) } } }; } pub(crate) use try_path; -use tufaceous_artifact::DisplayTags; #[derive(Debug, thiserror::Error)] #[error(transparent)] diff --git a/lib/src/zip_transport.rs b/lib/src/zip_transport.rs index 17368bb..3ac70b0 100644 --- a/lib/src/zip_transport.rs +++ b/lib/src/zip_transport.rs @@ -88,6 +88,12 @@ use url::Url; use crate::error::Error; use crate::error::ErrorKind; +macro_rules! try_archive_path { + ($result:expr, $kind:ident, $path:expr) => { + $crate::error::try_path!($result, $kind, archive_path: $path) + }; +} + // The length of the end-of-central-directory record if the comment is zero // bytes. Archives with a "comment" (data after the EOCD) are rejected. const EOCD_MAX_SEARCH_SPACE: u64 = 22; @@ -141,8 +147,10 @@ impl + Debug + Send + Sync + 'static> ZipTransport> { pub fn from_slice(data: T, log: &Logger) -> Result { let archive = ZipArchive::with_max_search_space(EOCD_MAX_SEARCH_SPACE) .locate_in_slice(data) - .map_err(|(_, error)| error) - .upgrade(None)?; + .map_err(|(_, source)| ErrorKind::ReadZip { + source, + archive_path: None, + })?; Self::from_impl_blocking(archive.into_zip_archive(), None, None, log) } } @@ -157,11 +165,13 @@ impl ZipTransport { tokio::task::spawn_blocking(move || { let archive_path = archive_path; let mut buffer = vec![0; rawzip::RECOMMENDED_BUFFER_SIZE]; - let archive = + let archive = try_archive_path!( ZipArchive::with_max_search_space(EOCD_MAX_SEARCH_SPACE) .locate_in_file(file, &mut buffer) - .map_err(|(_, error)| error) - .upgrade(archive_path.as_ref())?; + .map_err(|(_, error)| error), + ReadZip, + archive_path + ); Self::from_impl_blocking(archive, archive_path, Some(buffer), &log) }) .await? @@ -183,7 +193,7 @@ impl ZipTransport { let mut entries = HashMap::new(); let mut records = archive.entries(&mut buffer); while let Some(record) = - records.next_entry().upgrade(archive_path.as_ref())? + try_archive_path!(records.next_entry(), ReadZip, archive_path) { actual += 1; all_entries.push(( @@ -225,11 +235,16 @@ impl ZipTransport { let mut ranges = Vec::new(); for (wayfinder, raw_path) in all_entries { - let entry = - archive.get_entry(wayfinder).upgrade(archive_path.as_ref())?; - let header = entry - .local_header(&mut buffer) - .upgrade(archive_path.as_ref())?; + let entry = try_archive_path!( + archive.get_entry(wayfinder), + ReadZip, + archive_path + ); + let header = try_archive_path!( + entry.local_header(&mut buffer), + ReadZip, + archive_path + ); if raw_path != header.file_path().as_bytes() { return Err(ErrorKind::ZipPathMismatch { central: raw_path, @@ -337,15 +352,14 @@ impl ZipTransport { .await { Ok(Ok(bytes)) => Ok(Some((bytes, (outer_tx, task, url)))), - Ok(Err(error)) => Err(error.upgrade(url)), + Ok(Err(error)) => Err(error.into_tough_error(url)), Err(()) => { // The task hung up. If we're still here, then the task // reached EOF. await the task and return. match task.await { Ok(()) => Ok(None), - Err(error) => { - Err(ZipTransportError::from(error).upgrade(url)) - } + Err(error) => Err(ZipTransportError::from(error) + .into_tough_error(url)), } } } @@ -370,31 +384,15 @@ impl Transport Ok(Box::pin(self.clone().stream(entry_data, url))) } Some(Entry::Dir) => { - Err(ZipTransportError::IsADirectory.upgrade(url)) + Err(ZipTransportError::IsADirectory.into_tough_error(url)) } Some(Entry::Symlink) => { - Err(ZipTransportError::IsASymlink.upgrade(url)) + Err(ZipTransportError::IsASymlink.into_tough_error(url)) } Some(Entry::Duplicate) => { - Err(ZipTransportError::Duplicate.upgrade(url)) + Err(ZipTransportError::Duplicate.into_tough_error(url)) } - None => Err(ZipTransportError::FileNotFound.upgrade(url)), - } - } -} - -trait ResultExt { - fn upgrade(self, archive_path: Option<&Utf8PathBuf>) -> Result; -} - -impl ResultExt for Result { - fn upgrade(self, archive_path: Option<&Utf8PathBuf>) -> Result { - match self { - Ok(v) => Ok(v), - Err(source) => Err(ErrorKind::ReadZip { - source, - archive_path: archive_path.cloned(), - })?, + None => Err(ZipTransportError::FileNotFound.into_tough_error(url)), } } } @@ -425,7 +423,7 @@ pub enum ZipTransportError { } impl ZipTransportError { - fn upgrade(self, url: Url) -> TransportError { + fn into_tough_error(self, url: Url) -> TransportError { let kind = match &self { ZipTransportError::FileNotFound => TransportErrorKind::FileNotFound, From a5c876575198b0e712a7f1c06a77de360728ce28 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 28 Apr 2026 14:00:39 -0700 Subject: [PATCH 067/137] rust 1.94; improve readability of ZipTransport::from_impl_blocking --- artifact/Cargo.toml | 1 + artifact/src/rkth.rs | 5 +- artifact/src/version.rs | 5 -- lib/Cargo.toml | 1 + lib/src/repo/v1.rs | 2 +- lib/src/zip_transport.rs | 182 ++++++++++++++++++++++----------------- rust-toolchain.toml | 2 +- 7 files changed, 108 insertions(+), 90 deletions(-) diff --git a/artifact/Cargo.toml b/artifact/Cargo.toml index d047b8d..0c9dc61 100644 --- a/artifact/Cargo.toml +++ b/artifact/Cargo.toml @@ -4,6 +4,7 @@ version.workspace = true edition.workspace = true license.workspace = true publish.workspace = true +rust-version = "1.94" [features] hubtools = ["dep:hubtools"] diff --git a/artifact/src/rkth.rs b/artifact/src/rkth.rs index 64d1eef..5e39168 100644 --- a/artifact/src/rkth.rs +++ b/artifact/src/rkth.rs @@ -118,10 +118,7 @@ mod tests { let values = CA_MAP.values().collect::>(); assert_eq!(values.len(), CA_LIST.len()); // CA_LIST is ordered by friendly name - for window in CA_LIST.windows(2) { - let [(_, v1), (_, v2)] = window else { - panic!("slice::windows is broken") - }; + for [(_, v1), (_, v2)] in CA_LIST.array_windows() { assert!(v1 < v2, "{v1} incorrectly placed before {v2} in CA_LIST"); } } diff --git a/artifact/src/version.rs b/artifact/src/version.rs index ae19701..4bb71d0 100644 --- a/artifact/src/version.rs +++ b/artifact/src/version.rs @@ -2,11 +2,6 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -#![cfg_attr( - any(test, feature = "proptest"), - expect(clippy::explicit_deref_methods) -)] - use std::borrow::Cow; use std::fmt; use std::str::FromStr; diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 3ed8a93..7f7e2df 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -4,6 +4,7 @@ version.workspace = true edition.workspace = true license.workspace = true publish.workspace = true +rust-version = "1.94" [dependencies] atomicwrites.workspace = true diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs index 731c092..6b7a186 100644 --- a/lib/src/repo/v1.rs +++ b/lib/src/repo/v1.rs @@ -420,7 +420,7 @@ impl CompositeArtifact { let capacity = usize::try_from(length).unwrap_or_default(); let mut vec = Vec::with_capacity(capacity); reader.read_to_end(&mut vec).map(|_| vec).map_err(|source| { - ErrorKind::ReadFile { source, path: None } + Error::from(ErrorKind::ReadFile { source, path: None }) }) }) .await??; diff --git a/lib/src/zip_transport.rs b/lib/src/zip_transport.rs index 3ac70b0..c99ad4a 100644 --- a/lib/src/zip_transport.rs +++ b/lib/src/zip_transport.rs @@ -59,6 +59,7 @@ use std::fs::File; use std::io::Cursor; use std::io::Read; use std::sync::Arc; +use std::sync::LazyLock; use bytes::Bytes; use bytes::BytesMut; @@ -72,8 +73,7 @@ use rawzip::FileReader; use rawzip::ReaderAt; use rawzip::ZipArchive; use rawzip::ZipArchiveEntryWayfinder; -use rawzip::path::RawPath; -use rawzip::path::ZipFilePath; +use rawzip::ZipFileHeaderRecord; use slog::Logger; use slog::warn; use tokio::sync::mpsc; @@ -122,20 +122,6 @@ struct Inner { entries: HashMap, } -#[derive(Debug, Clone, Copy)] -enum Entry { - File(EntryData), - Dir, - Symlink, - Duplicate, -} - -#[derive(Debug, Clone, Copy)] -struct EntryData { - compression_method: CompressionMethod, - wayfinder: ZipArchiveEntryWayfinder, -} - // Manually implemented, as the derive macro adds an unnecessary `T: Clone`. impl Clone for ZipTransport { fn clone(&self) -> Self { @@ -187,46 +173,33 @@ impl ZipTransport { ) -> Result { let mut buffer = buffer.unwrap_or_else(|| vec![0; rawzip::RECOMMENDED_BUFFER_SIZE]); - let expected = archive.entries_hint(); - let mut actual: u64 = 0; - let mut all_entries = Vec::new(); + let mut entries = HashMap::new(); + // Keep a list of all central directory entries (regardless of whether + // the path is valid) to check for an unexpected number of entries, or + // overlapping file ranges. + let mut all_entries = Vec::new(); + let mut records = archive.entries(&mut buffer); while let Some(record) = try_archive_path!(records.next_entry(), ReadZip, archive_path) { - actual += 1; - all_entries.push(( - record.wayfinder(), - record.file_path().as_bytes().to_vec(), - )); - - let Some(url) = path_to_url(record.file_path(), log) else { - continue; - }; - match entries.entry(url) { - std::collections::hash_map::Entry::Occupied(mut e) => { - e.insert(Entry::Duplicate); - } - std::collections::hash_map::Entry::Vacant(e) => { - if record.is_dir() { - e.insert(Entry::Dir); - } else if record.mode().is_symlink() { - e.insert(Entry::Symlink); - } else { - let data = EntryData { - compression_method: record.compression_method(), - wayfinder: record.wayfinder(), - }; - e.insert(Entry::File(data)); - } - } + all_entries.push(RawEntry::from(&record)); + if let Some((url, entry)) = Entry::new(&record, log) { + entries + .entry(url) + .and_modify(|e| *e = Entry::Duplicate) + .or_insert(entry); } } - if expected != actual { + // First check: the number of entries in the central directory matches + // the value in the EOCD record. + let actual = + u64::try_from(all_entries.len()).expect("usize fits in u64"); + if archive.entries_hint() != actual { return Err(ErrorKind::ZipEntryCount { - expected, + expected: archive.entries_hint(), actual, archive_path, } @@ -234,7 +207,7 @@ impl ZipTransport { } let mut ranges = Vec::new(); - for (wayfinder, raw_path) in all_entries { + for RawEntry { raw_path, wayfinder } in all_entries { let entry = try_archive_path!( archive.get_entry(wayfinder), ReadZip, @@ -245,6 +218,8 @@ impl ZipTransport { ReadZip, archive_path ); + // Check that the path in the central directory matches the header + // that comes before the file data range. if raw_path != header.file_path().as_bytes() { return Err(ErrorKind::ZipPathMismatch { central: raw_path, @@ -269,10 +244,9 @@ impl ZipTransport { } // Check that no file ranges overlap with each other. ranges.sort_by_key(|(range, _)| range.start); - for window in ranges.windows(2) { - let [(earlier, earlier_path), (later, later_path)] = window else { - panic!("slice::windows is broken") - }; + for [(earlier, earlier_path), (later, later_path)] in + ranges.array_windows() + { if earlier.end > later.start { return Err(ErrorKind::ZipOverlappingRanges { earlier_path: earlier_path.clone(), @@ -397,6 +371,83 @@ impl Transport } } +#[derive(Debug, Clone, Copy)] +enum Entry { + File(EntryData), + Dir, + Symlink, + Duplicate, +} + +#[derive(Debug, Clone, Copy)] +struct EntryData { + compression_method: CompressionMethod, + wayfinder: ZipArchiveEntryWayfinder, +} + +impl Entry { + fn new( + record: &ZipFileHeaderRecord<'_>, + log: &Logger, + ) -> Option<(Url, Self)> { + static BASE_URL: LazyLock = LazyLock::new(|| { + Url::parse("zip:///").expect("`zip:///` is a valid URL") + }); + + let path = record.file_path(); + let path = path + .try_normalize() + .inspect_err(|err| { + warn!( + log, + "ignoring invalid path in zip archive"; + "path" => path.as_bytes().escape_ascii().to_string(), + "error" => err.to_string(), + ); + }) + .ok()?; + let url = BASE_URL + .join(path.as_str()) + .inspect_err(|err| { + warn!( + log, + "ignoring invalid path in zip archive"; + "path" => path.as_str(), + "error" => err.to_string(), + ); + }) + .ok()?; + + let entry = if record.is_dir() { + Entry::Dir + } else if record.mode().is_symlink() { + Entry::Symlink + } else { + Entry::File(EntryData { + compression_method: record.compression_method(), + wayfinder: record.wayfinder(), + }) + }; + + Some((url, entry)) + } +} + +#[derive(Debug)] +struct RawEntry { + raw_path: Vec, + wayfinder: ZipArchiveEntryWayfinder, +} + +impl<'a> From<&'a ZipFileHeaderRecord<'a>> for RawEntry { + fn from(record: &'a ZipFileHeaderRecord<'a>) -> Self { + Self { + raw_path: record.file_path().as_bytes().to_vec(), + wayfinder: record.wayfinder(), + } + } +} + /// Possible source errors of [`ZipTransport::fetch`]. #[derive(Debug, thiserror::Error)] #[non_exhaustive] @@ -439,30 +490,3 @@ impl ZipTransportError { TransportError::new_with_cause(kind, url, self) } } - -fn path_to_url(path: ZipFilePath>, log: &Logger) -> Option { - path.try_normalize() - .inspect_err(|err| { - warn!( - log, - "ignoring invalid path in zip archive"; - "path" => path.as_bytes().escape_ascii().to_string(), - "error" => err.to_string(), - ); - }) - .ok() - .and_then(|path| { - Url::parse("zip:///") - .expect("`zip:///` is a valid URL") - .join(path.as_str()) - .inspect_err(|err| { - warn!( - log, - "ignoring invalid path in zip archive"; - "path" => path.as_str(), - "error" => err.to_string(), - ); - }) - .ok() - }) -} diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 9788fa4..dfade50 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -2,5 +2,5 @@ # We choose a specific toolchain (rather than "stable") for repeatability. The # intent is to keep this up-to-date with recently-released stable Rust. -channel = "1.91.1" +channel = "1.94.0" profile = "default" From 33885f74135e3c43cf13749bd6ca1a63f4a84a80 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 30 Apr 2026 10:09:15 -0700 Subject: [PATCH 068/137] update rawzip --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3a98516..1560a77 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1829,9 +1829,9 @@ dependencies = [ [[package]] name = "rawzip" -version = "0.4.2" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27cc19f82b641448e861623f52a6a6413bbc0595b62a9d45bf31ccdf18aab72c" +checksum = "9d9575f44c8cf85bc843ad666dcdf20d05a7753772bef56eb2a5140282b32150" [[package]] name = "redox_syscall" diff --git a/Cargo.toml b/Cargo.toml index 9a9311c..52ff742 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,7 +33,7 @@ hubtools = { git = "https://github.com/oxidecomputer/hubtools.git", branch = "ma memchr = "2.7.4" proptest = "1.5.0" rats-corim = { git = "https://github.com/oxidecomputer/rats-corim.git", branch = "main" } -rawzip = "0.4.2" +rawzip = "0.4.4" regex = "1.11.1" schemars = "0.8.21" semver = { version = "1.0.25", features = ["serde"] } From cc29a9bec1e4a613fe1a6e2920a09f6081afe0c3 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Wed, 6 May 2026 11:01:34 -0700 Subject: [PATCH 069/137] make `to_map` fallible --- artifact/src/map.rs | 51 +++------------------------------ artifact/src/metadata.rs | 8 ++++-- artifact/src/tags.rs | 24 ++++++++++------ lib/src/check.rs | 18 ++++++------ lib/src/edit/editor.rs | 62 ++++++++++++++++++++++------------------ lib/src/edit/fake.rs | 2 +- lib/src/edit/input.rs | 50 ++++++++++++++++++-------------- lib/src/error.rs | 12 ++++++++ lib/src/repo/v1.rs | 30 ++++++++++++------- lib/tests/tests.rs | 2 +- 10 files changed, 129 insertions(+), 130 deletions(-) diff --git a/artifact/src/map.rs b/artifact/src/map.rs index 0a325b6..994bd56 100644 --- a/artifact/src/map.rs +++ b/artifact/src/map.rs @@ -13,53 +13,10 @@ pub(crate) fn from_map( serde_json::from_value(map.into_iter().collect()) } -/// Internal function to convert a `Serialize` to a `BTreeMap`. -/// -/// This should always succeed, and callers must use proptests to ensure that -/// it does. -/// -/// # Panics -/// -/// When debug assertions are enabled, this function panics if -/// `serde_json::to_value` returns a type other than `Value::Object` or an -/// error, or if any of the values in the object are not `Value::String`. -/// -/// When debug assertions are disabled, this function returns an empty map -/// if any of these conditions are hit. -pub(crate) fn to_map(s: &S) -> BTreeMap { - macro_rules! debug_panic { - ($($tt:tt)*) => { - if cfg!(debug_assertions) { - panic!($($tt)*); - } else { - return BTreeMap::default() - } - } - } - - let value_map = match serde_json::to_value(s) { - Ok(serde_json::Value::Object(map)) => map, - Ok(not_string) => debug_panic!( - "{} serialized to {not_string:?}, not an object", - std::any::type_name::() - ), - Err(err) => debug_panic!( - "failed to serialize {}: {err:?}", - std::any::type_name::() - ), - }; - let mut map = BTreeMap::new(); - for (key, value) in value_map { - if let serde_json::Value::String(value) = value { - map.insert(key, value); - } else { - debug_panic!( - "{key:?} in {} serialized to {value:?}, not a string", - std::any::type_name::() - ); - } - } - map +pub(crate) fn to_map( + s: &S, +) -> Result, serde_json::Error> { + serde_json::from_value(serde_json::to_value(s)?) } #[cfg(test)] diff --git a/artifact/src/metadata.rs b/artifact/src/metadata.rs index 794d97a..31e03d6 100644 --- a/artifact/src/metadata.rs +++ b/artifact/src/metadata.rs @@ -38,7 +38,9 @@ impl Metadata { } /// Serialize structured metadata into a string mapping. - pub fn to_map(&self) -> BTreeMap { + pub fn to_map( + &self, + ) -> Result, serde_json::Error> { crate::map::to_map(self) } } @@ -61,12 +63,12 @@ mod tests { #[test] fn default_is_empty() { - assert!(Metadata::default().to_map().is_empty()); + assert!(Metadata::default().to_map().unwrap().is_empty()); } #[proptest] fn metadata_roundtrip(metadata: Metadata) { - let map = metadata.to_map(); + let map = metadata.to_map().unwrap(); assert_eq!(Metadata::from_map(map).unwrap(), metadata); } } diff --git a/artifact/src/tags.rs b/artifact/src/tags.rs index 43ced7e..6f7e797 100644 --- a/artifact/src/tags.rs +++ b/artifact/src/tags.rs @@ -16,8 +16,7 @@ use crate::installinator::InstallinatorArtifactKind; // // NOTE: This struct must serialize and deserialize from a mapping of // string keys to string values. The `tags_roundtrip` test covers this -// (crate::map::to_map panics when debug assertions are enabled if this does -// not hold). +// (crate::map::to_map returns an error if this does not hold). #[derive( Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, )] @@ -52,11 +51,6 @@ pub enum KnownArtifactTags { } impl KnownArtifactTags { - /// Returns an adapter for displaying the tags as a human-readable string. - pub fn display(&self) -> DisplayTags<'static> { - self.to_tags().into() - } - /// Resolves known tags from a tag mapping. /// /// # Errors @@ -70,7 +64,9 @@ impl KnownArtifactTags { } /// Converts these known tags to a tag mapping. - pub fn to_tags(&self) -> BTreeMap { + pub fn to_tags( + &self, + ) -> Result, serde_json::Error> { crate::map::to_map(self) } @@ -94,6 +90,13 @@ impl KnownArtifactTags { } } +impl Display for KnownArtifactTags { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let tags = self.to_tags().map_err(|_| std::fmt::Error)?; + DisplayTags::from(tags).fmt(f) + } +} + macro_rules! display_serialize { ($ty:ty) => { impl std::fmt::Display for $ty { @@ -347,9 +350,12 @@ mod tests { use crate::RotSlot; use crate::RotTags; + /// [`KnownArtifactTags::to_tags`] is not allowed to fail (but is marked + /// fallible because it's Serde under the hood); its result must round trip + /// back to the same value when deserialized. #[proptest] fn tags_roundtrip(tags: KnownArtifactTags) { - let map = tags.to_tags(); + let map = tags.to_tags().unwrap(); assert_eq!(KnownArtifactTags::from_tags(map).unwrap(), tags); } diff --git a/lib/src/check.rs b/lib/src/check.rs index 1848dbc..6d73922 100644 --- a/lib/src/check.rs +++ b/lib/src/check.rs @@ -44,13 +44,13 @@ impl Repository { } } - match self.structured_metadata() { - Some(structured_metadata) => { - // Serialize the metadata back to a mapping and verify there - // are no unexpected changes. - let map = structured_metadata.to_map(); - if self.metadata() != &map { - let diff = self.metadata().diff(&map); + match self + .structured_metadata() + .and_then(|structured_metadata| structured_metadata.to_map().ok()) + { + Some(metadata) => { + if self.metadata() != &metadata { + let diff = self.metadata().diff(&metadata); for (key, leaf) in diff.modified() { problems.push(CheckProblem::UnexpectedMetadataDiff { key: key.clone(), @@ -149,7 +149,7 @@ pub enum CheckProblem { }, /// An artifact matching these tags was not found. - #[error("no artifact matching {}", .0.display())] + #[error("no artifact matching {0}")] MissingArtifact(KnownArtifactTags), /// An artifact's target name is not in the repository. @@ -160,7 +160,7 @@ pub enum CheckProblem { MissingTarget { target_name: String }, /// Multiple artifacts for these tags were not expected. - #[error("multiple artifacts found matching {}", .0.display())] + #[error("multiple artifacts found matching {0}")] MultipleArtifacts(KnownArtifactTags), /// The repository metadata is unexpectedly different from the parsed diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index 43db147..1328dfa 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -86,7 +86,7 @@ impl<'a> RepositoryEditor<'a> { ) -> Result { let source = FileSource::open(path).await?; let input = Input::measurement_corpus(source, None).await?; - Ok(self.insert_input(input)) + self.insert_input(input) } /// Add a fake measurement corpus to the repository. @@ -100,7 +100,7 @@ impl<'a> RepositoryEditor<'a> { version: ArtifactVersion, ) -> Result { let input = Input::fake_measurement_corpus(hashes, version, None)?; - Ok(self.insert_input(input)) + self.insert_input(input) } /// Add an OS image to the repository. @@ -120,7 +120,7 @@ impl<'a> RepositoryEditor<'a> { self.artifact_version.clone()?, ) .await?; - Ok(self.insert_input(input)) + self.insert_input(input) } /// Add a fake OS image to the repository. @@ -130,7 +130,7 @@ impl<'a> RepositoryEditor<'a> { self.artifact_version.clone()?, None, ); - Ok(self.insert_input(input)) + self.insert_input(input) } /// Add a Root of Trust Hubris archive to the repository. @@ -143,7 +143,7 @@ impl<'a> RepositoryEditor<'a> { path: Utf8PathBuf, ) -> Result { let source = FileSource::open(path).await?; - Ok(self.insert_input(Input::rot_archive(source, None, rot_slot).await?)) + self.insert_input(Input::rot_archive(source, None, rot_slot).await?) } /// Add a Root of Trust Bootloader Hubris archive to the repository. @@ -154,8 +154,7 @@ impl<'a> RepositoryEditor<'a> { path: Utf8PathBuf, ) -> Result { let source = FileSource::open(path).await?; - Ok(self - .insert_input(Input::rot_bootloader_archive(source, None).await?)) + self.insert_input(Input::rot_bootloader_archive(source, None).await?) } /// Add a Service Processor Hubris archive to the repository. @@ -163,7 +162,7 @@ impl<'a> RepositoryEditor<'a> { /// Tags are automatically determined based on the image caboose. pub async fn sp_archive(self, path: Utf8PathBuf) -> Result { let source = FileSource::open(path).await?; - Ok(self.insert_input(Input::sp_archive(source, None).await?)) + self.insert_input(Input::sp_archive(source, None).await?) } /// Add a fake Root of Trust Hubris archive to the repository. @@ -175,7 +174,7 @@ impl<'a> RepositoryEditor<'a> { self.artifact_version.clone()?, None, )?; - Ok(self.insert_input(input)) + self.insert_input(input) } /// Add a fake Root of Trust Bootloader Hubris archive to the repository. @@ -190,7 +189,7 @@ impl<'a> RepositoryEditor<'a> { self.artifact_version.clone()?, None, )?; - Ok(self.insert_input(input)) + self.insert_input(input) } /// Add a fake Service Processor Hubris archive to the repository. @@ -199,7 +198,7 @@ impl<'a> RepositoryEditor<'a> { pub fn fake_sp_archive(self, tags: SpTags) -> Result { let input = Input::fake_sp_archive(tags, self.artifact_version.clone()?, None)?; - Ok(self.insert_input(input)) + self.insert_input(input) } /// Add a zone image to the repository. @@ -207,7 +206,7 @@ impl<'a> RepositoryEditor<'a> { /// The `zone_name` tag is automatically determined based on the layer /// metadata (the `oxide.json` file that starts zone tarballs). pub async fn zone_image(self, path: Utf8PathBuf) -> Result { - Ok(self.insert_input(Input::zone_image(path).await?)) + self.insert_input(Input::zone_image(path).await?) } /// Add a fake zone image to the repository. @@ -225,7 +224,7 @@ impl<'a> RepositoryEditor<'a> { self.artifact_version.clone()?, None, )?; - Ok(self.insert_input(input)) + self.insert_input(input) } /// Attempt to guess the artifact kind at `path` and add it to the @@ -238,14 +237,14 @@ impl<'a> RepositoryEditor<'a> { path: Utf8PathBuf, ) -> Result { let input = Input::guess(path, self.artifact_version.clone()?).await?; - Ok(self.insert_input(input)) + self.insert_input(input) } - fn insert_input(mut self, input: Input) -> Self + fn insert_input(mut self, input: Input) -> Result where T: Into>, { - for output in input.outputs() { + for output in input.outputs()? { if let Some(artifact) = output.to_artifact_schema() { self.artifacts .entry(artifact.target_name.clone()) @@ -257,7 +256,7 @@ impl<'a> RepositoryEditor<'a> { .or_default() .push(output.source.into()); } - self + Ok(self) } /// Add a non-artifact target to the repository. @@ -293,16 +292,22 @@ impl<'a> RepositoryEditor<'a> { version: ArtifactVersion, tags: &KnownArtifactTags, length: u64, - ) -> Self { + ) -> Result { let prefix = format!("{target_name}\n{version}\n{tags:?}\n"); self.targets .entry(target_name.clone()) .or_default() .push(BytesSource::fake_padded(prefix, length).into()); self.artifacts.entry(target_name.clone()).or_default().insert( - ArtifactSchema { target_name, version, tags: tags.to_tags() }, + ArtifactSchema { + target_name, + version, + tags: tags + .to_tags() + .map_err(ErrorKind::ConvertKnownTagsToMap)?, + }, ); - self + Ok(self) } /// Create a fake repository for testing purposes. @@ -310,7 +315,7 @@ impl<'a> RepositoryEditor<'a> { let mut editor = Self::new(system_version); let version = editor.artifact_version.clone()?; for input in Input::fake(&version, None)? { - editor = editor.insert_input(input); + editor = editor.insert_input(input)?; } Ok(editor) } @@ -328,15 +333,16 @@ impl<'a> RepositoryEditor<'a> { ) -> Result { let mut editor = Self::new(system_version); for input in Input::fake(artifact_version, Some(interior_version))? { - editor = editor.insert_input(input); + editor = editor.insert_input(input)?; } Ok(editor) } /// Set the repository-level metadata. - pub fn metadata(mut self, metadata: &Metadata) -> Self { - self.metadata = metadata.to_map(); - self + pub fn metadata(mut self, metadata: &Metadata) -> Result { + self.metadata = + metadata.to_map().map_err(ErrorKind::ConvertMetadataToMap)?; + Ok(self) } /// Create a `RepositoryEditor` from a loaded repository. @@ -546,10 +552,10 @@ pub(crate) fn generate_installinator_document( } let source = BytesSource::json(&document) .map_err(ErrorKind::SerializeInstallinator)?; - Ok(Output::new( + Output::new( target_name, version, - KnownArtifactTags::InstallinatorDocument, + &KnownArtifactTags::InstallinatorDocument, source, - )) + ) } diff --git a/lib/src/edit/fake.rs b/lib/src/edit/fake.rs index 4e63981..e567c8d 100644 --- a/lib/src/edit/fake.rs +++ b/lib/src/edit/fake.rs @@ -42,7 +42,7 @@ impl ArtifactSetExt for ArtifactSet { fn fake(version: ArtifactVersion) -> Result { let mut artifacts = ArtifactSet::default(); for input in Input::fake(&version, None)? { - for output in input.outputs() { + for output in input.outputs()? { artifacts.extend(output.into_artifact()); } } diff --git a/lib/src/edit/input.rs b/lib/src/edit/input.rs index 4aac1f2..6663f47 100644 --- a/lib/src/edit/input.rs +++ b/lib/src/edit/input.rs @@ -24,6 +24,8 @@ use crate::COSMO_PHASE_1_PATH; use crate::GIMLET_PHASE_1_PATH; use crate::PHASE_2_PATH; use crate::edit::source::BytesSource; +use crate::error::Error; +use crate::error::ErrorKind; use crate::schema::ArtifactSchema; #[derive(Debug)] @@ -67,15 +69,15 @@ pub(crate) enum Input { } impl Input { - pub(crate) fn outputs(self) -> Vec> { - match self { + pub(crate) fn outputs(self) -> Result>, Error> { + Ok(match self { Input::MeasurementCorpus { source, corim_id, sha256, version } => { let target_name = format!( "measurements/{corim_id}-{}.cbor", hex::encode(sha256) ); let tags = KnownArtifactTags::MeasurementCorpus; - vec![Output::new(target_name, version, tags, source)] + vec![Output::new(target_name, version, &tags, source)?] } Input::OsImages { cosmo_phase_1, @@ -90,23 +92,23 @@ impl Input { vec.push(Output::new( base.join(COSMO_PHASE_1_PATH).into(), version.clone(), - OsPhase1Tags { os_board: OsBoard::COSMO, os_variant } + &OsPhase1Tags { os_board: OsBoard::COSMO, os_variant } .into(), cosmo_phase_1, - )); + )?); vec.push(Output::new( base.join(GIMLET_PHASE_1_PATH).into(), version.clone(), - OsPhase1Tags { os_board: OsBoard::GIMLET, os_variant } + &OsPhase1Tags { os_board: OsBoard::GIMLET, os_variant } .into(), gimlet_phase_1, - )); + )?); vec.push(Output::new( base.join(PHASE_2_PATH).into(), version, - OsPhase2Tags { os_variant }.into(), + &OsPhase2Tags { os_variant }.into(), phase_2, - )); + )?); for (file_name, source) in extra_targets { vec.push(Output::extra( base.join(file_name).into(), @@ -122,7 +124,7 @@ impl Input { sign = tags.rot_rkth, slot = tags.rot_slot ); - vec![Output::new(target_name, version, tags.into(), source)] + vec![Output::new(target_name, version, &tags.into(), source)?] } Input::RotBootloader { source, tags, version } => { let target_name = format!( @@ -130,12 +132,17 @@ impl Input { board = tags.rot_board, sign = tags.rot_rkth ); - vec![Output::new(target_name, version, tags.into(), source)] + vec![Output::new(target_name, version, &tags.into(), source)?] } Input::Sp { source, tags, name, version } => { let target_name = format!("sp/{name}-{version}.zip"); if tags.sp_board.as_str() == name { - vec![Output::new(target_name, version, tags.into(), source)] + vec![Output::new( + target_name, + version, + &tags.into(), + source, + )?] } else { // This is likely a lab image. As of writing these are // stored in the TUF repo for manufacturing but are @@ -148,9 +155,9 @@ impl Input { } Input::Zone { source, file_name, tags, version } => { let target_name = format!("zones/{file_name}"); - vec![Output::new(target_name, version, tags.into(), source)] + vec![Output::new(target_name, version, &tags.into(), source)?] } - } + }) } } @@ -163,7 +170,7 @@ pub(crate) struct Output { #[derive(Debug)] pub(crate) struct ArtifactData { - tags: KnownArtifactTags, + tags: BTreeMap, version: ArtifactVersion, } @@ -171,14 +178,15 @@ impl Output { pub(crate) fn new( target_name: String, version: ArtifactVersion, - tags: KnownArtifactTags, + tags: &KnownArtifactTags, source: Source, - ) -> Self { - Output { + ) -> Result { + let tags = tags.to_tags().map_err(ErrorKind::ConvertKnownTagsToMap)?; + Ok(Output { target_name, source, artifact_data: Some(ArtifactData { tags, version }), - } + }) } fn extra(target_name: String, source: Source) -> Self { @@ -190,7 +198,7 @@ impl Output { Some(ArtifactSchema { target_name: self.target_name.clone(), version: data.version.clone(), - tags: data.tags.to_tags(), + tags: data.tags.clone(), }) } } @@ -205,7 +213,7 @@ impl Output { Some(Artifact { target_name: self.target_name, version: data.version, - tags: data.tags.to_tags(), + tags: data.tags, hash: ArtifactHash(hasher.finalize().into()), length: self.source.length(), }) diff --git a/lib/src/error.rs b/lib/src/error.rs index a725da2..c623646 100644 --- a/lib/src/error.rs +++ b/lib/src/error.rs @@ -179,6 +179,11 @@ pub enum ErrorKind { ArtifactVersion(#[from] tufaceous_artifact::ArtifactVersionError), #[error("failed to guess what kind of artifact {path} is")] GuessArtifact { path: Utf8PathBuf }, + #[error( + "failed to convert metadata structure into mapping \ + (please report this logic error in Tufaceous)" + )] + ConvertMetadataToMap(#[source] serde_json::Error), #[error("target name collision on {target_name}")] TargetNameCollision { target_name: String }, #[error( @@ -200,6 +205,11 @@ pub enum ErrorKind { #[error("failed to parse signing root")] ParseSigningRoot(#[source] serde_json::Error), + #[error( + "failed to convert known tags structure into mapping \ + (please report this logic error in Tufaceous)" + )] + ConvertKnownTagsToMap(#[source] serde_json::Error), #[error("failed to read composite artifact {target}")] ReadCompositeArtifact { source: std::io::Error, target: String }, #[error("failed to read oxide.json from {path}")] @@ -231,6 +241,7 @@ impl ErrorKind { | ErrorKind::TargetNotFound { .. } | ErrorKind::ParseTargetJson { .. } | ErrorKind::ArtifactVersion(_) + | ErrorKind::ConvertKnownTagsToMap(_) | ErrorKind::ReadCompositeArtifact { .. } | ErrorKind::ReadZoneOxideJson { .. } => true, @@ -255,6 +266,7 @@ impl ErrorKind { | ErrorKind::SerializeRoot(_) | ErrorKind::RoleVerify(_) | ErrorKind::GuessArtifact { .. } + | ErrorKind::ConvertMetadataToMap(_) | ErrorKind::TargetNameCollision { .. } | ErrorKind::DisallowedTagCollision { .. } | ErrorKind::SerializeArtifacts(_) diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs index 6b7a186..dac3146 100644 --- a/lib/src/repo/v1.rs +++ b/lib/src/repo/v1.rs @@ -171,7 +171,7 @@ pub(crate) async fn from_loaded( &mut unpacked, OsVariant::Host, &version, - ); + )?; continue; } V1KnownArtifactKind::Trampoline => { @@ -180,7 +180,7 @@ pub(crate) async fn from_loaded( &mut unpacked, OsVariant::Recovery, &version, - ); + )?; continue; } @@ -206,7 +206,7 @@ pub(crate) async fn from_loaded( }; let target_name = target; - let tags = tags.to_tags(); + let tags = tags.to_tags().map_err(ErrorKind::ConvertKnownTagsToMap)?; artifacts.insert(Artifact { target_name, version, tags, hash, length }); } @@ -451,7 +451,9 @@ impl CompositeArtifact { artifacts.insert(Artifact { target_name: target_name.clone(), version: version.clone(), - tags: tags.to_tags(), + tags: tags + .to_tags() + .map_err(ErrorKind::ConvertKnownTagsToMap)?, hash, length, }); @@ -468,7 +470,7 @@ impl CompositeArtifact { unpacked: &mut Unpacked, os_variant: OsVariant, version: &ArtifactVersion, - ) { + ) -> Result<(), Error> { for (file_name, tags) in [ ( COSMO_PHASE_1_PATH, @@ -497,12 +499,15 @@ impl CompositeArtifact { artifacts.insert(Artifact { target_name: target_name.clone(), version: version.clone(), - tags: tags.to_tags(), + tags: tags + .to_tags() + .map_err(ErrorKind::ConvertKnownTagsToMap)?, hash: entry.hash, length: entry.length, }); unpacked.entries.insert(target_name, entry); } + Ok(()) } async fn read_control_plane( @@ -523,13 +528,14 @@ impl CompositeArtifact { ) .await?; let file = file.into_inner(); + let tags = + KnownArtifactTags::Zone(ZoneTags { zone_name: layer_info.pkg }); artifacts.insert(Artifact { target_name: target_name.clone(), version: layer_info.version, - tags: KnownArtifactTags::Zone(ZoneTags { - zone_name: layer_info.pkg, - }) - .to_tags(), + tags: tags + .to_tags() + .map_err(ErrorKind::ConvertKnownTagsToMap)?, hash, length, }); @@ -600,7 +606,9 @@ async fn generate_installinator_document( artifacts.insert(Artifact { target_name: target_name.clone(), version, - tags: KnownArtifactTags::InstallinatorDocument.to_tags(), + tags: KnownArtifactTags::InstallinatorDocument + .to_tags() + .map_err(ErrorKind::ConvertKnownTagsToMap)?, hash: unpacked_artifact.hash, length: unpacked_artifact.length, }); diff --git a/lib/tests/tests.rs b/lib/tests/tests.rs index 39b2c22..f0f3bca 100644 --- a/lib/tests/tests.rs +++ b/lib/tests/tests.rs @@ -112,7 +112,7 @@ async fn empty_artifact() -> Result<(), Error> { "1.0.0".parse().unwrap(), &KnownArtifactTags::InstallinatorDocument, 0, - ) + )? .finish() .await? .generate_root() From f401a2789eda9c8ead7f254e03908100ca188b67 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 21 May 2026 13:23:13 -0700 Subject: [PATCH 070/137] keep installinator documents if --no-installinator-document --- bin/src/edit.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/bin/src/edit.rs b/bin/src/edit.rs index b608042..a76c79d 100644 --- a/bin/src/edit.rs +++ b/bin/src/edit.rs @@ -42,11 +42,13 @@ impl Args { let mut editor = RepositoryEditor::from_repo(&repo)? .generate_installinator_document(!self.no_installinator_document); - // always remove an existing installinator document - for artifact in - repo.artifacts().get_all(&KnownArtifactTags::InstallinatorDocument) - { - editor = editor.remove_target(&artifact.target_name); + if !self.no_installinator_document { + for artifact in repo + .artifacts() + .get_all(&KnownArtifactTags::InstallinatorDocument) + { + editor = editor.remove_target(&artifact.target_name); + } } if let Some(version) = self.version { From e53bd8f8b1d86ef84ba5eaae43822cccab5ff817 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 21 May 2026 13:46:19 -0700 Subject: [PATCH 071/137] rename ArtifactSet::get -> get_only --- artifact/src/set.rs | 5 ++++- lib/src/check.rs | 7 +++++-- lib/src/repo.rs | 2 +- lib/tests/v1_compatibility.rs | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/artifact/src/set.rs b/artifact/src/set.rs index 53c0efd..f8d8747 100644 --- a/artifact/src/set.rs +++ b/artifact/src/set.rs @@ -79,7 +79,10 @@ impl ArtifactSet { /// # Errors /// /// Returns an error if there is not exactly one artifact matching `tags`. - pub fn get(&self, tags: &KnownArtifactTags) -> Result<&Artifact, GetError> { + pub fn get_only( + &self, + tags: &KnownArtifactTags, + ) -> Result<&Artifact, GetError> { let set = self.known.get(tags).ok_or(GetError::NotFound)?; if set.len() == 1 && let Some(artifact) = set.first() diff --git a/lib/src/check.rs b/lib/src/check.rs index 6d73922..ba81724 100644 --- a/lib/src/check.rs +++ b/lib/src/check.rs @@ -103,13 +103,16 @@ impl Repository { .push(KnownArtifactTags::OsPhase2(OsPhase2Tags { os_variant })); } for tags in expected { - if matches!(self.artifacts().get(&tags), Err(GetError::NotFound)) { + if matches!( + self.artifacts().get_only(&tags), + Err(GetError::NotFound) + ) { problems.push(CheckProblem::MissingArtifact(tags)); } } if let Ok(artifact) = - self.artifacts().get(&KnownArtifactTags::InstallinatorDocument) + self.artifacts().get_only(&KnownArtifactTags::InstallinatorDocument) && let Ok(stream) = self.read_target(&artifact.target_name).await && let Ok(bytes) = stream.map_ok(Vec::from).try_concat().await && let Ok(doc) = serde_json::from_slice::( diff --git a/lib/src/repo.rs b/lib/src/repo.rs index cbc2529..a9312df 100644 --- a/lib/src/repo.rs +++ b/lib/src/repo.rs @@ -242,7 +242,7 @@ impl Repository { self: &Arc, tags: &KnownArtifactTags, ) -> Result { - let artifact = self.artifacts.get(tags)?.clone(); + let artifact = self.artifacts.get_only(tags)?.clone(); Ok(ArtifactHandle { artifact, repo: Arc::clone(self) }) } diff --git a/lib/tests/v1_compatibility.rs b/lib/tests/v1_compatibility.rs index 3018401..bc8f5f3 100644 --- a/lib/tests/v1_compatibility.rs +++ b/lib/tests/v1_compatibility.rs @@ -73,7 +73,7 @@ async fn v1_fake() -> Result<(), Error> { } let seen = expected .iter() - .map(|tags| repo.artifacts().get(tags).unwrap().clone()) + .map(|tags| repo.artifacts().get_only(tags).unwrap().clone()) .collect::(); // And there should be no unexpected artifacts: assert_eq!(&seen, repo.artifacts()); From ced63067c6fda2e30973c34381d979aa2b3665e3 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 21 May 2026 14:32:21 -0700 Subject: [PATCH 072/137] archive_* args to Repository::from_loaded are never Some --- lib/src/loader.rs | 2 -- lib/src/repo.rs | 19 +++++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/lib/src/loader.rs b/lib/src/loader.rs index 161d076..65d5056 100644 --- a/lib/src/loader.rs +++ b/lib/src/loader.rs @@ -318,8 +318,6 @@ impl RepositoryLoader { repo, log, trust_root, - None, - None, self.v1_compatibility, ) .await; diff --git a/lib/src/repo.rs b/lib/src/repo.rs index a9312df..6928786 100644 --- a/lib/src/repo.rs +++ b/lib/src/repo.rs @@ -46,11 +46,16 @@ pub struct Repository { inner: tough::Repository, system_version: Version, trust_root: Vec, - pub(crate) archive_path: Option, - pub(crate) archive_sha256: Option<[u8; 32]>, artifacts: ArtifactSet, metadata: BTreeMap, v1_unpacked: Option, + + // These are set directly by the ZIP archive convenience methods in the + // loader module. + // TODO after v2 merge: Move the loader module under this repo module so + // that this doesn't need to be pub(crate). + pub(crate) archive_path: Option, + pub(crate) archive_sha256: Option<[u8; 32]>, } impl Repository { @@ -105,8 +110,6 @@ impl Repository { repo: tough::Repository, log: &Logger, trust_root: Vec, - archive_path: Option, - archive_sha256: Option<[u8; 32]>, v1_compatibility: bool, ) -> Result { let Some(ArtifactSetSchema { system_version, artifacts, metadata }) = @@ -119,12 +122,12 @@ impl Repository { return Ok(Repository { inner: repo, trust_root, - archive_path, - archive_sha256, system_version, artifacts, metadata: BTreeMap::new(), v1_unpacked: Some(v1_unpacked), + archive_path: None, + archive_sha256: None, }); } @@ -144,12 +147,12 @@ impl Repository { Ok(Repository { inner: repo, trust_root, - archive_path, - archive_sha256, system_version, artifacts, metadata, v1_unpacked: None, + archive_path: None, + archive_sha256: None, }) } From 115132a43e1bc716ac24d05c25221dec1ff49e9a Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 21 May 2026 15:05:40 -0700 Subject: [PATCH 073/137] KnownArtifactTags tweaks and Artifact docs --- artifact/src/artifact.rs | 11 ++++++++++- artifact/src/tags.rs | 33 ++++++++++++++++++++++++++++++++- lib/src/check.rs | 4 +--- lib/src/edit/editor.rs | 2 +- 4 files changed, 44 insertions(+), 6 deletions(-) diff --git a/artifact/src/artifact.rs b/artifact/src/artifact.rs index 50a70b6..30cf777 100644 --- a/artifact/src/artifact.rs +++ b/artifact/src/artifact.rs @@ -17,6 +17,10 @@ use crate::DisplayTags; use crate::KnownArtifactTags; /// An artifact in a repository. +/// +/// Artifacts are files present in a repository (in TUF parlance, "targets") +/// that the control plane must be aware of and that are used during +/// control-plane-driven update of the system. #[derive( Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, )] @@ -52,7 +56,12 @@ pub struct Artifact { impl Artifact { /// Resolves [`Artifact::tags`] into [`KnownArtifactTags`]. /// - /// Returns `None` if the tags do not resolve to any known artifact. + /// Returns `None` if the tags do not resolve to any known artifact. This + /// is expected if the repository was generated by a future version of + /// Tufaceous that knows about some new artifact. + /// + /// If you instead want an error when the tags do not resolve to any known + /// artifact, use [`KnownArtifactTags::from_tags`]. pub fn known_tags(&self) -> Option { KnownArtifactTags::from_tags(self.tags.clone()).ok() } diff --git a/artifact/src/tags.rs b/artifact/src/tags.rs index 6f7e797..fe0e5cb 100644 --- a/artifact/src/tags.rs +++ b/artifact/src/tags.rs @@ -22,6 +22,7 @@ use crate::installinator::InstallinatorArtifactKind; )] #[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] #[serde(tag = "kind", rename_all = "snake_case")] +#[non_exhaustive] pub enum KnownArtifactTags { /// JSON document describing the artifacts Installinator is responsible for /// writing during mupdate and sled recovery. @@ -85,7 +86,37 @@ impl KnownArtifactTags { zone_name: zone_name.clone(), }) } - _ => None, + + KnownArtifactTags::InstallinatorDocument + | KnownArtifactTags::OsPhase1(_) + | KnownArtifactTags::OsPhase2(OsPhase2Tags { + os_variant: OsVariant::Recovery, + }) + | KnownArtifactTags::Rot(_) + | KnownArtifactTags::RotBootloader(_) + | KnownArtifactTags::Sp(_) => None, + } + } + + /// Returns `true` if this tag set is allowed to appear in the repository + /// more than once. + /// + /// # ⚠️ Causality Hazard ⚠️ + /// + /// This definition is subject to change in future repositories. Do **not** + /// use this to enforce logic if you are possibly reading a repository made + /// by a different version of Tufaceous. + pub fn allow_multiple_artifacts(&self) -> bool { + match self { + KnownArtifactTags::MeasurementCorpus => true, + + KnownArtifactTags::InstallinatorDocument + | KnownArtifactTags::OsPhase1(_) + | KnownArtifactTags::OsPhase2(_) + | KnownArtifactTags::Rot(_) + | KnownArtifactTags::RotBootloader(_) + | KnownArtifactTags::Sp(_) + | KnownArtifactTags::Zone(_) => false, } } } diff --git a/lib/src/check.rs b/lib/src/check.rs index ba81724..282f7ff 100644 --- a/lib/src/check.rs +++ b/lib/src/check.rs @@ -82,9 +82,7 @@ impl Repository { // For all tags we expect to see a single artifact, except for // MeasurementCorpus. for (tags, iter) in self.artifacts().known() { - if !matches!(tags, KnownArtifactTags::MeasurementCorpus) - && iter.len() > 1 - { + if iter.len() > 1 && !tags.allow_multiple_artifacts() { problems.push(CheckProblem::MultipleArtifacts(tags.clone())); } } diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index 1328dfa..4e21442 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -409,7 +409,7 @@ impl<'a> RepositoryEditor<'a> { let mut seen_tags = HashMap::new(); for (target_name, artifact) in &artifacts { if let Some(tags) = artifact.known_tags() - && matches!(tags, KnownArtifactTags::MeasurementCorpus) + && tags.allow_multiple_artifacts() { continue; } From 3984261cc6a7dd4fb09535bcc3d4e6940c0bee62 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 21 May 2026 15:08:16 -0700 Subject: [PATCH 074/137] also use the "causality hazard" warning here --- lib/src/check.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/src/check.rs b/lib/src/check.rs index 282f7ff..7ea3026 100644 --- a/lib/src/check.rs +++ b/lib/src/check.rs @@ -21,9 +21,12 @@ use crate::Repository; impl Repository { /// Check the repository for consistency and other problems. /// - /// This must *not* be used to determine whether to accept a repository by - /// the control plane; it assumes that the current version of Tufaceous is - /// the same one that was used to generate the repository. + /// # ⚠️ Causality Hazard ⚠️ + /// + /// The definition of a well-formed repository is subject to change. This + /// must **not** be used for repositories that may have been created by + /// another version of Tufaceous, such as when the control plane decides to + /// accept a repository. pub async fn check_problems(&self) -> Vec { let mut problems = Vec::new(); From 227fb0c91bd1a48a427278ad94aa65819a00c940 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 21 May 2026 15:51:43 -0700 Subject: [PATCH 075/137] clarify the v1 compat stuff in the installinator module --- artifact/src/installinator.rs | 60 +++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/artifact/src/installinator.rs b/artifact/src/installinator.rs index 0015f42..e65e9c3 100644 --- a/artifact/src/installinator.rs +++ b/artifact/src/installinator.rs @@ -12,13 +12,48 @@ use crate::ArtifactVersion; /// Artifact-specific information used by Installinator. /// -/// This document is treated as an opaque blob by Wicketd and Nexus, since -/// we'd like previous versions of those services to be able to process newer -/// versions of this document. +/// There are (normally) no backwards compatibility constraints for this +/// document, because the version of Tufaceous used to generate it (via the +/// control plane releng tooling) is the same as the version of Tufaceous used +/// to read it (via Installinator), and this is enforced by the use of workspace +/// dependencies in Omicron. /// -/// There are no backwards compatibility constraints for this document. The -/// version of Installinator that processes this document is the same as the -/// version of Tufaceous that creates it. +/// Both Wicketd and Nexus treat this document as an opaque blob, since we'd +/// like previous versions of those services to be able to start the "sled +/// recovery" workflow in which the recovery phase 1 image is flashed to a +/// sled, which retrieves the recovery phase 2 image via the SP, which starts +/// Installinator and retrieves the artifacts required to boot normally via the +/// bootstrap network. +/// +/// # v1 compatibility notes +/// +/// In the first release after Tufaceous v2 is integrated in Wicket and +/// Installinator, this will be the situation mupdate is in: +/// +/// - Installinator is using Tufaceous v2 to read the Installinator document. +/// - Wicket on the currently-running system is using Tufaceous v1, and does +/// not extract the composite control plane artifact; thus the individual zone +/// artifacts are not available via the bootstrap network. +/// - The Installinator document was written by Tufaceous v1, because Wicket +/// can only understand v1 repos. (Tufaceous v2 will not generate v1 repos, so +/// releng will use both versions until we can stop generating v1 repos.) +/// +/// This violates the constraint that the same version of Tufaceous is used to +/// generate and read the Installinator document. So, Installinator needs its +/// own backwards compatibility code, which some temporary code in this module +/// supports: +/// +/// - `name` is accepted as an alias for [`InstallinatorArtifact::file_name`]. +/// We renamed this field in Tufaceous v2 for clarity. +/// - `ControlPlane` is an accepted variant for [`InstallinatorArtifactKind`]. +/// This indicates to Installinator that an older version of Wicket was used +/// to read the repository, and that it will need to fetch the composite +/// artifact and unpack the control plane zones. +/// +/// These temporary compatibility shims can be removed (along with +/// Installinator's v1 compatibility code) in the control plane release after +/// Tufaceous v2 was integrated into Wicket and Installinator, assuming that we +/// continue to disallow skipping major versions during upgrades. #[derive(Debug, Clone, Deserialize, Serialize)] pub struct InstallinatorDocument { /// The system version of the repository this document is associated with. @@ -47,9 +82,7 @@ pub struct InstallinatorArtifact { pub hash: ArtifactHash, /// A file name without directory separators; not necessarily the target /// name. - // This alias is present for backwards compatibility with Tufaceous v1 - // repositories uploaded to older versions of Wicket that are still using - // Tufaceous v1. This alias can be removed when the transition is complete. + // (temporary alias; see "v1 compatibility notes" above) #[serde(alias = "name")] pub file_name: String, } @@ -70,12 +103,7 @@ pub enum InstallinatorArtifactKind { /// file. This may differ from the file name. zone_name: String, }, - /// A tarball of control plane zones. Used for backwards compatibility - /// only. - /// - /// This variant is present for backwards compatibility with Tufaceous - /// v1 repositories uploaded to older versions of Wicket that are still - /// using Tufaceous v1. This variant can be removed when the transition - /// is complete. + /// A tarball of control plane zones. See "v1 compatibility notes" on + /// [`InstallinatorDocument`]. ControlPlane, } From a4c160ab5b2e20a74d2bcb55eaf48fc015f3b48a Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 21 May 2026 15:54:57 -0700 Subject: [PATCH 076/137] deserialize_from_empty -> deserialize_previously_valid --- artifact/src/metadata.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/artifact/src/metadata.rs b/artifact/src/metadata.rs index 31e03d6..2dc21e1 100644 --- a/artifact/src/metadata.rs +++ b/artifact/src/metadata.rs @@ -22,6 +22,9 @@ use serde::Serialize; // 1. Wrap all fields in [`Option`] and mark with // `#[serde(skip_serializing_if = "Option::is_none")]`. // 2. Mark nested structs with `#[serde(flatten)]`. +// +// If you add metadata, please add to the list in the +// `deserialize_previously_valid` test below. #[derive(Debug, Default, Clone, PartialEq, Eq, Deserialize, Serialize)] #[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] pub struct Metadata { @@ -53,12 +56,12 @@ mod tests { use crate::Metadata; - /// **Do not change this test when adding new metadata.** This structure - /// must be capable of deserializing older metadata, and the oldest metadata - /// is no metadata at all. #[test] - fn deserialize_from_empty() { - Metadata::from_map(BTreeMap::new()).unwrap(); + fn deserialize_previously_valid() { + #[expect(clippy::single_element_loop)] + for previously_valid in [BTreeMap::new()] { + Metadata::from_map(previously_valid).unwrap(); + } } #[test] From 4a410374f958fa76cad079686b3fb6560a0393c7 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 21 May 2026 15:57:56 -0700 Subject: [PATCH 077/137] add TODO for LoadOptions --- bin/src/edit.rs | 2 ++ bin/src/list_targets.rs | 2 ++ bin/src/show_target.rs | 2 ++ bin/src/verify.rs | 2 ++ 4 files changed, 8 insertions(+) diff --git a/bin/src/edit.rs b/bin/src/edit.rs index a76c79d..890cc78 100644 --- a/bin/src/edit.rs +++ b/bin/src/edit.rs @@ -35,6 +35,8 @@ pub struct Args { impl Args { pub async fn run(self) -> Result<()> { let repo = RepositoryLoader::new() + // TODO after v2 merge: Create a LoadOptions struct for configuring + // expiration enforcement and trust store behavior. .expiration_enforcement(ExpirationEnforcement::Unsafe) .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) .load_zip_path(self.repo.clone(), &crate::LOG) diff --git a/bin/src/list_targets.rs b/bin/src/list_targets.rs index 0d25368..14b1956 100644 --- a/bin/src/list_targets.rs +++ b/bin/src/list_targets.rs @@ -19,6 +19,8 @@ pub struct Args { impl Args { pub async fn run(self) -> Result<()> { let repo = RepositoryLoader::new() + // TODO after v2 merge: Create a LoadOptions struct for configuring + // expiration enforcement and trust store behavior. .expiration_enforcement(ExpirationEnforcement::Unsafe) .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) .load_zip_path(self.repo.clone(), &crate::LOG) diff --git a/bin/src/show_target.rs b/bin/src/show_target.rs index 689e2f3..ae3888b 100644 --- a/bin/src/show_target.rs +++ b/bin/src/show_target.rs @@ -22,6 +22,8 @@ pub struct Args { impl Args { pub async fn run(self) -> Result<()> { let repo = RepositoryLoader::new() + // TODO after v2 merge: Create a LoadOptions struct for configuring + // expiration enforcement and trust store behavior. .expiration_enforcement(ExpirationEnforcement::Unsafe) .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) .load_zip_path(self.repo.clone(), &crate::LOG) diff --git a/bin/src/verify.rs b/bin/src/verify.rs index da5b60b..f4cb29f 100644 --- a/bin/src/verify.rs +++ b/bin/src/verify.rs @@ -32,6 +32,8 @@ fn default_threads() -> usize { impl Args { pub async fn run(self) -> Result<()> { let repo = RepositoryLoader::new() + // TODO after v2 merge: Create a LoadOptions struct for configuring + // expiration enforcement and trust store behavior. .expiration_enforcement(ExpirationEnforcement::Unsafe) .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) .v1_compatibility(true) From 4fb79bca104127c88e6f6c257ab6a4b9fa19484e Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 21 May 2026 16:04:10 -0700 Subject: [PATCH 078/137] fixup! rename ArtifactSet::get -> get_only --- artifact/src/set.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/artifact/src/set.rs b/artifact/src/set.rs index f8d8747..9d2da96 100644 --- a/artifact/src/set.rs +++ b/artifact/src/set.rs @@ -96,8 +96,8 @@ impl ArtifactSet { /// Returns the set of all artifacts matching `tags`. /// /// If you are not certain that you want all matching artifacts, prefer - /// [`ArtifactSet::get`] instead. The control plane must not randomly choose - /// from matching artifacts when it expects only one. + /// [`ArtifactSet::get_only`] instead. The control plane must not randomly + /// choose from matching artifacts when it expects only one. pub fn get_all(&self, tags: &KnownArtifactTags) -> &BTreeSet { static EMPTY: BTreeSet = BTreeSet::new(); self.known.get(tags).unwrap_or(&EMPTY) @@ -263,7 +263,7 @@ impl<'a> Iterator for Iter<'a> { } } -/// Returned by [`ArtifactSet::get`] when there is not exactly one matching +/// Returned by [`ArtifactSet::get_only`] when there is not exactly one matching /// artifact. #[derive(Debug, thiserror::Error)] pub enum GetError { From 4ea661bbe687ee0ea9a84d74c3a5616eb8fbac13 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 11 Jun 2026 11:49:09 -0700 Subject: [PATCH 079/137] lock hubtools/rats-corim to same commits as omicron --- Cargo.lock | 7 ++++--- Cargo.toml | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e3413f9..e53062f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -982,9 +982,10 @@ dependencies = [ [[package]] name = "hubtools" -version = "0.4.6" -source = "git+https://github.com/oxidecomputer/hubtools.git?branch=main#3b72a550f2b5ec7ad489b8822b26239b7176bfdd" +version = "0.4.7" +source = "git+https://github.com/oxidecomputer/hubtools.git?rev=2b1ef9b38d75563ea800baa3b17327eec17b1b7a#2b1ef9b38d75563ea800baa3b17327eec17b1b7a" dependencies = [ + "digest", "hex", "lpc55_areas", "lpc55_sign", @@ -1704,7 +1705,7 @@ dependencies = [ [[package]] name = "rats-corim" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/rats-corim.git?branch=main#f0d5d5168d3d31487a56df32c676b0c6240bcc6b" +source = "git+https://github.com/oxidecomputer/rats-corim.git?rev=f0d5d5168d3d31487a56df32c676b0c6240bcc6b#f0d5d5168d3d31487a56df32c676b0c6240bcc6b" dependencies = [ "ciborium", "ciborium-io", diff --git a/Cargo.toml b/Cargo.toml index 7207517..5b3077f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,10 +29,10 @@ expectorate = "1.1.0" flate2 = "1.0.35" futures-util = "0.3.31" hex = "0.4.3" -hubtools = { git = "https://github.com/oxidecomputer/hubtools.git", branch = "main" } +hubtools = { git = "https://github.com/oxidecomputer/hubtools.git", rev = "2b1ef9b38d75563ea800baa3b17327eec17b1b7a" } memchr = "2.7.4" proptest = "1.5.0" -rats-corim = { git = "https://github.com/oxidecomputer/rats-corim.git", branch = "main" } +rats-corim = { git = "https://github.com/oxidecomputer/rats-corim.git", rev = "f0d5d5168d3d31487a56df32c676b0c6240bcc6b" } rawzip = "0.4.4" regex = "1.11.1" schemars = "0.8.21" From d64ef973ca72fcab30ca2e9e0486cb6c8087148d Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 11 Jun 2026 11:50:41 -0700 Subject: [PATCH 080/137] update packages implicated by `cargo audit` --- Cargo.lock | 61 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e53062f..e6aa08c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -627,12 +627,12 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.11" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ "powerfmt", - "serde", + "serde_core", ] [[package]] @@ -1402,9 +1402,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.1.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" @@ -1665,9 +1665,9 @@ checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" [[package]] name = "rand" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", "rand_chacha", @@ -1863,9 +1863,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.19" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "934b404430bb06b3fae2cba809eb45a1ab1aecd64491213d7c3301b88393f8d1" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "aws-lc-rs", "log", @@ -1878,15 +1878,18 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.10.0" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "zeroize", +] [[package]] name = "rustls-webpki" -version = "0.102.8" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "aws-lc-rs", "ring", @@ -1986,10 +1989,11 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.218" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8dfc9d19bdbf6d17e22319da49161d5d0108e4188e8b680aef6299eed22df60" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ + "serde_core", "serde_derive", ] @@ -2004,11 +2008,20 @@ dependencies = [ "smallvec 0.6.14", ] +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + [[package]] name = "serde_derive" -version = "1.0.218" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09503e191f4e797cb8aac08e9a4a4695c5edf6a2e70e376d961ddd5c969f82b" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", @@ -2486,30 +2499,30 @@ dependencies = [ [[package]] name = "time" -version = "0.3.36" +version = "0.3.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ "deranged", "itoa", "num-conv", "powerfmt", - "serde", + "serde_core", "time-core", "time-macros", ] [[package]] name = "time-core" -version = "0.1.2" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" [[package]] name = "time-macros" -version = "0.2.18" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" dependencies = [ "num-conv", "time-core", From e021b770b087de271ac974ee02bd2e057a8b0be6 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 11 Jun 2026 12:23:34 -0700 Subject: [PATCH 081/137] serde_human_bytes -> byte-wrapper; sha256 0.11 --- Cargo.lock | 135 ++++++++++++++++++++++------- Cargo.toml | 5 +- artifact/Cargo.toml | 5 +- artifact/src/artifact.rs | 33 ++----- artifact/src/rkth.rs | 3 +- lib/Cargo.toml | 1 + lib/src/edit/input.rs | 8 +- lib/src/edit/measurement_corpus.rs | 2 +- lib/src/edit/source.rs | 4 +- lib/src/loader.rs | 5 +- lib/src/repo/v1.rs | 5 +- 11 files changed, 132 insertions(+), 74 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e6aa08c..ce349f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -269,6 +269,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "bstr" version = "1.10.0" @@ -285,6 +294,18 @@ version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +[[package]] +name = "byte-wrapper" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b041d2328ab2041b4e9d8f21e3b0b5b8bb61e9a23bb9171387771c314ea340" +dependencies = [ + "hex", + "schemars 0.8.22", + "serde_core", + "serde_json", +] + [[package]] name = "byteorder" version = "1.5.0" @@ -469,6 +490,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -484,6 +511,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc" version = "3.2.1" @@ -545,6 +581,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "daft" version = "0.1.1" @@ -607,7 +652,7 @@ version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" dependencies = [ - "const-oid", + "const-oid 0.9.6", "der_derive", "flagset", "pem-rfc7468", @@ -641,9 +686,29 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.6", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", +] + +[[package]] +name = "digest-io" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2de63d600bc7fab91180bc17385f29b342468dc8ef2af09dceba450a293de3da" +dependencies = [ + "digest 0.11.3", ] [[package]] @@ -976,16 +1041,13 @@ name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -dependencies = [ - "serde", -] [[package]] name = "hubtools" version = "0.4.7" source = "git+https://github.com/oxidecomputer/hubtools.git?rev=2b1ef9b38d75563ea800baa3b17327eec17b1b7a#2b1ef9b38d75563ea800baa3b17327eec17b1b7a" dependencies = [ - "digest", + "digest 0.10.7", "hex", "lpc55_areas", "lpc55_sign", @@ -1001,6 +1063,15 @@ dependencies = [ "zip", ] +[[package]] +name = "hybrid-array" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +dependencies = [ + "typenum", +] + [[package]] name = "iana-time-zone" version = "0.1.61" @@ -1316,7 +1387,7 @@ version = "0.3.4" source = "git+https://github.com/oxidecomputer/lpc55_support#17d04af60b3a4fd82c77b1a33ca5370943cd25d9" dependencies = [ "byteorder", - "const-oid", + "const-oid 0.9.6", "crc-any", "der", "env_logger", @@ -1329,7 +1400,7 @@ dependencies = [ "rsa", "serde", "serde-hex", - "sha2", + "sha2 0.10.8", "thiserror 1.0.69", "x509-cert", "zerocopy 0.6.6", @@ -1813,8 +1884,8 @@ version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ - "const-oid", - "digest", + "const-oid 0.9.6", + "digest 0.10.7", "num-bigint-dig", "num-integer", "num-traits", @@ -1822,7 +1893,7 @@ dependencies = [ "pkcs8", "rand_core", "serde", - "sha2", + "sha2 0.10.8", "signature", "spki", "subtle", @@ -2039,15 +2110,6 @@ dependencies = [ "syn 2.0.96", ] -[[package]] -name = "serde_human_bytes" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/serde_human_bytes.git?branch=main#0a09794501b6208120528c3b457d5f3a8cb17424" -dependencies = [ - "hex", - "serde", -] - [[package]] name = "serde_json" version = "1.0.139" @@ -2117,8 +2179,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.14", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -2133,7 +2206,7 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "digest", + "digest 0.10.7", "rand_core", ] @@ -2737,6 +2810,7 @@ dependencies = [ "chrono", "ciborium", "daft", + "digest-io", "flate2", "futures-util", "hex", @@ -2747,7 +2821,7 @@ dependencies = [ "semver", "serde", "serde_json", - "sha2", + "sha2 0.11.0", "slog", "tar", "thiserror 2.0.12", @@ -2763,15 +2837,14 @@ dependencies = [ name = "tufaceous-artifact" version = "0.2.0" dependencies = [ + "byte-wrapper", "daft", "expectorate", - "hex", "hubtools", "proptest", "regex", "schemars 0.8.22", "serde", - "serde_human_bytes", "serde_json", "test-strategy", "thiserror 2.0.12", @@ -2815,9 +2888,9 @@ checksum = "82205ffd44a9697e34fc145491aa47310f9871540bb7909eaa9365e0a9a46607" [[package]] name = "typenum" -version = "1.17.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "unarray" @@ -3140,7 +3213,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" dependencies = [ - "const-oid", + "const-oid 0.9.6", "der", "spki", "tls_codec", diff --git a/Cargo.toml b/Cargo.toml index 5b3077f..0ded5e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ anyhow = "1.0.96" assert_cmd = "2.0.16" atomicwrites = "0.4.4" aws-lc-rs = "1.16.2" +byte-wrapper = { version = "0.1.0", default-features = false, features = ["alloc", "hex", "serde"] } bytes = "1.11.1" camino = "1.1.9" camino-tempfile = "1.1.1" @@ -25,6 +26,7 @@ chrono = { version = "0.4.40", default-features = false, features = ["std"] } ciborium = "0.2.2" clap = { version = "4.5.31", features = ["cargo", "derive", "wrap_help"] } daft = { version = "0.1.1", features = ["derive"] } +digest-io = "0.1.0" expectorate = "1.1.0" flate2 = "1.0.35" futures-util = "0.3.31" @@ -38,9 +40,8 @@ regex = "1.11.1" schemars = "0.8.21" semver = { version = "1.0.25", features = ["serde"] } serde = { version = "1.0.218", features = ["derive"] } -serde_human_bytes = { git = "https://github.com/oxidecomputer/serde_human_bytes.git", branch = "main" } serde_json = "1.0.139" -sha2 = "0.10.8" +sha2 = "0.11.0" slog = "2.7.0" slog-async = "2.8.0" slog-envlogger = "2.2.0" diff --git a/artifact/Cargo.toml b/artifact/Cargo.toml index 0c9dc61..2fc8c80 100644 --- a/artifact/Cargo.toml +++ b/artifact/Cargo.toml @@ -9,16 +9,15 @@ rust-version = "1.94" [features] hubtools = ["dep:hubtools"] proptest = ["dep:proptest", "dep:test-strategy"] -schemars = ["dep:schemars"] +schemars = ["dep:schemars", "byte-wrapper/schemars08"] [dependencies] +byte-wrapper.workspace = true daft.workspace = true -hex.workspace = true hubtools = { workspace = true, optional = true } proptest = { workspace = true, optional = true } schemars = { workspace = true, optional = true } serde.workspace = true -serde_human_bytes.workspace = true serde_json.workspace = true test-strategy = { workspace = true, optional = true } thiserror.workspace = true diff --git a/artifact/src/artifact.rs b/artifact/src/artifact.rs index 30cf777..0a4fb67 100644 --- a/artifact/src/artifact.rs +++ b/artifact/src/artifact.rs @@ -7,8 +7,9 @@ use std::fmt::Debug; use std::fmt::Display; use std::str::FromStr; +use byte_wrapper::HexArray; +use byte_wrapper::ParseHexError; use daft::Diffable; -use hex::FromHexError; use serde::Deserialize; use serde::Serialize; @@ -91,56 +92,40 @@ impl Artifact { #[cfg_attr(feature = "proptest", derive(test_strategy::Arbitrary))] #[serde(transparent)] pub struct ArtifactHash( - #[serde(with = "serde_human_bytes::hex_array")] + #[serde(with = "HexArray::<32>")] #[cfg_attr( feature = "schemars", - schemars(schema_with = "hex_schema::<32>") + schemars(schema_with = "HexArray::<32>::json_schema") )] pub [u8; 32], ); impl AsRef<[u8]> for ArtifactHash { fn as_ref(&self) -> &[u8] { - &self.0 + self.0.as_slice() } } impl Debug for ArtifactHash { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_tuple("ArtifactHash").field(&hex::encode(self.0)).finish() + f.debug_tuple("ArtifactHash").field(&HexArray(self.0)).finish() } } impl Display for ArtifactHash { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - Display::fmt(&hex::encode(self.0), f) + Display::fmt(&HexArray(self.0), f) } } impl FromStr for ArtifactHash { - type Err = FromHexError; + type Err = ParseHexError; fn from_str(s: &str) -> std::result::Result { - let mut out = [0u8; 32]; - hex::decode_to_slice(s, &mut out)?; - Ok(Self(out)) + s.parse() } } -/// Produce an OpenAPI schema describing a hex array of a specific length (e.g., -/// a hash digest). -#[cfg(feature = "schemars")] -fn hex_schema( - generator: &mut schemars::SchemaGenerator, -) -> schemars::schema::Schema { - use schemars::JsonSchema; - - let mut schema: schemars::schema::SchemaObject = - ::json_schema(generator).into(); - schema.format = Some(format!("hex string ({N} bytes)")); - schema.into() -} - #[cfg(test)] mod tests { use super::*; diff --git a/artifact/src/rkth.rs b/artifact/src/rkth.rs index 5e39168..602a484 100644 --- a/artifact/src/rkth.rs +++ b/artifact/src/rkth.rs @@ -7,6 +7,7 @@ use std::fmt::Debug; use std::fmt::Display; use std::sync::LazyLock; +use byte_wrapper::HexArray; use serde::Deserialize; use serde::Serialize; @@ -70,7 +71,7 @@ pub struct RotKeyTableHash(pub Option); impl RotKeyTableHash { /// Create a `RotKeyTableHash` from an array of 32 bytes. pub fn from_bytes(hash: [u8; 32]) -> Self { - Self(Some(hex::encode(hash))) + Self(Some(HexArray(hash).to_string())) } /// Returns a friendly name for the CA this RKTH represents, if one is diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 7f7e2df..dc78a27 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -15,6 +15,7 @@ camino-tempfile.workspace = true chrono.workspace = true ciborium.workspace = true daft.workspace = true +digest-io.workspace = true flate2.workspace = true futures-util.workspace = true hex.workspace = true diff --git a/lib/src/edit/input.rs b/lib/src/edit/input.rs index 6663f47..2c07570 100644 --- a/lib/src/edit/input.rs +++ b/lib/src/edit/input.rs @@ -72,10 +72,8 @@ impl Input { pub(crate) fn outputs(self) -> Result>, Error> { Ok(match self { Input::MeasurementCorpus { source, corim_id, sha256, version } => { - let target_name = format!( - "measurements/{corim_id}-{}.cbor", - hex::encode(sha256) - ); + let target_name = + format!("measurements/{corim_id}-{sha256}.cbor",); let tags = KnownArtifactTags::MeasurementCorpus; vec![Output::new(target_name, version, &tags, source)?] } @@ -214,7 +212,7 @@ impl Output { target_name: self.target_name, version: data.version, tags: data.tags, - hash: ArtifactHash(hasher.finalize().into()), + hash: ArtifactHash(hasher.finalize().0), length: self.source.length(), }) } diff --git a/lib/src/edit/measurement_corpus.rs b/lib/src/edit/measurement_corpus.rs index 4193df5..5bd2964 100644 --- a/lib/src/edit/measurement_corpus.rs +++ b/lib/src/edit/measurement_corpus.rs @@ -93,7 +93,7 @@ impl Input { .map_err(ErrorKind::SerializeFakeMeasurementCorpus)?; Ok(Input::MeasurementCorpus { corim_id: corim.id, - sha256: ArtifactHash(Sha256::digest(&bytes).into()), + sha256: ArtifactHash(Sha256::digest(&bytes).0), source: BytesSource::new(bytes), version, }) diff --git a/lib/src/edit/source.rs b/lib/src/edit/source.rs index 86ba8db..fe05698 100644 --- a/lib/src/edit/source.rs +++ b/lib/src/edit/source.rs @@ -154,7 +154,7 @@ impl BytesSource { while let Some(Ok(bytes)) = stream.next().await { hasher.update(&bytes); } - *self.sha256.insert(ArtifactHash(hasher.finalize().into())) + *self.sha256.insert(ArtifactHash(hasher.finalize().0)) } pub(crate) async fn into_target(mut self) -> Target<'static> { @@ -231,7 +231,7 @@ impl FileSource { }, ) .await?; - let sha256 = ArtifactHash(hasher.finalize().into()); + let sha256 = ArtifactHash(hasher.finalize().0); Ok({ let mut guard = self.inner.length_sha256.lock().expect("poisoned"); *guard.insert((length, sha256)) diff --git a/lib/src/loader.rs b/lib/src/loader.rs index 65d5056..0c0ef1d 100644 --- a/lib/src/loader.rs +++ b/lib/src/loader.rs @@ -7,6 +7,7 @@ use std::io::Seek; use bytes::Bytes; use camino::Utf8PathBuf; +use digest_io::IoWrapper; use futures_util::Stream; use futures_util::TryStreamExt; use futures_util::pin_mut; @@ -190,14 +191,14 @@ impl RepositoryLoader { let (file, sha256) = if self.compute_archive_sha256 { let archive_path = archive_path.clone(); tokio::task::spawn_blocking(move || { - let mut hasher = Sha256::new(); + let mut hasher = IoWrapper(Sha256::new()); try_path!( file.rewind() .and_then(|()| std::io::copy(&mut file, &mut hasher)), ReadFile, archive_path ); - Ok::<_, Error>((file, Some(hasher.finalize().into()))) + Ok::<_, Error>((file, Some(hasher.0.finalize().0))) }) .await?? } else { diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs index dac3146..6b3a6f0 100644 --- a/lib/src/repo/v1.rs +++ b/lib/src/repo/v1.rs @@ -256,7 +256,7 @@ impl UnpackedArtifact { length += u64::try_from(len).expect("usize fits in u64"); } let file = Arc::new(file.into()); - let hash = ArtifactHash(hasher.finalize().into()); + let hash = ArtifactHash(hasher.finalize().0); Ok(Self { file, hash, length }) } @@ -280,8 +280,7 @@ impl UnpackedArtifact { .await??; let bytes = buf.split().freeze(); if bytes.is_empty() { - let msg = if this.hash - != ArtifactHash(hasher.finalize().into()) + let msg = if this.hash != ArtifactHash(hasher.finalize().0) { "invalid checksum" } else if this.length != bytes_read { From b8f65d30b52379acb77f83d1a378294bd151a14b Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 11 Jun 2026 16:11:48 -0700 Subject: [PATCH 082/137] remove target_name from Artifact --- artifact/src/artifact.rs | 7 -- bin/src/edit.rs | 8 +- lib/src/edit/editor.rs | 35 ++++-- lib/src/edit/fake.rs | 10 +- lib/src/edit/input.rs | 18 +-- lib/src/error.rs | 23 ++-- lib/src/lib.rs | 2 - lib/src/repo.rs | 207 +++++++++++++++++++++---------- lib/src/{ => repo}/check.rs | 64 ++++++++-- lib/src/repo/v1.rs | 239 ++++++++++++++++++++---------------- lib/src/util.rs | 22 +++- lib/tests/tests.rs | 4 +- 12 files changed, 403 insertions(+), 236 deletions(-) rename lib/src/{ => repo}/check.rs (76%) diff --git a/artifact/src/artifact.rs b/artifact/src/artifact.rs index 0a4fb67..262aa5a 100644 --- a/artifact/src/artifact.rs +++ b/artifact/src/artifact.rs @@ -26,13 +26,6 @@ use crate::KnownArtifactTags; Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, )] pub struct Artifact { - /// The TUF target name, if this value is associated with a loaded - /// repository. - /// - /// If you are instantiating this struct from other records (such as a - /// database), you can use `String::new()` here. - pub target_name: String, - /// The version of the artifact. pub version: ArtifactVersion, diff --git a/bin/src/edit.rs b/bin/src/edit.rs index 890cc78..4cee034 100644 --- a/bin/src/edit.rs +++ b/bin/src/edit.rs @@ -45,12 +45,8 @@ impl Args { .generate_installinator_document(!self.no_installinator_document); if !self.no_installinator_document { - for artifact in repo - .artifacts() - .get_all(&KnownArtifactTags::InstallinatorDocument) - { - editor = editor.remove_target(&artifact.target_name); - } + editor = editor + .remove_artifacts(&KnownArtifactTags::InstallinatorDocument)?; } if let Some(version) = self.version { diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index 4e21442..3da94fa 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -277,6 +277,23 @@ impl<'a> RepositoryEditor<'a> { Ok(self) } + /// Remove artifacts matching the given tags. + pub fn remove_artifacts( + mut self, + filter: &KnownArtifactTags, + ) -> Result { + let filter = + filter.to_tags().map_err(ErrorKind::ConvertKnownTagsToMap)?; + let removed = self.artifacts.extract_if(|_, artifacts| { + artifacts.retain(|artifact| artifact.tags != filter); + artifacts.is_empty() // extract_if: remove from map if true + }); + for (target_name, _) in removed { + self.targets.remove(&target_name); + } + Ok(self) + } + /// Remove a target with the given target name. pub fn remove_target(mut self, target_name: &str) -> Self { self.targets.remove(target_name); @@ -356,10 +373,12 @@ impl<'a> RepositoryEditor<'a> { /// Import all of the artifacts and targets from a repository into this /// editor. pub fn import_repo(mut self, repo: &'a Repository) -> Result { - if repo.is_v1() { - return Err(ErrorKind::ImportV1Repo.into()); + for artifact in repo.to_artifact_schema()? { + self.artifacts + .entry(artifact.target_name.clone()) + .or_default() + .insert(artifact); } - for (target_name, target) in repo.targets() { if target_name.raw() == ArtifactSetSchema::TARGET_NAME { continue; @@ -374,16 +393,6 @@ impl<'a> RepositoryEditor<'a> { .into(), ); } - for artifact in repo.artifacts() { - self.artifacts - .entry(artifact.target_name.clone()) - .or_default() - .insert(ArtifactSchema { - target_name: artifact.target_name.clone(), - version: artifact.version.clone(), - tags: artifact.tags.clone(), - }); - } self.metadata = repo.metadata().clone(); Ok(self) } diff --git a/lib/src/edit/fake.rs b/lib/src/edit/fake.rs index e567c8d..1bee715 100644 --- a/lib/src/edit/fake.rs +++ b/lib/src/edit/fake.rs @@ -2,6 +2,8 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use std::collections::HashMap; + use tufaceous_artifact::ArtifactSet; use tufaceous_artifact::ArtifactVersion; use tufaceous_artifact::OsVariant; @@ -40,7 +42,7 @@ pub trait ArtifactSetExt: Sized { impl ArtifactSetExt for ArtifactSet { fn fake(version: ArtifactVersion) -> Result { - let mut artifacts = ArtifactSet::default(); + let mut artifacts = HashMap::new(); for input in Input::fake(&version, None)? { for output in input.outputs()? { artifacts.extend(output.into_artifact()); @@ -48,10 +50,10 @@ impl ArtifactSetExt for ArtifactSet { } artifacts.extend( generate_installinator_document( - artifacts.iter().map(|artifact| { + artifacts.iter().map(|(target_name, artifact)| { ( ArtifactSchema { - target_name: artifact.target_name.clone(), + target_name: target_name.clone(), version: artifact.version.clone(), tags: artifact.tags.clone(), }, @@ -62,7 +64,7 @@ impl ArtifactSetExt for ArtifactSet { )? .into_artifact(), ); - Ok(artifacts) + Ok(artifacts.into_values().collect()) } } diff --git a/lib/src/edit/input.rs b/lib/src/edit/input.rs index 2c07570..e44fd78 100644 --- a/lib/src/edit/input.rs +++ b/lib/src/edit/input.rs @@ -202,18 +202,20 @@ impl Output { } impl Output { - pub(crate) fn into_artifact(self) -> Option { + pub(crate) fn into_artifact(self) -> Option<(String, Artifact)> { let data = self.artifact_data?; let mut hasher = Sha256::new(); for bytes in self.source.iter_bytes() { hasher.update(bytes); } - Some(Artifact { - target_name: self.target_name, - version: data.version, - tags: data.tags, - hash: ArtifactHash(hasher.finalize().0), - length: self.source.length(), - }) + Some(( + self.target_name, + Artifact { + version: data.version, + tags: data.tags, + hash: ArtifactHash(hasher.finalize().0), + length: self.source.length(), + }, + )) } } diff --git a/lib/src/error.rs b/lib/src/error.rs index c623646..ef5798f 100644 --- a/lib/src/error.rs +++ b/lib/src/error.rs @@ -14,6 +14,7 @@ use std::ops::Range; use crate::ZipTransportError; use camino::Utf8PathBuf; use tough::TransportErrorKind; +use tufaceous_artifact::Artifact; use tufaceous_artifact::DisplayTags; macro_rules! try_path { @@ -170,6 +171,15 @@ pub enum ErrorKind { UrlJoin { source: url::ParseError, url: &'static str, base: String }, #[error("no trust roots provided to load repository")] NoTrustRoots, + #[error( + "artifact ({tags} version={version} hash={hash} length={length}) \ + not found", + tags = .0.display_tags(), + version = .0.version, + hash = .0.hash, + length = .0.length, + )] + ArtifactNotFound(Artifact), #[error("target {target_name} not found")] TargetNotFound { target_name: String }, #[error("failed to parse target {target}")] @@ -238,6 +248,7 @@ impl ErrorKind { | ErrorKind::TargetsBaseUrlUnset | ErrorKind::UrlJoin { .. } | ErrorKind::NoTrustRoots + | ErrorKind::ArtifactNotFound { .. } | ErrorKind::TargetNotFound { .. } | ErrorKind::ParseTargetJson { .. } | ErrorKind::ArtifactVersion(_) @@ -380,9 +391,6 @@ impl Debug for DebugByteString<'_> { #[cfg(test)] mod tests { - use std::error::Error as _; - use std::fmt::Write; - use crate::error::Error; use crate::error::ErrorKind; @@ -392,15 +400,8 @@ mod tests { source: std::io::Error::from(std::io::ErrorKind::NotFound), path: Some("/nowhere/in/particular".into()), }); - let mut chain = err.to_string(); - let mut source = err.source(); - while let Some(err) = source { - write!(chain, ": {err}").unwrap(); - source = err.source(); - } - assert_eq!( - chain, + crate::util::error_chain(&err), "failed to open file /nowhere/in/particular: entity not found" ); } diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 27a1354..0ca62bf 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -2,7 +2,6 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -mod check; pub mod edit; pub mod error; mod loader; @@ -12,7 +11,6 @@ mod util; mod zip_transport; mod zip_writer; -pub use check::*; pub use loader::*; pub use repo::*; pub use zip_transport::*; diff --git a/lib/src/repo.rs b/lib/src/repo.rs index 6928786..ad65b2d 100644 --- a/lib/src/repo.rs +++ b/lib/src/repo.rs @@ -2,6 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +mod check; mod v1; use std::collections::BTreeMap; @@ -14,6 +15,7 @@ use camino::Utf8Path; use camino::Utf8PathBuf; use futures_util::Stream; use futures_util::TryStreamExt; +use rawzip::FileReader; use semver::Version; use serde::de::DeserializeOwned; use slog::Logger; @@ -22,7 +24,6 @@ use tokio::sync::Semaphore; use tokio::sync::TryAcquireError; use tokio::task::JoinSet; use tough::TargetName; -use tough::schema::Hashes; use tough::schema::Target; use tufaceous_artifact::Artifact; use tufaceous_artifact::ArtifactHash; @@ -34,6 +35,7 @@ use tufaceous_artifact::artifact_set::GetError; use crate::RepositoryLoader; use crate::error::Error; use crate::error::ErrorKind; +pub use crate::repo::check::CheckProblem; use crate::schema::ArtifactSchema; use crate::schema::ArtifactSetSchema; @@ -47,8 +49,8 @@ pub struct Repository { system_version: Version, trust_root: Vec, artifacts: ArtifactSet, + artifact_data: BTreeMap, metadata: BTreeMap, - v1_unpacked: Option, // These are set directly by the ZIP archive convenience methods in the // loader module. @@ -116,16 +118,15 @@ impl Repository { read_target_json(&repo, ArtifactSetSchema::TARGET_NAME).await? else { if v1_compatibility - && let Some((system_version, artifacts, v1_unpacked)) = - v1::from_loaded(&repo, log).await? + && let Some(partial) = v1::from_loaded(&repo, log).await? { return Ok(Repository { inner: repo, trust_root, - system_version, - artifacts, + system_version: partial.system_version, + artifacts: partial.artifacts, + artifact_data: partial.artifact_data, metadata: BTreeMap::new(), - v1_unpacked: Some(v1_unpacked), archive_path: None, archive_sha256: None, }); @@ -137,11 +138,16 @@ impl Repository { .into()); }; - let artifacts = artifacts + let (artifacts, artifact_data) = artifacts .into_iter() .filter_map(|ArtifactSchema { target_name, version, tags }| { - let (hash, length) = sha256_length(&repo, log, &target_name)?; - Some(Artifact { target_name, version, tags, hash, length }) + let (hash, length) = + target_meta_skip(&repo, log, &target_name)?; + let artifact = Artifact { version, tags, hash, length }; + Some(( + artifact.clone(), + (artifact, ArtifactData::Target { target_name }), + )) }) .collect(); Ok(Repository { @@ -149,8 +155,8 @@ impl Repository { trust_root, system_version, artifacts, + artifact_data, metadata, - v1_unpacked: None, archive_path: None, archive_sha256: None, }) @@ -188,6 +194,32 @@ impl Repository { &self.artifacts } + /// Returns an [`ArtifactSchema`] for the artifacts in the repository. + /// + /// This is used by [`crate::edit::RepositoryEditor::import_repo`]. + /// + /// # Errors + /// + /// If this was a v1 repository, returns an error; we don't have a valid + /// target name for unpacked artifacts. + pub(crate) fn to_artifact_schema( + &self, + ) -> Result, Error> { + self.artifact_data + .iter() + .map(|(artifact, data)| match data { + ArtifactData::Target { target_name } => Ok(ArtifactSchema { + target_name: target_name.clone(), + version: artifact.version.clone(), + tags: artifact.tags.clone(), + }), + ArtifactData::V1Unpacked { .. } => { + Err(ErrorKind::ImportV1Repo.into()) + } + }) + .collect() + } + pub fn metadata(&self) -> &BTreeMap { &self.metadata } @@ -196,39 +228,44 @@ impl Repository { Metadata::from_map(self.metadata.clone()).ok() } - pub fn is_v1(&self) -> bool { - self.v1_unpacked.is_some() - } - - pub(crate) fn contains_target(&self, target: &str) -> bool { - if let Ok(target_name) = target.parse() - && self.targets().contains_key(&target_name) - { - true - } else if let Some(unpacked) = &self.v1_unpacked - && unpacked.entries.contains_key(target) - { - true - } else { - false + pub async fn read_artifact( + &self, + artifact: &Artifact, + ) -> Result { + let data = self + .artifact_data + .get(artifact) + .ok_or_else(|| ErrorKind::ArtifactNotFound(artifact.clone()))?; + match data { + ArtifactData::Target { target_name } => { + self.read_target(target_name).await + } + ArtifactData::V1Unpacked { file, .. } => { + let unpacked = v1::UnpackedArtifact { + file: file.clone(), + hash: artifact.hash, + length: artifact.length, + }; + Ok(Box::pin(unpacked.stream())) + } } } + /// Read a target from the underlying TUF repository by its target name. + /// + /// If you have an [`Artifact`], use [`Self::read_artifact`]. pub async fn read_target( &self, - target: &str, + target_name: &str, ) -> Result { - if let Some(stream) = read_target(&self.inner, target).await? { - return Ok(Box::pin(stream)); - } - - if let Some(unpacked) = &self.v1_unpacked - && let Some(entry) = unpacked.entries.get(target).cloned() - { - return Ok(Box::pin(entry.stream())); + if let Some(stream) = read_target(&self.inner, target_name).await? { + Ok(Box::pin(stream)) + } else { + Err(ErrorKind::TargetNotFound { + target_name: target_name.to_owned(), + } + .into()) } - - Err(ErrorKind::TargetNotFound { target_name: target.to_owned() }.into()) } /// Returns an [`ArtifactHandle`] for the one (and only one) artifact @@ -325,6 +362,33 @@ impl Repository { } } +/// While we have v1 compatibility, artifacts might come from the underlying +/// `tough::Repository` or from an unpacked file on disk. When we drop all v1 +/// compatibility code we can remove this indirection. +#[derive(Debug, Clone)] +enum ArtifactData { + /// The artifact is read from the underlying `tough::Repository`. + Target { target_name: String }, + /// The artifact was unpacked from a composite artifact in a v1 repository, + /// and is read from a temporary file on disk. + V1Unpacked { + file: Arc, + original_target_name: String, + inner_path: Utf8PathBuf, + }, +} + +impl ArtifactData { + fn original_target_name(&self) -> &str { + match self { + ArtifactData::Target { target_name } => target_name, + ArtifactData::V1Unpacked { original_target_name, .. } => { + original_target_name + } + } + } +} + #[derive(Debug, Clone)] pub struct ArtifactHandle { artifact: Artifact, @@ -341,7 +405,7 @@ impl ArtifactHandle { } pub async fn stream(&self) -> Result { - self.repo.read_target(&self.artifact.target_name).await + self.repo.read_artifact(&self.artifact).await } } @@ -384,43 +448,54 @@ async fn read_target_json( }) } -fn sha256_length( +fn target_meta_inner( + target: &Target, +) -> Result<(ArtifactHash, u64), InvalidTargetError> { + Ok(( + ArtifactHash(target.hashes.sha256.as_ref().try_into().map_err( + |_| InvalidTargetError::ChecksumLength { + sha256: target.hashes.sha256.to_vec(), + }, + )?), + target.length, + )) +} + +fn target_meta( + repo: &tough::Repository, + target_name: &str, +) -> Result<(ArtifactHash, u64), InvalidTargetError> { + let name = TargetName::new(target_name) + .map_err(|_| InvalidTargetError::NameRejected)?; + let Some(target) = repo.targets().signed.targets.get(&name) else { + return Err(InvalidTargetError::NotFound); + }; + target_meta_inner(target) +} + +fn target_meta_skip( repo: &tough::Repository, log: &Logger, target_name: &str, ) -> Option<(ArtifactHash, u64)> { - let parsed_name = TargetName::new(target_name) + target_meta(repo, target_name) .inspect_err(|error| { warn!( log, "skipping artifact"; "target_name" => &target_name, - "error" => error.to_string(), + "error" => crate::util::error_chain(&error), ); }) - .ok()?; - let Some(target) = repo.targets().signed.targets.get(&parsed_name) else { - warn!( - log, - "skipping artifact"; - "target_name" => &target_name, - "error" => "target not found", - ); - return None; - }; - let Hashes { sha256, .. } = &target.hashes; - let sha256 = sha256 - .as_ref() - .try_into() - .inspect_err(|_| { - warn!( - log, - "skipping artifact"; - "target_name" => &target_name, - "error" => "incorrect checksum length", - "sha256" => hex::encode(sha256), - ); - }) - .ok()?; - Some((ArtifactHash(sha256), target.length)) + .ok() +} + +#[derive(Debug, thiserror::Error)] +enum InvalidTargetError { + #[error("target name rejected by tough")] + NameRejected, + #[error("target not found")] + NotFound, + #[error("incorrect sha256 length for {:?}", hex::encode(.sha256))] + ChecksumLength { sha256: Vec }, } diff --git a/lib/src/check.rs b/lib/src/repo/check.rs similarity index 76% rename from lib/src/check.rs rename to lib/src/repo/check.rs index 7ea3026..1adecf7 100644 --- a/lib/src/check.rs +++ b/lib/src/repo/check.rs @@ -17,6 +17,10 @@ use tufaceous_artifact::OsVariant; use tufaceous_artifact::artifact_set::GetError; use crate::Repository; +use crate::repo::ArtifactData; +use crate::repo::InvalidTargetError; +use crate::repo::target_meta; +use crate::repo::target_meta_inner; impl Repository { /// Check the repository for consistency and other problems. @@ -27,21 +31,37 @@ impl Repository { /// must **not** be used for repositories that may have been created by /// another version of Tufaceous, such as when the control plane decides to /// accept a repository. + #[allow(clippy::too_many_lines)] pub async fn check_problems(&self) -> Vec { let mut problems = Vec::new(); - for artifact in self.artifacts() { - // Check that all targets listed in `artifacts-v2.json` actually - // exist in the repository. - if !self.contains_target(&artifact.target_name) { - problems.push(CheckProblem::MissingTarget { - target_name: artifact.target_name.clone(), + for (target_name, target) in self.targets() { + if target_name.raw() != target_name.resolved() { + problems.push(CheckProblem::BadTargetName { + target_name: target_name.raw().to_owned(), }); } + if let Err(error) = target_meta_inner(target) { + problems.push(CheckProblem::from_invalid_target( + error, + target_name.raw().to_owned(), + )); + } + } + + for (artifact, data) in &self.artifact_data { + if let ArtifactData::Target { target_name } = data + && let Err(error) = target_meta(&self.inner, target_name) + { + problems.push(CheckProblem::from_invalid_target( + error, + target_name.to_owned(), + )); + } // Check that no artifact contains unknown tags. if artifact.known_tags().is_none() { problems.push(CheckProblem::UnknownTags { - target_name: artifact.target_name.clone(), + target_name: data.original_target_name().to_owned(), tags: artifact.tags.clone(), }); } @@ -114,7 +134,7 @@ impl Repository { if let Ok(artifact) = self.artifacts().get_only(&KnownArtifactTags::InstallinatorDocument) - && let Ok(stream) = self.read_target(&artifact.target_name).await + && let Ok(stream) = self.read_artifact(artifact).await && let Ok(bytes) = stream.map_ok(Vec::from).try_concat().await && let Ok(doc) = serde_json::from_slice::( &bytes, @@ -163,6 +183,17 @@ pub enum CheckProblem { )] MissingTarget { target_name: String }, + /// A target name in the repository is not well-formed. + #[error("target name {target_name} is not well-formed")] + BadTargetName { target_name: String }, + + /// A SHA-256 checksum in the TUF repository metadata has an invalid length. + #[error( + "target {target_name} has SHA-256 checksum {} with invalid length", + hex::encode(.sha256) + )] + TargetHashLengthMismatch { target_name: String, sha256: Vec }, + /// Multiple artifacts for these tags were not expected. #[error("multiple artifacts found matching {0}")] MultipleArtifacts(KnownArtifactTags), @@ -194,3 +225,20 @@ pub enum CheckProblem { )] UnknownTags { target_name: String, tags: BTreeMap }, } + +impl CheckProblem { + fn from_invalid_target( + source: InvalidTargetError, + target_name: String, + ) -> Self { + match source { + InvalidTargetError::NameRejected => { + Self::BadTargetName { target_name } + } + InvalidTargetError::NotFound => Self::MissingTarget { target_name }, + InvalidTargetError::ChecksumLength { sha256 } => { + Self::TargetHashLengthMismatch { target_name, sha256 } + } + } + } +} diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs index 6b3a6f0..4405500 100644 --- a/lib/src/repo/v1.rs +++ b/lib/src/repo/v1.rs @@ -2,6 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use std::collections::BTreeMap; use std::collections::HashMap; use std::io::BufRead; use std::io::BufReader; @@ -13,6 +14,7 @@ use bytes::Buf; use bytes::Bytes; use bytes::BytesMut; use camino::FromPathBufError; +use camino::Utf8Path; use camino::Utf8PathBuf; use flate2::bufread::GzDecoder; use futures_util::Stream; @@ -53,33 +55,49 @@ use crate::error::DebugByteString; use crate::error::Error; use crate::error::ErrorKind; use crate::error::try_path; +use crate::repo::ArtifactData; use crate::repo::read_target; use crate::repo::read_target_json; use crate::repo::read_target_vec; -use crate::repo::sha256_length; +use crate::repo::target_meta_skip; use crate::util::ArtifactExt; -#[derive(Debug, Clone)] -pub(crate) struct Unpacked { - pub(crate) entries: HashMap, +pub(super) struct PartialRepository { + pub(super) system_version: Version, + pub(super) artifacts: ArtifactSet, + pub(super) artifact_data: BTreeMap, +} + +impl PartialRepository { + fn insert(&mut self, artifact: Artifact, data: ArtifactData) { + self.artifacts.insert(artifact.clone()); + self.artifact_data.insert(artifact, data); + } + + fn original_target_name(&self, artifact: &Artifact) -> Option<&str> { + self.artifact_data.get(artifact).map(ArtifactData::original_target_name) + } } #[expect(clippy::too_many_lines)] pub(crate) async fn from_loaded( repo: &tough::Repository, log: &Logger, -) -> Result, Error> { +) -> Result, Error> { let Some(V1ArtifactSetSchema { system_version, artifacts: v1_artifacts }) = read_target_json(repo, V1ArtifactSetSchema::TARGET_NAME).await? else { return Ok(None); }; - let mut artifacts = ArtifactSet::default(); - let mut unpacked = Unpacked { entries: HashMap::new() }; + let mut partial = PartialRepository { + system_version, + artifacts: ArtifactSet::default(), + artifact_data: BTreeMap::new(), + }; let mut installinator_document = None; for V1Artifact { version, kind, target } in v1_artifacts { - let Some((hash, length)) = sha256_length(repo, log, &target) else { + let Some((hash, length)) = target_meta_skip(repo, log, &target) else { continue; }; let kind = match kind { @@ -135,15 +153,15 @@ pub(crate) async fn from_loaded( &target, KnownArtifactTags::from_rot_bootloader_caboose, )?; - if artifacts.get_all(&tags).iter().any(|artifact| { - if let Some(existing) = - sha256_length(repo, log, &artifact.target_name) - && (hash, length) == existing - { + if partial.artifacts.get_all(&tags).iter().any(|artifact| { + if hash == artifact.hash && length == artifact.length { + let existing = partial + .original_target_name(artifact) + .unwrap_or("???"); info!( log, "skipping duplicate RoT bootloader image"; - "existing" => &artifact.target_name, + "existing" => &existing, "skipped" => &target, ); true @@ -160,15 +178,14 @@ pub(crate) async fn from_loaded( | V1KnownArtifactKind::SwitchRot => { CompositeArtifact::unpack(repo, target) .await? - .read_rot(log, &mut artifacts, &mut unpacked, version) + .read_rot(log, &mut partial, version) .await?; continue; } V1KnownArtifactKind::Host => { CompositeArtifact::unpack(repo, target).await?.read_os_image( - &mut artifacts, - &mut unpacked, + &mut partial, OsVariant::Host, &version, )?; @@ -176,8 +193,7 @@ pub(crate) async fn from_loaded( } V1KnownArtifactKind::Trampoline => { CompositeArtifact::unpack(repo, target).await?.read_os_image( - &mut artifacts, - &mut unpacked, + &mut partial, OsVariant::Recovery, &version, )?; @@ -195,7 +211,7 @@ pub(crate) async fn from_loaded( V1KnownArtifactKind::ControlPlane => { CompositeArtifact::unpack(repo, target) .await? - .read_control_plane(&mut artifacts, &mut unpacked) + .read_control_plane(&mut partial) .await?; continue; } @@ -205,28 +221,24 @@ pub(crate) async fn from_loaded( } }; - let target_name = target; let tags = tags.to_tags().map_err(ErrorKind::ConvertKnownTagsToMap)?; - artifacts.insert(Artifact { target_name, version, tags, hash, length }); + partial.insert( + Artifact { version, tags, hash, length }, + ArtifactData::Target { target_name: target }, + ); } if let Some((version, target)) = installinator_document { - generate_installinator_document( - &mut artifacts, - &mut unpacked, - version, - target, - ) - .await?; + generate_installinator_document(&mut partial, version, target).await?; } - Ok(Some((system_version, artifacts, unpacked))) + Ok(Some(partial)) } #[derive(Debug, Clone)] -pub(crate) struct UnpackedArtifact { - file: Arc, - hash: ArtifactHash, - length: u64, +pub(super) struct UnpackedArtifact { + pub(super) file: Arc, + pub(super) hash: ArtifactHash, + pub(super) length: u64, } impl UnpackedArtifact { @@ -402,8 +414,7 @@ impl CompositeArtifact { async fn read_rot( mut self, log: &Logger, - artifacts: &mut ArtifactSet, - unpacked: &mut Unpacked, + partial: &mut PartialRepository, version: ArtifactVersion, ) -> Result<(), Error> { for slot in [RotSlot::A, RotSlot::B] { @@ -424,21 +435,21 @@ impl CompositeArtifact { }) .await??; - let target_name = format!("{}/{path}", self.original_target_name); - let tags = caboose_tags(image, &target_name, |caboose| { - KnownArtifactTags::from_rot_caboose(caboose, slot) - })?; - if artifacts.get_all(&tags).iter().any(|artifact| { - if let Some(existing) = - unpacked.entries.get(&artifact.target_name) - && hash == existing.hash - && length == existing.length - { + let tags = caboose_tags( + image, + &format!("{}/{path}", self.original_target_name), + |caboose| KnownArtifactTags::from_rot_caboose(caboose, slot), + )?; + if partial.artifacts.get_all(&tags).iter().any(|artifact| { + if hash == artifact.hash && length == artifact.length { + let existing = + partial.original_target_name(artifact).unwrap_or("???"); info!( log, "skipping duplicate RoT image"; - "existing" => &artifact.target_name, - "skipped" => &target_name, + "existing_target" => &existing, + "skipped_target" => &self.original_target_name, + "skipped_inner_file" => &path.as_str(), ); true } else { @@ -447,26 +458,28 @@ impl CompositeArtifact { }) { continue; } - artifacts.insert(Artifact { - target_name: target_name.clone(), - version: version.clone(), - tags: tags - .to_tags() - .map_err(ErrorKind::ConvertKnownTagsToMap)?, - hash, - length, - }); - unpacked - .entries - .insert(target_name, UnpackedArtifact { file, hash, length }); + partial.insert( + Artifact { + version: version.clone(), + tags: tags + .to_tags() + .map_err(ErrorKind::ConvertKnownTagsToMap)?, + hash, + length, + }, + ArtifactData::V1Unpacked { + file, + original_target_name: self.original_target_name.clone(), + inner_path: path, + }, + ); } Ok(()) } fn read_os_image( mut self, - artifacts: &mut ArtifactSet, - unpacked: &mut Unpacked, + partial: &mut PartialRepository, os_variant: OsVariant, version: &ArtifactVersion, ) -> Result<(), Error> { @@ -494,53 +507,57 @@ impl CompositeArtifact { let Some(entry) = self.entries.remove(&path) else { continue; }; - let target_name = format!("{}/{path}", self.original_target_name); - artifacts.insert(Artifact { - target_name: target_name.clone(), - version: version.clone(), - tags: tags - .to_tags() - .map_err(ErrorKind::ConvertKnownTagsToMap)?, - hash: entry.hash, - length: entry.length, - }); - unpacked.entries.insert(target_name, entry); + partial.insert( + Artifact { + version: version.clone(), + tags: tags + .to_tags() + .map_err(ErrorKind::ConvertKnownTagsToMap)?, + hash: entry.hash, + length: entry.length, + }, + ArtifactData::V1Unpacked { + file: entry.file, + original_target_name: self.original_target_name.clone(), + inner_path: path, + }, + ); } Ok(()) } async fn read_control_plane( self, - artifacts: &mut ArtifactSet, - unpacked: &mut Unpacked, + partial: &mut PartialRepository, ) -> Result<(), Error> { for (tar_path, UnpackedArtifact { file, hash, length }) in self.entries { if !tar_path.starts_with("zones/") { continue; } - let target_name = - format!("{}/{tar_path}", self.original_target_name); let (file, layer_info) = crate::util::read_zone_layer_info( RangeReader::new(file, 0..length), - target_name.clone().into(), + Utf8Path::new(&self.original_target_name).join(&tar_path), ) .await?; let file = file.into_inner(); let tags = KnownArtifactTags::Zone(ZoneTags { zone_name: layer_info.pkg }); - artifacts.insert(Artifact { - target_name: target_name.clone(), - version: layer_info.version, - tags: tags - .to_tags() - .map_err(ErrorKind::ConvertKnownTagsToMap)?, - hash, - length, - }); - unpacked - .entries - .insert(target_name, UnpackedArtifact { file, hash, length }); + partial.insert( + Artifact { + version: layer_info.version, + tags: tags + .to_tags() + .map_err(ErrorKind::ConvertKnownTagsToMap)?, + hash, + length, + }, + ArtifactData::V1Unpacked { + file, + original_target_name: self.original_target_name.clone(), + inner_path: tar_path, + }, + ); } Ok(()) } @@ -582,15 +599,20 @@ impl BufRead for MpscReader { } async fn generate_installinator_document( - artifacts: &mut ArtifactSet, - unpacked: &mut Unpacked, + partial: &mut PartialRepository, version: ArtifactVersion, - original_target: String, + original_target_name: String, ) -> Result<(), Error> { - let target_name = format!("{original_target}/v2.json"); let mut document = InstallinatorDocument::empty(version.clone()); - document.artifacts = - artifacts.iter().filter_map(Artifact::to_installinator).collect(); + for (artifact, data) in &partial.artifact_data { + let target_name = match data { + ArtifactData::Target { target_name } => target_name, + ArtifactData::V1Unpacked { inner_path, .. } => inner_path.as_str(), + }; + if let Some(installinator) = artifact.to_installinator(target_name) { + document.artifacts.insert(installinator); + } + } let mut json = serde_json::to_string_pretty(&document) .map_err(ErrorKind::SerializeInstallinator)?; @@ -602,16 +624,21 @@ async fn generate_installinator_document( }) .await??; - artifacts.insert(Artifact { - target_name: target_name.clone(), - version, - tags: KnownArtifactTags::InstallinatorDocument - .to_tags() - .map_err(ErrorKind::ConvertKnownTagsToMap)?, - hash: unpacked_artifact.hash, - length: unpacked_artifact.length, - }); - unpacked.entries.insert(target_name, unpacked_artifact); + partial.insert( + Artifact { + version, + tags: KnownArtifactTags::InstallinatorDocument + .to_tags() + .map_err(ErrorKind::ConvertKnownTagsToMap)?, + hash: unpacked_artifact.hash, + length: unpacked_artifact.length, + }, + ArtifactData::V1Unpacked { + file: unpacked_artifact.file, + original_target_name, + inner_path: "v2.json".into(), + }, + ); Ok(()) } diff --git a/lib/src/util.rs b/lib/src/util.rs index 4f140f2..97a7146 100644 --- a/lib/src/util.rs +++ b/lib/src/util.rs @@ -25,12 +25,18 @@ use crate::error::ErrorKind; use crate::error::try_path; pub(crate) trait ArtifactExt { - fn to_installinator(&self) -> Option; + fn to_installinator( + &self, + target_name: &str, + ) -> Option; } impl ArtifactExt for Artifact { - fn to_installinator(&self) -> Option { - installinator_artifact(self.tags.clone(), self.hash, &self.target_name) + fn to_installinator( + &self, + target_name: &str, + ) -> Option { + installinator_artifact(self.tags.clone(), self.hash, target_name) } } @@ -115,3 +121,13 @@ impl DirEntry { .expect("path created through DirEntry must have a file name") } } + +pub(crate) fn error_chain(mut error: &dyn std::error::Error) -> String { + let mut s = error.to_string(); + while let Some(source) = error.source() { + s.push_str(": "); + s.push_str(&source.to_string()); + error = source; + } + s +} diff --git a/lib/tests/tests.rs b/lib/tests/tests.rs index f0f3bca..3f9a4d6 100644 --- a/lib/tests/tests.rs +++ b/lib/tests/tests.rs @@ -37,7 +37,7 @@ async fn it_works() -> Result<(), Error> { .load_zip_buffer(zip, &log) .await?; for artifact in repo.artifacts() { - repo.read_target(&artifact.target_name) + repo.read_artifact(artifact) .await? .map_ok(|bytes| bytes.to_vec()) .try_concat() @@ -127,7 +127,7 @@ async fn empty_artifact() -> Result<(), Error> { let artifacts = repo.artifacts().iter().collect::>(); assert_eq!(artifacts.len(), 1); let data = repo - .read_target(&artifacts[0].target_name) + .read_artifact(artifacts[0]) .await? .map_ok(|bytes| bytes.to_vec()) .try_concat() From 38ebade980fd1f89613c7185b600cdf7d08825a4 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 11 Jun 2026 16:14:09 -0700 Subject: [PATCH 083/137] fix doc anchors on ZipTransport --- lib/src/zip_transport.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/src/zip_transport.rs b/lib/src/zip_transport.rs index c99ad4a..5c52879 100644 --- a/lib/src/zip_transport.rs +++ b/lib/src/zip_transport.rs @@ -106,11 +106,11 @@ const EOCD_MAX_SEARCH_SPACE: u64 = 22; /// /// Convenience methods for setting the correct transport /// and base URLs for loading Tufaceous-generated -/// repositories are [`RepositoryLoader::load_zip_slice`] and +/// repositories are [`RepositoryLoader::load_zip_buffer`] and /// [`RepositoryLoader::load_zip_file`]. /// -/// [`RepositoryLoader::load_zip_slice`]: [`crate::RepositoryLoader::load_zip_slice`] -/// [`RepositoryLoader::load_zip_file`]: [`crate::RepositoryLoader::load_zip_file`] +/// [`RepositoryLoader::load_zip_buffer`]: crate::RepositoryLoader::load_zip_buffer +/// [`RepositoryLoader::load_zip_file`]: crate::RepositoryLoader::load_zip_file #[derive(Debug)] pub struct ZipTransport { inner: Arc>, From 14f1fcb898a00c01758c93e2e380178730418f9b Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 11 Jun 2026 16:29:30 -0700 Subject: [PATCH 084/137] macro (and documentation) for "usize fits in u64" --- lib/src/edit/source.rs | 11 ++++------- lib/src/lib.rs | 16 ++++++++++++++++ lib/src/repo/v1.rs | 5 ++--- lib/src/zip_transport.rs | 3 +-- 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/lib/src/edit/source.rs b/lib/src/edit/source.rs index fe05698..a9c9abb 100644 --- a/lib/src/edit/source.rs +++ b/lib/src/edit/source.rs @@ -116,7 +116,7 @@ impl BytesSource { if let Ok(length) = usize::try_from(length) { bytes.truncate(length); } - length - u64::try_from(bytes.len()).expect("usize fits in u64") + length - usize64!(bytes.len()) } None => 0, }; @@ -140,9 +140,7 @@ impl BytesSource { } pub(crate) fn length(&self) -> u64 { - self.fake_length.unwrap_or_else(|| { - self.bytes.len().try_into().expect("usize fits in u64") - }) + self.fake_length.unwrap_or_else(|| usize64!(self.bytes.len())) } async fn sha256(&mut self) -> ArtifactHash { @@ -221,8 +219,7 @@ impl FileSource { .try_fold( (0u64, Sha256::new()), |(mut length, mut hasher), bytes| { - length += - u64::try_from(bytes.len()).expect("usize fits in u64"); + length += usize64!(bytes.len()); hasher.update(&bytes); if let Some(vec) = vec.as_mut() { vec.extend_from_slice(&bytes); @@ -293,7 +290,7 @@ impl FileSource { if n == 0 { return Ok(None); } - offset += u64::try_from(n).expect("usize fits in u64"); + offset += usize64!(n); buf.truncate(n); Ok(Some((buf.split().freeze(), (inner, buf, offset)))) }) diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 0ca62bf..bc6e2a4 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -2,6 +2,22 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +/// This library very commonly needs to convert from usize (Rust's standard +/// length type) to u64 (standard for file sizes in operating system APIs). This +/// is fallible so that Rust can be adapted to systems with pointer sizes larger +/// than 64 bits. In 2026 these systems do not appear to exist nor appear to be +/// on the horizon. +/// +/// This is a shorthand to assert that a value is a usize, then convert and +/// panic on failure; the failure is impossible and elided on 64-bit or lower +/// systems. +macro_rules! usize64 { + ($usize:expr) => {{ + let x: usize = $usize; + u64::try_from(x).expect("usize fits in u64") + }}; +} + pub mod edit; pub mod error; mod loader; diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs index 4405500..5052c45 100644 --- a/lib/src/repo/v1.rs +++ b/lib/src/repo/v1.rs @@ -265,7 +265,7 @@ impl UnpackedArtifact { })?; hasher.update(buf); reader.consume(len); - length += u64::try_from(len).expect("usize fits in u64"); + length += usize64!(len); } let file = Arc::new(file.into()); let hash = ArtifactHash(hasher.finalize().0); @@ -309,8 +309,7 @@ impl UnpackedArtifact { ); } hasher.update(&bytes); - bytes_read += - u64::try_from(bytes.len()).expect("usize fits in u64"); + bytes_read += usize64!(bytes.len()); Ok(Some((bytes, (this, buf, hasher, bytes_read)))) }, ) diff --git a/lib/src/zip_transport.rs b/lib/src/zip_transport.rs index 5c52879..b7c6af7 100644 --- a/lib/src/zip_transport.rs +++ b/lib/src/zip_transport.rs @@ -195,8 +195,7 @@ impl ZipTransport { // First check: the number of entries in the central directory matches // the value in the EOCD record. - let actual = - u64::try_from(all_entries.len()).expect("usize fits in u64"); + let actual = usize64!(all_entries.len()); if archive.entries_hint() != actual { return Err(ErrorKind::ZipEntryCount { expected: archive.entries_hint(), From 7f4168f90dba34376e32d9870530dd979b446b72 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 11 Jun 2026 16:29:46 -0700 Subject: [PATCH 085/137] bail early if more zip entries than EOCD claims --- lib/src/error.rs | 16 ++++++++++++++-- lib/src/zip_transport.rs | 11 ++++++++++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/lib/src/error.rs b/lib/src/error.rs index ef5798f..5755293 100644 --- a/lib/src/error.rs +++ b/lib/src/error.rs @@ -72,11 +72,12 @@ pub enum ErrorKind { #[error( "zip archive{archive_path}'s end of central directory record \ expects {expected} entries, but found {actual} entries", - archive_path = SpacePath(archive_path) + archive_path = SpacePath(archive_path), + actual = OrMore(actual), )] ZipEntryCount { expected: u64, - actual: u64, + actual: Option, archive_path: Option, }, #[error( @@ -377,6 +378,17 @@ impl Display for SpacePath<'_> { } } +struct OrMore<'a>(&'a Option); + +impl Display for OrMore<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.0 { + Some(value) => write!(f, "{value}"), + None => write!(f, "more"), + } + } +} + pub(crate) struct DebugByteString<'a>(pub(crate) &'a [u8]); impl Debug for DebugByteString<'_> { diff --git a/lib/src/zip_transport.rs b/lib/src/zip_transport.rs index b7c6af7..f0c9979 100644 --- a/lib/src/zip_transport.rs +++ b/lib/src/zip_transport.rs @@ -184,6 +184,15 @@ impl ZipTransport { while let Some(record) = try_archive_path!(records.next_entry(), ReadZip, archive_path) { + if usize64!(all_entries.len()) >= archive.entries_hint() { + return Err(ErrorKind::ZipEntryCount { + expected: archive.entries_hint(), + actual: None, + archive_path, + } + .into()); + } + all_entries.push(RawEntry::from(&record)); if let Some((url, entry)) = Entry::new(&record, log) { entries @@ -199,7 +208,7 @@ impl ZipTransport { if archive.entries_hint() != actual { return Err(ErrorKind::ZipEntryCount { expected: archive.entries_hint(), - actual, + actual: Some(actual), archive_path, } .into()); From be1d9438dcb882b5dd4b88d79968727424a6bcba Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 11 Jun 2026 16:37:08 -0700 Subject: [PATCH 086/137] lift spawn_blocking out of ZipTransport::from_file --- lib/src/loader.rs | 33 ++++++++++++++++++++------------- lib/src/zip_transport.rs | 25 ++++++++++--------------- 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/lib/src/loader.rs b/lib/src/loader.rs index 0c0ef1d..dd2f36e 100644 --- a/lib/src/loader.rs +++ b/lib/src/loader.rs @@ -188,25 +188,32 @@ impl RepositoryLoader { archive_path: Option, log: &Logger, ) -> Result { - let (file, sha256) = if self.compute_archive_sha256 { + let (sha256, transport) = { let archive_path = archive_path.clone(); + let log = log.clone(); + let compute_archive_sha256 = self.compute_archive_sha256; tokio::task::spawn_blocking(move || { - let mut hasher = IoWrapper(Sha256::new()); - try_path!( - file.rewind() - .and_then(|()| std::io::copy(&mut file, &mut hasher)), - ReadFile, - archive_path - ); - Ok::<_, Error>((file, Some(hasher.0.finalize().0))) + let sha256 = if compute_archive_sha256 { + let mut hasher = IoWrapper(Sha256::new()); + try_path!( + file.rewind().and_then(|()| std::io::copy( + &mut file, + &mut hasher + )), + ReadFile, + archive_path + ); + Some(hasher.0.finalize().0) + } else { + None + }; + let transport = + ZipTransport::from_file_blocking(file, archive_path, &log)?; + Ok::<_, Error>((sha256, transport)) }) .await?? - } else { - (file, None) }; - let transport = - ZipTransport::from_file(file, archive_path.clone(), log).await?; let mut repo = self.zip_base_urls().load(transport, log).await?; repo.archive_path = archive_path; repo.archive_sha256 = sha256; diff --git a/lib/src/zip_transport.rs b/lib/src/zip_transport.rs index f0c9979..df71109 100644 --- a/lib/src/zip_transport.rs +++ b/lib/src/zip_transport.rs @@ -142,25 +142,20 @@ impl + Debug + Send + Sync + 'static> ZipTransport> { } impl ZipTransport { - pub async fn from_file( + pub fn from_file_blocking( file: File, archive_path: Option, log: &Logger, ) -> Result { - let log = log.clone(); - tokio::task::spawn_blocking(move || { - let archive_path = archive_path; - let mut buffer = vec![0; rawzip::RECOMMENDED_BUFFER_SIZE]; - let archive = try_archive_path!( - ZipArchive::with_max_search_space(EOCD_MAX_SEARCH_SPACE) - .locate_in_file(file, &mut buffer) - .map_err(|(_, error)| error), - ReadZip, - archive_path - ); - Self::from_impl_blocking(archive, archive_path, Some(buffer), &log) - }) - .await? + let mut buffer = vec![0; rawzip::RECOMMENDED_BUFFER_SIZE]; + let archive = try_archive_path!( + ZipArchive::with_max_search_space(EOCD_MAX_SEARCH_SPACE) + .locate_in_file(file, &mut buffer) + .map_err(|(_, error)| error), + ReadZip, + archive_path + ); + Self::from_impl_blocking(archive, archive_path, Some(buffer), log) } } From 0268a3cf324ae4fd3fce02581896f59474a7f560 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 11 Jun 2026 20:53:21 -0700 Subject: [PATCH 087/137] doc comments for tufaceous_artifact::map --- artifact/src/map.rs | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/artifact/src/map.rs b/artifact/src/map.rs index 994bd56..2cb43db 100644 --- a/artifact/src/map.rs +++ b/artifact/src/map.rs @@ -2,21 +2,47 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +//! Utilities for mappings of string keys to string values +//! (`BTreeMap`). +//! +//! Tufaceous uses these string mappings for future-proofing artifact-level +//! metadata ("tags") and repository-level metadata, so that all of the +//! repository information can be losslessly entered into a database even if an +//! older version of Tufaceous is used to read a repository. + +// TODO: This (as of writing) is the only use of serde_json in the entire +// tufaceous-artifact crate, and it's not even for writing/parsing JSON. +// This is "unfortunate but reasonable": anything we expect to be using +// tufaceous-artifact is highly likely to use serde_json anyway. + use std::collections::BTreeMap; +use serde::Deserialize; use serde::Serialize; use serde::de::DeserializeOwned; +/// Creates a struct `D` from a string mapping, round-tripping via +/// `serde_json::Value`. +/// +/// This could someday be implemented without round-tripping through +/// `serde_json::Value` but implementing a proper deserializer is a lot of +/// effort that, as of writing this comment, is out of scope. pub(crate) fn from_map( map: BTreeMap, ) -> Result { - serde_json::from_value(map.into_iter().collect()) + D::deserialize(map.into_iter().collect::()) } +/// Creates a string mapping from a struct `S`, round-tripping via +/// `serde_json::Value`. +/// +/// This could someday be implemented without round-tripping through +/// `serde_json::Value` but implementing a proper serializer is a lot of effort +/// that, as of writing this comment, is out of scope. pub(crate) fn to_map( s: &S, ) -> Result, serde_json::Error> { - serde_json::from_value(serde_json::to_value(s)?) + BTreeMap::deserialize(serde_json::to_value(s)?) } #[cfg(test)] From 41a264fbeb8200a522810f4b5ef47cb37bf8f6c8 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 11 Jun 2026 20:55:17 -0700 Subject: [PATCH 088/137] fix comment note on Metadata --- artifact/src/metadata.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/artifact/src/metadata.rs b/artifact/src/metadata.rs index 2dc21e1..ef20b62 100644 --- a/artifact/src/metadata.rs +++ b/artifact/src/metadata.rs @@ -11,8 +11,8 @@ use serde::Serialize; // // NOTE: Similar to KnownArtifactTags, this struct must serialize to // and deserialize from a mapping of string keys to string values. The -// `metadata_roundtrip` test covers this (crate::map::to_map panics when debug -// assertions are enabled if this does not hold). +// `metadata_roundtrip` test covers this (crate::map::to_map returns an error if +// this does not hold). // // Additionally, it is a requirement that all fields are optional; it must // be possible to deserialize this struct from older metadata, and the oldest From 7a7e03db4aa7679c268b3575a2bc089102cd98fe Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 11 Jun 2026 21:18:12 -0700 Subject: [PATCH 089/137] move Option out of RotKeyTableHash --- artifact/src/hubris.rs | 10 ++++++---- artifact/src/rkth.rs | 31 ++++++++++++++++++------------- artifact/src/tags.rs | 12 ++++++------ lib/src/edit/fake.rs | 4 ++-- lib/src/edit/hubris_archive.rs | 8 ++++---- lib/src/edit/input.rs | 11 +++++++++-- lib/tests/v1_compatibility.rs | 4 ++-- 7 files changed, 47 insertions(+), 33 deletions(-) diff --git a/artifact/src/hubris.rs b/artifact/src/hubris.rs index d49cacb..9750d81 100644 --- a/artifact/src/hubris.rs +++ b/artifact/src/hubris.rs @@ -29,7 +29,7 @@ impl RotTags { ) -> Result { Ok(Self { rot_board: read_board(caboose)?, - rot_rkth: RotKeyTableHash(read_sign(caboose)?), + rot_rkth: read_sign(caboose)?, rot_slot: slot, }) } @@ -46,7 +46,7 @@ impl RotBootloaderTags { pub fn from_caboose(caboose: &Caboose) -> Result { Ok(Self { rot_board: read_board(caboose)?, - rot_rkth: RotKeyTableHash(read_sign(caboose)?), + rot_rkth: read_sign(caboose)?, }) } } @@ -120,9 +120,11 @@ fn read_board(caboose: &Caboose) -> Result { Ok(utf8(caboose.board()?, "BORD")?.to_owned()) } -fn read_sign(caboose: &Caboose) -> Result, ReadCabooseError> { +fn read_sign( + caboose: &Caboose, +) -> Result, ReadCabooseError> { match caboose.sign() { - Ok(sign) => Ok(Some(utf8(sign, "SIGN")?.to_owned())), + Ok(sign) => Ok(Some(RotKeyTableHash::new(utf8(sign, "SIGN")?))), Err(CabooseError::MissingTag { .. }) => Ok(None), Err(error) => Err(error.into()), } diff --git a/artifact/src/rkth.rs b/artifact/src/rkth.rs index 602a484..5c86cd3 100644 --- a/artifact/src/rkth.rs +++ b/artifact/src/rkth.rs @@ -66,23 +66,34 @@ static CA_MAP: LazyLock> = )] #[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] #[serde(transparent)] -pub struct RotKeyTableHash(pub Option); +pub struct RotKeyTableHash(pub String); impl RotKeyTableHash { + /// Create a `RotKeyTableHash` from a string. + pub fn new(s: impl Into) -> Self { + Self(s.into()) + } + + /// Returns a string slice of the underlying string. + pub fn as_str(&self) -> &str { + &self.0 + } + /// Create a `RotKeyTableHash` from an array of 32 bytes. pub fn from_bytes(hash: [u8; 32]) -> Self { - Self(Some(HexArray(hash).to_string())) + Self(HexArray(hash).to_string()) } /// Returns a friendly name for the CA this RKTH represents, if one is /// known. Returns `None` otherwise. pub fn friendly_ca_name(&self) -> Option<&'static str> { - let inner = self.0.as_deref()?; - CA_MAP.get(inner).copied() + CA_MAP.get(self.as_str()).copied() } +} - pub(crate) fn is_none(&self) -> bool { - self.0.is_none() +impl AsRef for RotKeyTableHash { + fn as_ref(&self) -> &str { + self.as_str() } } @@ -94,13 +105,7 @@ impl Debug for RotKeyTableHash { impl Display for RotKeyTableHash { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self.0.as_deref() { - Some(inner) => { - let s = CA_MAP.get(inner).copied().unwrap_or(inner); - write!(f, "{s}") - } - None => write!(f, "unsigned"), - } + write!(f, "{}", self.friendly_ca_name().unwrap_or(self.as_str())) } } diff --git a/artifact/src/tags.rs b/artifact/src/tags.rs index fe0e5cb..84aaf86 100644 --- a/artifact/src/tags.rs +++ b/artifact/src/tags.rs @@ -230,8 +230,8 @@ pub struct RotTags { /// For unsigned images this will not be present; this will generally /// never occur in release repos but can be useful on hardware that has /// not fully made it through manufacturing yet. - #[serde(skip_serializing_if = "RotKeyTableHash::is_none")] - pub rot_rkth: RotKeyTableHash, + #[serde(skip_serializing_if = "Option::is_none")] + pub rot_rkth: Option, /// RoT images are compiled for two different locations in flash; this /// identifies which slot this image belongs to. pub rot_slot: RotSlot, @@ -280,8 +280,8 @@ pub struct RotBootloaderTags { /// For unsigned images this will not be present; this will generally /// never occur in release repos but can be useful on hardware that has /// not fully made it through manufacturing yet. - #[serde(skip_serializing_if = "RotKeyTableHash::is_none")] - pub rot_rkth: RotKeyTableHash, + #[serde(skip_serializing_if = "Option::is_none")] + pub rot_rkth: Option, } impl From for KnownArtifactTags { @@ -402,7 +402,7 @@ mod tests { KnownArtifactTags::from_tags(tags.clone()).unwrap(), KnownArtifactTags::Rot(RotTags { rot_board: "oxide-rot-1".to_owned(), - rot_rkth: RotKeyTableHash(None), + rot_rkth: None, rot_slot: RotSlot::A }) ); @@ -411,7 +411,7 @@ mod tests { KnownArtifactTags::from_tags(tags).unwrap(), KnownArtifactTags::Rot(RotTags { rot_board: "oxide-rot-1".to_owned(), - rot_rkth: RotKeyTableHash(Some("meow".to_owned())), + rot_rkth: Some(RotKeyTableHash::new("meow")), rot_slot: RotSlot::A }) ); diff --git a/lib/src/edit/fake.rs b/lib/src/edit/fake.rs index 1bee715..bdc7d53 100644 --- a/lib/src/edit/fake.rs +++ b/lib/src/edit/fake.rs @@ -92,7 +92,7 @@ impl Input { inputs.push(Self::fake_rot_archive( RotTags { rot_board: "SimRot".into(), - rot_rkth: RotKeyTableHash(Some(FAKE_SIGN.into())), + rot_rkth: Some(RotKeyTableHash::new(FAKE_SIGN)), rot_slot: slot, }, version.clone(), @@ -102,7 +102,7 @@ impl Input { inputs.push(Self::fake_rot_bootloader_archive( RotBootloaderTags { rot_board: "SimRot".into(), - rot_rkth: RotKeyTableHash(Some(FAKE_SIGN.into())), + rot_rkth: Some(RotKeyTableHash::new(FAKE_SIGN)), }, version.clone(), interior_version, diff --git a/lib/src/edit/hubris_archive.rs b/lib/src/edit/hubris_archive.rs index 8ed6aa9..4d9e59c 100644 --- a/lib/src/edit/hubris_archive.rs +++ b/lib/src/edit/hubris_archive.rs @@ -179,7 +179,7 @@ impl Input { ) -> Result { let data = CabooseData { board: &tags.sp_board, - sign: &RotKeyTableHash(None), + sign: &None, commit: "this-is-a-fake-sp", version: interior_version.unwrap_or(&version), }; @@ -190,7 +190,7 @@ impl Input { struct CabooseData<'a> { board: &'a str, - sign: &'a RotKeyTableHash, + sign: &'a Option, commit: &'static str, version: &'a ArtifactVersion, } @@ -202,8 +202,8 @@ impl CabooseData<'_> { .name(self.board) .git_commit(self.commit) .version(self.version.to_string()); - if let Some(sign) = self.sign.0.as_ref() { - builder = builder.sign(sign); + if let Some(sign) = self.sign { + builder = builder.sign(sign.as_str()); } let caboose = builder.build(); diff --git a/lib/src/edit/input.rs b/lib/src/edit/input.rs index e44fd78..fd8bbe9 100644 --- a/lib/src/edit/input.rs +++ b/lib/src/edit/input.rs @@ -16,6 +16,7 @@ use tufaceous_artifact::OsPhase1Tags; use tufaceous_artifact::OsPhase2Tags; use tufaceous_artifact::OsVariant; use tufaceous_artifact::RotBootloaderTags; +use tufaceous_artifact::RotKeyTableHash; use tufaceous_artifact::RotTags; use tufaceous_artifact::SpTags; use tufaceous_artifact::ZoneTags; @@ -119,7 +120,10 @@ impl Input { let target_name = format!( "rot/{board}-{sign}-{version}-slot-{slot}.zip", board = tags.rot_board, - sign = tags.rot_rkth, + sign = tags + .rot_rkth + .as_ref() + .map_or("unsigned", RotKeyTableHash::as_str), slot = tags.rot_slot ); vec![Output::new(target_name, version, &tags.into(), source)?] @@ -128,7 +132,10 @@ impl Input { let target_name = format!( "rot-bootloader/{board}-{sign}-{version}.zip", board = tags.rot_board, - sign = tags.rot_rkth + sign = tags + .rot_rkth + .as_ref() + .map_or("unsigned", RotKeyTableHash::as_str), ); vec![Output::new(target_name, version, &tags.into(), source)?] } diff --git a/lib/tests/v1_compatibility.rs b/lib/tests/v1_compatibility.rs index bc8f5f3..865ad33 100644 --- a/lib/tests/v1_compatibility.rs +++ b/lib/tests/v1_compatibility.rs @@ -51,7 +51,7 @@ async fn v1_fake() -> Result<(), Error> { expected.push( RotTags { rot_board: "SimRot".into(), - rot_rkth: RotKeyTableHash(Some(rot_rkth.into())), + rot_rkth: Some(RotKeyTableHash::new(rot_rkth)), rot_slot, } .into(), @@ -60,7 +60,7 @@ async fn v1_fake() -> Result<(), Error> { expected.push( RotBootloaderTags { rot_board: "SimRot".into(), - rot_rkth: RotKeyTableHash(Some(rot_rkth.into())), + rot_rkth: Some(RotKeyTableHash::new(rot_rkth)), } .into(), ); From 4695ec1bf42802f6333626d92ef2716fc3b98e88 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 11 Jun 2026 21:25:19 -0700 Subject: [PATCH 090/137] make RepositoryEditor::new fallible --- bin/src/build.rs | 4 ++-- lib/src/edit/editor.rs | 45 ++++++++++++++++++------------------------ lib/tests/tests.rs | 4 ++-- 3 files changed, 23 insertions(+), 30 deletions(-) diff --git a/bin/src/build.rs b/bin/src/build.rs index 73a804f..7382203 100644 --- a/bin/src/build.rs +++ b/bin/src/build.rs @@ -29,10 +29,10 @@ pub struct Args { impl Args { pub async fn run(self) -> Result<()> { let mut editor = if self.fake { - RepositoryEditor::fake(self.version)? + RepositoryEditor::fake(self.version) } else { RepositoryEditor::new(self.version) - }; + }?; editor = editor .generate_installinator_document(!self.no_installinator_document); for path in self.artifacts { diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index 3da94fa..707fd55 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -14,7 +14,6 @@ use semver::Version; use tokio::task::JoinSet; use tufaceous_artifact::ArtifactHash; use tufaceous_artifact::ArtifactVersion; -use tufaceous_artifact::ArtifactVersionError; use tufaceous_artifact::InstallinatorDocument; use tufaceous_artifact::KnownArtifactTags; use tufaceous_artifact::Metadata; @@ -42,7 +41,7 @@ use crate::schema::ArtifactSetSchema; #[must_use] pub struct RepositoryEditor<'a> { system_version: Version, - artifact_version: Result, + artifact_version: ArtifactVersion, generate_installinator_document: bool, targets: HashMap>>, artifacts: HashMap>, @@ -51,15 +50,15 @@ pub struct RepositoryEditor<'a> { impl<'a> RepositoryEditor<'a> { /// Create an empty repository editor. - pub fn new(system_version: Version) -> Self { - Self { - artifact_version: ArtifactVersion::new(system_version.to_string()), + pub fn new(system_version: Version) -> Result { + Ok(Self { + artifact_version: ArtifactVersion::new(system_version.to_string())?, system_version, generate_installinator_document: true, targets: HashMap::new(), artifacts: HashMap::new(), metadata: BTreeMap::new(), - } + }) } /// Change the system version of the repository. @@ -117,7 +116,7 @@ impl<'a> RepositoryEditor<'a> { variant, output_dir, None, - self.artifact_version.clone()?, + self.artifact_version.clone(), ) .await?; self.insert_input(input) @@ -125,11 +124,8 @@ impl<'a> RepositoryEditor<'a> { /// Add a fake OS image to the repository. pub fn fake_os_image(self, variant: OsVariant) -> Result { - let input = Input::fake_os_images( - variant, - self.artifact_version.clone()?, - None, - ); + let input = + Input::fake_os_images(variant, self.artifact_version.clone(), None); self.insert_input(input) } @@ -169,11 +165,8 @@ impl<'a> RepositoryEditor<'a> { /// /// This will generate a fake Hubris archive with the appropriate tags. pub fn fake_rot_archive(self, tags: RotTags) -> Result { - let input = Input::fake_rot_archive( - tags, - self.artifact_version.clone()?, - None, - )?; + let input = + Input::fake_rot_archive(tags, self.artifact_version.clone(), None)?; self.insert_input(input) } @@ -186,7 +179,7 @@ impl<'a> RepositoryEditor<'a> { ) -> Result { let input = Input::fake_rot_bootloader_archive( tags, - self.artifact_version.clone()?, + self.artifact_version.clone(), None, )?; self.insert_input(input) @@ -197,7 +190,7 @@ impl<'a> RepositoryEditor<'a> { /// This will generate a fake Hubris archive with the appropriate tags. pub fn fake_sp_archive(self, tags: SpTags) -> Result { let input = - Input::fake_sp_archive(tags, self.artifact_version.clone()?, None)?; + Input::fake_sp_archive(tags, self.artifact_version.clone(), None)?; self.insert_input(input) } @@ -221,7 +214,7 @@ impl<'a> RepositoryEditor<'a> { let input = Input::fake_zone_image( zone_name, file_name, - self.artifact_version.clone()?, + self.artifact_version.clone(), None, )?; self.insert_input(input) @@ -236,7 +229,7 @@ impl<'a> RepositoryEditor<'a> { self, path: Utf8PathBuf, ) -> Result { - let input = Input::guess(path, self.artifact_version.clone()?).await?; + let input = Input::guess(path, self.artifact_version.clone()).await?; self.insert_input(input) } @@ -329,8 +322,8 @@ impl<'a> RepositoryEditor<'a> { /// Create a fake repository for testing purposes. pub fn fake(system_version: Version) -> Result { - let mut editor = Self::new(system_version); - let version = editor.artifact_version.clone()?; + let mut editor = Self::new(system_version)?; + let version = editor.artifact_version.clone(); for input in Input::fake(&version, None)? { editor = editor.insert_input(input)?; } @@ -348,7 +341,7 @@ impl<'a> RepositoryEditor<'a> { artifact_version: &ArtifactVersion, interior_version: &ArtifactVersion, ) -> Result { - let mut editor = Self::new(system_version); + let mut editor = Self::new(system_version)?; for input in Input::fake(artifact_version, Some(interior_version))? { editor = editor.insert_input(input)?; } @@ -367,7 +360,7 @@ impl<'a> RepositoryEditor<'a> { /// This creates an editor with references to all of the artifacts and /// targets in the original repository. pub fn from_repo(repo: &'a Repository) -> Result { - Self::new(repo.system_version().clone()).import_repo(repo) + Self::new(repo.system_version().clone())?.import_repo(repo) } /// Import all of the artifacts and targets from a repository into this @@ -480,7 +473,7 @@ impl<'a> RepositoryEditor<'a> { let target = targets.0.get(&artifact.target_name)?; Some((artifact, target.sha256.as_slice())) }), - self.artifact_version.clone()?, + self.artifact_version.clone(), )?; if let Some(artifact) = output.to_artifact_schema() { if let Some(existing) = artifacts.get(&artifact.target_name) { diff --git a/lib/tests/tests.rs b/lib/tests/tests.rs index 3f9a4d6..cd43c24 100644 --- a/lib/tests/tests.rs +++ b/lib/tests/tests.rs @@ -84,7 +84,7 @@ async fn verify_targets() -> Result<(), Error> { #[tokio::test] async fn no_artifacts() -> Result<(), Error> { let log = slog::Logger::root(slog::Discard, slog::o!()); - let zip = RepositoryEditor::new(V1) + let zip = RepositoryEditor::new(V1)? .generate_installinator_document(false) .finish() .await? @@ -105,7 +105,7 @@ async fn no_artifacts() -> Result<(), Error> { #[tokio::test] async fn empty_artifact() -> Result<(), Error> { let log = slog::Logger::root(slog::Discard, slog::o!()); - let zip = RepositoryEditor::new(V1) + let zip = RepositoryEditor::new(V1)? .generate_installinator_document(false) .fake_artifact( "empty.img".to_owned(), From e3594cee7bab7dda6a382a5c396844dd1fed8689 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 11 Jun 2026 21:27:52 -0700 Subject: [PATCH 091/137] also set artifact_version in RepositoryEditor::system_version --- bin/src/edit.rs | 2 +- lib/src/edit/editor.rs | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/bin/src/edit.rs b/bin/src/edit.rs index 4cee034..498732e 100644 --- a/bin/src/edit.rs +++ b/bin/src/edit.rs @@ -50,7 +50,7 @@ impl Args { } if let Some(version) = self.version { - editor = editor.system_version(version); + editor = editor.system_version(version)?; } for target_name in &self.remove_targets { editor = editor.remove_target(target_name); diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index 707fd55..63fa1df 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -62,8 +62,15 @@ impl<'a> RepositoryEditor<'a> { } /// Change the system version of the repository. - pub fn system_version(self, system_version: Version) -> Self { - Self { system_version, ..self } + pub fn system_version( + self, + system_version: Version, + ) -> Result { + Ok(Self { + artifact_version: ArtifactVersion::new(system_version.to_string())?, + system_version, + ..self + }) } /// Sets whether an Installinator document should be generated based on the From d0dc9572f2f21d6e11917ce46add83950ab10e54 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 11 Jun 2026 21:30:09 -0700 Subject: [PATCH 092/137] verbify RepositoryEditor's methods --- bin/src/build.rs | 2 +- bin/src/edit.rs | 6 ++++-- lib/src/edit/editor.rs | 40 +++++++++++++++++++++++----------------- lib/tests/check.rs | 2 +- lib/tests/tests.rs | 6 +++--- 5 files changed, 32 insertions(+), 24 deletions(-) diff --git a/bin/src/build.rs b/bin/src/build.rs index 7382203..2247bfc 100644 --- a/bin/src/build.rs +++ b/bin/src/build.rs @@ -34,7 +34,7 @@ impl Args { RepositoryEditor::new(self.version) }?; editor = editor - .generate_installinator_document(!self.no_installinator_document); + .set_generate_installinator_document(!self.no_installinator_document); for path in self.artifacts { editor = editor.guess_artifact(path).await?; } diff --git a/bin/src/edit.rs b/bin/src/edit.rs index 498732e..a431229 100644 --- a/bin/src/edit.rs +++ b/bin/src/edit.rs @@ -42,7 +42,9 @@ impl Args { .load_zip_path(self.repo.clone(), &crate::LOG) .await?; let mut editor = RepositoryEditor::from_repo(&repo)? - .generate_installinator_document(!self.no_installinator_document); + .set_generate_installinator_document( + !self.no_installinator_document, + ); if !self.no_installinator_document { editor = editor @@ -50,7 +52,7 @@ impl Args { } if let Some(version) = self.version { - editor = editor.system_version(version)?; + editor = editor.set_system_version(version)?; } for target_name in &self.remove_targets { editor = editor.remove_target(target_name); diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index 63fa1df..66b87a3 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -62,7 +62,7 @@ impl<'a> RepositoryEditor<'a> { } /// Change the system version of the repository. - pub fn system_version( + pub fn set_system_version( self, system_version: Version, ) -> Result { @@ -78,7 +78,7 @@ impl<'a> RepositoryEditor<'a> { /// /// Defaults to `true`. The document is generated during /// [`RepositoryEditor::finish`]. - pub fn generate_installinator_document( + pub fn set_generate_installinator_document( self, generate_installinator_document: bool, ) -> Self { @@ -86,7 +86,7 @@ impl<'a> RepositoryEditor<'a> { } /// Add a measurement corpus to the repository. - pub async fn measurement_corpus( + pub async fn add_measurement_corpus( self, path: Utf8PathBuf, ) -> Result { @@ -100,7 +100,7 @@ impl<'a> RepositoryEditor<'a> { /// `hashes` specifies the number of SHA256 hashes to list in the CoRIM /// document. This can be used to create different documents at the same /// version. - pub fn fake_measurement_corpus( + pub fn add_fake_measurement_corpus( self, hashes: usize, version: ArtifactVersion, @@ -114,7 +114,7 @@ impl<'a> RepositoryEditor<'a> { /// `output_dir` is a path to the output directory for `helios-build image` /// (the `-o` argument). This directory contains `cosmo.rom`, `gimlet.rom`, /// and `zfs.img`. - pub async fn os_image_dir( + pub async fn add_os_image_dir( self, variant: OsVariant, output_dir: &Utf8Path, @@ -130,7 +130,7 @@ impl<'a> RepositoryEditor<'a> { } /// Add a fake OS image to the repository. - pub fn fake_os_image(self, variant: OsVariant) -> Result { + pub fn add_fake_os_image(self, variant: OsVariant) -> Result { let input = Input::fake_os_images(variant, self.artifact_version.clone(), None); self.insert_input(input) @@ -140,7 +140,7 @@ impl<'a> RepositoryEditor<'a> { /// /// Tags are automatically determined based on the image caboose, except /// for `slot` which must be specified. - pub async fn rot_archive( + pub async fn add_rot_archive( self, rot_slot: RotSlot, path: Utf8PathBuf, @@ -152,7 +152,7 @@ impl<'a> RepositoryEditor<'a> { /// Add a Root of Trust Bootloader Hubris archive to the repository. /// /// Tags are automatically determined based on the image caboose. - pub async fn rot_bootloader_archive( + pub async fn add_rot_bootloader_archive( self, path: Utf8PathBuf, ) -> Result { @@ -163,7 +163,10 @@ impl<'a> RepositoryEditor<'a> { /// Add a Service Processor Hubris archive to the repository. /// /// Tags are automatically determined based on the image caboose. - pub async fn sp_archive(self, path: Utf8PathBuf) -> Result { + pub async fn add_sp_archive( + self, + path: Utf8PathBuf, + ) -> Result { let source = FileSource::open(path).await?; self.insert_input(Input::sp_archive(source, None).await?) } @@ -171,7 +174,7 @@ impl<'a> RepositoryEditor<'a> { /// Add a fake Root of Trust Hubris archive to the repository. /// /// This will generate a fake Hubris archive with the appropriate tags. - pub fn fake_rot_archive(self, tags: RotTags) -> Result { + pub fn add_fake_rot_archive(self, tags: RotTags) -> Result { let input = Input::fake_rot_archive(tags, self.artifact_version.clone(), None)?; self.insert_input(input) @@ -180,7 +183,7 @@ impl<'a> RepositoryEditor<'a> { /// Add a fake Root of Trust Bootloader Hubris archive to the repository. /// /// This will generate a fake Hubris archive with the appropriate tags. - pub fn fake_rot_bootloader_archive( + pub fn add_fake_rot_bootloader_archive( self, tags: RotBootloaderTags, ) -> Result { @@ -195,7 +198,7 @@ impl<'a> RepositoryEditor<'a> { /// Add a fake Service Processor Hubris archive to the repository. /// /// This will generate a fake Hubris archive with the appropriate tags. - pub fn fake_sp_archive(self, tags: SpTags) -> Result { + pub fn add_fake_sp_archive(self, tags: SpTags) -> Result { let input = Input::fake_sp_archive(tags, self.artifact_version.clone(), None)?; self.insert_input(input) @@ -205,7 +208,10 @@ impl<'a> RepositoryEditor<'a> { /// /// The `zone_name` tag is automatically determined based on the layer /// metadata (the `oxide.json` file that starts zone tarballs). - pub async fn zone_image(self, path: Utf8PathBuf) -> Result { + pub async fn add_zone_image( + self, + path: Utf8PathBuf, + ) -> Result { self.insert_input(Input::zone_image(path).await?) } @@ -213,7 +219,7 @@ impl<'a> RepositoryEditor<'a> { /// /// This will generate a tarball containing a matching `oxide.json` layer /// metadata file. - pub fn fake_zone_image( + pub fn add_fake_zone_image( self, zone_name: String, file_name: String, @@ -265,7 +271,7 @@ impl<'a> RepositoryEditor<'a> { /// but will not be copied onto sleds for use by the control plane. This is /// intended for ancillary files that are useful to systems other than the /// control plane. - pub async fn extra_target( + pub async fn add_extra_target( mut self, target_name: String, path: Utf8PathBuf, @@ -303,7 +309,7 @@ impl<'a> RepositoryEditor<'a> { /// Manually adds a fake artifact. Don't use this if building a real /// repository; use one of the other `fake_*` methods instead. - pub fn fake_artifact( + pub fn add_fake_artifact( mut self, target_name: String, version: ArtifactVersion, @@ -356,7 +362,7 @@ impl<'a> RepositoryEditor<'a> { } /// Set the repository-level metadata. - pub fn metadata(mut self, metadata: &Metadata) -> Result { + pub fn set_metadata(mut self, metadata: &Metadata) -> Result { self.metadata = metadata.to_map().map_err(ErrorKind::ConvertMetadataToMap)?; Ok(self) diff --git a/lib/tests/check.rs b/lib/tests/check.rs index 78b4839..af0c582 100644 --- a/lib/tests/check.rs +++ b/lib/tests/check.rs @@ -27,7 +27,7 @@ async fn fake_checks_out() -> Result<(), Error> { async fn missing_installinator() -> Result<(), Error> { let log = slog::Logger::root(slog::Discard, slog::o!()); let zip = RepositoryEditor::fake(V1)? - .generate_installinator_document(false) + .set_generate_installinator_document(false) .finish() .await? .generate_root() diff --git a/lib/tests/tests.rs b/lib/tests/tests.rs index cd43c24..98c9d24 100644 --- a/lib/tests/tests.rs +++ b/lib/tests/tests.rs @@ -85,7 +85,7 @@ async fn verify_targets() -> Result<(), Error> { async fn no_artifacts() -> Result<(), Error> { let log = slog::Logger::root(slog::Discard, slog::o!()); let zip = RepositoryEditor::new(V1)? - .generate_installinator_document(false) + .set_generate_installinator_document(false) .finish() .await? .generate_root() @@ -106,8 +106,8 @@ async fn no_artifacts() -> Result<(), Error> { async fn empty_artifact() -> Result<(), Error> { let log = slog::Logger::root(slog::Discard, slog::o!()); let zip = RepositoryEditor::new(V1)? - .generate_installinator_document(false) - .fake_artifact( + .set_generate_installinator_document(false) + .add_fake_artifact( "empty.img".to_owned(), "1.0.0".parse().unwrap(), &KnownArtifactTags::InstallinatorDocument, From cfd60a792d3fd7c708ecb98fdfd6e9ee1337cfc6 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 11 Jun 2026 21:35:18 -0700 Subject: [PATCH 093/137] sprinkle `impl` into RepositoryEditor method args --- lib/src/edit/editor.rs | 43 ++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index 66b87a3..f299b9f 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -88,9 +88,9 @@ impl<'a> RepositoryEditor<'a> { /// Add a measurement corpus to the repository. pub async fn add_measurement_corpus( self, - path: Utf8PathBuf, + path: impl Into, ) -> Result { - let source = FileSource::open(path).await?; + let source = FileSource::open(path.into()).await?; let input = Input::measurement_corpus(source, None).await?; self.insert_input(input) } @@ -117,11 +117,11 @@ impl<'a> RepositoryEditor<'a> { pub async fn add_os_image_dir( self, variant: OsVariant, - output_dir: &Utf8Path, + output_dir: impl AsRef, ) -> Result { let input = Input::os_images( variant, - output_dir, + output_dir.as_ref(), None, self.artifact_version.clone(), ) @@ -143,9 +143,9 @@ impl<'a> RepositoryEditor<'a> { pub async fn add_rot_archive( self, rot_slot: RotSlot, - path: Utf8PathBuf, + path: impl Into, ) -> Result { - let source = FileSource::open(path).await?; + let source = FileSource::open(path.into()).await?; self.insert_input(Input::rot_archive(source, None, rot_slot).await?) } @@ -154,9 +154,9 @@ impl<'a> RepositoryEditor<'a> { /// Tags are automatically determined based on the image caboose. pub async fn add_rot_bootloader_archive( self, - path: Utf8PathBuf, + path: impl Into, ) -> Result { - let source = FileSource::open(path).await?; + let source = FileSource::open(path.into()).await?; self.insert_input(Input::rot_bootloader_archive(source, None).await?) } @@ -165,9 +165,9 @@ impl<'a> RepositoryEditor<'a> { /// Tags are automatically determined based on the image caboose. pub async fn add_sp_archive( self, - path: Utf8PathBuf, + path: impl Into, ) -> Result { - let source = FileSource::open(path).await?; + let source = FileSource::open(path.into()).await?; self.insert_input(Input::sp_archive(source, None).await?) } @@ -210,9 +210,9 @@ impl<'a> RepositoryEditor<'a> { /// metadata (the `oxide.json` file that starts zone tarballs). pub async fn add_zone_image( self, - path: Utf8PathBuf, + path: impl Into, ) -> Result { - self.insert_input(Input::zone_image(path).await?) + self.insert_input(Input::zone_image(path.into()).await?) } /// Add a fake zone image to the repository. @@ -240,9 +240,10 @@ impl<'a> RepositoryEditor<'a> { /// Automation should not be making any guesses. pub async fn guess_artifact( self, - path: Utf8PathBuf, + path: impl Into, ) -> Result { - let input = Input::guess(path, self.artifact_version.clone()).await?; + let input = + Input::guess(path.into(), self.artifact_version.clone()).await?; self.insert_input(input) } @@ -273,13 +274,13 @@ impl<'a> RepositoryEditor<'a> { /// control plane. pub async fn add_extra_target( mut self, - target_name: String, - path: Utf8PathBuf, + target_name: impl Into, + path: impl Into, ) -> Result { self.targets - .entry(target_name) + .entry(target_name.into()) .or_default() - .push(FileSource::open(path).await?.into()); + .push(FileSource::open(path.into()).await?.into()); Ok(self) } @@ -301,7 +302,8 @@ impl<'a> RepositoryEditor<'a> { } /// Remove a target with the given target name. - pub fn remove_target(mut self, target_name: &str) -> Self { + pub fn remove_target(mut self, target_name: impl AsRef) -> Self { + let target_name = target_name.as_ref(); self.targets.remove(target_name); self.artifacts.remove(target_name); self @@ -311,11 +313,12 @@ impl<'a> RepositoryEditor<'a> { /// repository; use one of the other `fake_*` methods instead. pub fn add_fake_artifact( mut self, - target_name: String, + target_name: impl Into, version: ArtifactVersion, tags: &KnownArtifactTags, length: u64, ) -> Result { + let target_name = target_name.into(); let prefix = format!("{target_name}\n{version}\n{tags:?}\n"); self.targets .entry(target_name.clone()) From ef620c865e3a73ccfda118d4701c2cf4cbd19ba4 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 11 Jun 2026 21:46:35 -0700 Subject: [PATCH 094/137] remove RepositoryEditor::add_fake_artifact --- lib/src/edit/editor.rs | 92 +++++++++++++++++++++++++++++------------- lib/tests/tests.rs | 34 ---------------- 2 files changed, 65 insertions(+), 61 deletions(-) diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index f299b9f..10be31b 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -309,33 +309,6 @@ impl<'a> RepositoryEditor<'a> { self } - /// Manually adds a fake artifact. Don't use this if building a real - /// repository; use one of the other `fake_*` methods instead. - pub fn add_fake_artifact( - mut self, - target_name: impl Into, - version: ArtifactVersion, - tags: &KnownArtifactTags, - length: u64, - ) -> Result { - let target_name = target_name.into(); - let prefix = format!("{target_name}\n{version}\n{tags:?}\n"); - self.targets - .entry(target_name.clone()) - .or_default() - .push(BytesSource::fake_padded(prefix, length).into()); - self.artifacts.entry(target_name.clone()).or_default().insert( - ArtifactSchema { - target_name, - version, - tags: tags - .to_tags() - .map_err(ErrorKind::ConvertKnownTagsToMap)?, - }, - ); - Ok(self) - } - /// Create a fake repository for testing purposes. pub fn fake(system_version: Version) -> Result { let mut editor = Self::new(system_version)?; @@ -577,3 +550,68 @@ pub(crate) fn generate_installinator_document( source, ) } + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use bytes::Bytes; + use chrono::Utc; + use futures_util::TryStreamExt; + use semver::Version; + use tufaceous_artifact::ArtifactVersion; + + use crate::RepositoryLoader; + use crate::TrustStoreBehavior; + use crate::edit::RepositoryEditor; + use crate::edit::source::BytesSource; + use crate::error::Error; + use crate::schema::ArtifactSchema; + + /// Test that all of the repository creation and loading machinery works + /// when an artifact is zero bytes. (This otherwise wouldn't be tested as + /// none of the fake artifacts are empty.) + #[tokio::test] + async fn empty_artifact() -> Result<(), Error> { + let log = slog::Logger::root(slog::Discard, slog::o!()); + let mut editor = RepositoryEditor::new(Version::new(1, 0, 0))? + .set_generate_installinator_document(false); + // Manually add an empty artifact. + let target_name = "empty".to_string(); + editor + .targets + .entry(target_name.clone()) + .or_default() + .push(BytesSource::new(Bytes::new()).into()); + editor.artifacts.entry(target_name.clone()).or_default().insert( + ArtifactSchema { + target_name, + version: ArtifactVersion::new_const("1.0.0"), + tags: BTreeMap::new(), + }, + ); + + let zip = editor + .finish() + .await? + .generate_root() + .sign() + .await? + .write_zip(Vec::new(), Utc::now()) + .await?; + let repo = RepositoryLoader::new() + .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .load_zip_buffer(zip, &log) + .await?; + let artifacts = repo.artifacts().iter().collect::>(); + assert_eq!(artifacts.len(), 1); + let data = repo + .read_artifact(artifacts[0]) + .await? + .map_ok(|bytes| bytes.to_vec()) + .try_concat() + .await?; + assert!(data.is_empty()); + Ok(()) + } +} diff --git a/lib/tests/tests.rs b/lib/tests/tests.rs index 98c9d24..e94b395 100644 --- a/lib/tests/tests.rs +++ b/lib/tests/tests.rs @@ -102,40 +102,6 @@ async fn no_artifacts() -> Result<(), Error> { Ok(()) } -#[tokio::test] -async fn empty_artifact() -> Result<(), Error> { - let log = slog::Logger::root(slog::Discard, slog::o!()); - let zip = RepositoryEditor::new(V1)? - .set_generate_installinator_document(false) - .add_fake_artifact( - "empty.img".to_owned(), - "1.0.0".parse().unwrap(), - &KnownArtifactTags::InstallinatorDocument, - 0, - )? - .finish() - .await? - .generate_root() - .sign() - .await? - .write_zip(Vec::new(), Utc::now()) - .await?; - let repo = RepositoryLoader::new() - .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) - .load_zip_buffer(zip, &log) - .await?; - let artifacts = repo.artifacts().iter().collect::>(); - assert_eq!(artifacts.len(), 1); - let data = repo - .read_artifact(artifacts[0]) - .await? - .map_ok(|bytes| bytes.to_vec()) - .try_concat() - .await?; - assert!(data.is_empty()); - Ok(()) -} - #[tokio::test] async fn inconsistent_fake() -> Result<(), Error> { let log = slog::Logger::root(slog::Discard, slog::o!()); From 94c93c4c6fc4771850d96783c7b7c45a07838e0d Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 11 Jun 2026 22:13:54 -0700 Subject: [PATCH 095/137] properly handle invalid hash lengths from imported repos --- lib/src/edit/editor.rs | 18 ++++++++---------- lib/src/edit/fake.rs | 2 +- lib/src/edit/sign.rs | 4 ++-- lib/src/edit/source.rs | 23 ++++++++++++++--------- lib/src/error.rs | 6 ++++++ 5 files changed, 31 insertions(+), 22 deletions(-) diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index 10be31b..edfb39b 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -441,7 +441,7 @@ impl<'a> RepositoryEditor<'a> { tasks.spawn(future); } TargetSource::Repository(source) => { - all_targets.push((target_name, source.into_target())); + all_targets.push((target_name, source.into_target()?)); } } } @@ -460,7 +460,7 @@ impl<'a> RepositoryEditor<'a> { let output = generate_installinator_document( artifacts.values().filter_map(|artifact| { let target = targets.0.get(&artifact.target_name)?; - Some((artifact, target.sha256.as_slice())) + Some((artifact, target.sha256)) }), self.artifact_version.clone(), )?; @@ -524,20 +524,18 @@ impl<'a> TargetMap<'a> { } pub(crate) fn generate_installinator_document( - artifacts: impl Iterator, impl AsRef<[u8]>)>, + artifacts: impl Iterator, ArtifactHash)>, version: ArtifactVersion, ) -> Result, Error> { let target_name = format!("installinator_document-{version}.json"); let mut document = InstallinatorDocument::empty(version.clone()); for (artifact, hash) in artifacts { let artifact = artifact.as_ref(); - if let Ok(hash) = hash.as_ref().try_into().map(ArtifactHash) - && let Some(artifact) = crate::util::installinator_artifact( - artifact.tags.clone(), - hash, - &artifact.target_name, - ) - { + if let Some(artifact) = crate::util::installinator_artifact( + artifact.tags.clone(), + hash, + &artifact.target_name, + ) { document.artifacts.insert(artifact); } } diff --git a/lib/src/edit/fake.rs b/lib/src/edit/fake.rs index bdc7d53..cab8288 100644 --- a/lib/src/edit/fake.rs +++ b/lib/src/edit/fake.rs @@ -57,7 +57,7 @@ impl ArtifactSetExt for ArtifactSet { version: artifact.version.clone(), tags: artifact.tags.clone(), }, - &artifact.hash, + artifact.hash, ) }), version, diff --git a/lib/src/edit/sign.rs b/lib/src/edit/sign.rs index fcde750..d1e393a 100644 --- a/lib/src/edit/sign.rs +++ b/lib/src/edit/sign.rs @@ -155,12 +155,12 @@ impl<'a> UnsignedRepository<'a> { let mut sources = BTreeMap::new(); for (target_name, Target { length, sha256, source }) in self.targets { let path = if consistent_snapshot { - format!("{}.{target_name}", hex::encode(&sha256)) + format!("{}.{target_name}", hex::encode(sha256)) } else { target_name.clone() }; let hashes = tough::schema::Hashes { - sha256: sha256.into(), + sha256: sha256.0.to_vec().into(), _extra: HashMap::new(), }; let target = tough::schema::Target { diff --git a/lib/src/edit/source.rs b/lib/src/edit/source.rs index a9c9abb..57884d8 100644 --- a/lib/src/edit/source.rs +++ b/lib/src/edit/source.rs @@ -33,7 +33,7 @@ use crate::error::try_path; #[derive(Debug)] pub(crate) struct Target<'a> { pub(crate) length: u64, - pub(crate) sha256: Vec, + pub(crate) sha256: ArtifactHash, pub(crate) source: TargetSource<'a>, } @@ -158,7 +158,7 @@ impl BytesSource { pub(crate) async fn into_target(mut self) -> Target<'static> { Target { length: self.length(), - sha256: self.sha256().await.0.to_vec(), + sha256: self.sha256().await, source: self.into(), } } @@ -207,7 +207,7 @@ impl FileSource { Some(inner) => inner, None => self.read_impl(None).await?, }; - Ok(Target { length, sha256: sha256.0.to_vec(), source: self.into() }) + Ok(Target { length, sha256, source: self.into() }) } async fn read_impl( @@ -309,12 +309,17 @@ pub(crate) struct RepositorySource<'a> { } impl<'a> RepositorySource<'a> { - pub(crate) fn into_target(self) -> Target<'a> { - Target { - length: self.length, - sha256: self.sha256.clone(), - source: self.into(), - } + pub(crate) fn into_target(self) -> Result, Error> { + let sha256 = match self.sha256.as_slice().try_into() { + Ok(sha256) => ArtifactHash(sha256), + Err(_) => { + return Err(ErrorKind::InvalidImportedHashLength { + target_name: self.target_name, + } + .into()); + } + }; + Ok(Target { length: self.length, sha256, source: self.into() }) } pub(crate) fn stream(&self) -> impl Stream> { diff --git a/lib/src/error.rs b/lib/src/error.rs index 5755293..d8f5648 100644 --- a/lib/src/error.rs +++ b/lib/src/error.rs @@ -207,6 +207,11 @@ pub enum ErrorKind { second_target_name: String, tags: BTreeMap, }, + #[error( + "target {target_name} from imported repository has invalid SHA-256 \ + hash length; remove the target before calling finish" + )] + InvalidImportedHashLength { target_name: String }, #[error("failed to serialize artifacts document")] SerializeArtifacts(#[source] serde_json::Error), #[error("failed to serialize Installinator document")] @@ -281,6 +286,7 @@ impl ErrorKind { | ErrorKind::ConvertMetadataToMap(_) | ErrorKind::TargetNameCollision { .. } | ErrorKind::DisallowedTagCollision { .. } + | ErrorKind::InvalidImportedHashLength { .. } | ErrorKind::SerializeArtifacts(_) | ErrorKind::SerializeInstallinator(_) | ErrorKind::NoSigningRoot From 41b93ccb83567153fd7852c169c4e12ec1ee1b5f Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 11 Jun 2026 22:41:38 -0700 Subject: [PATCH 096/137] immediately check for conflicting artifact definitions --- lib/src/edit/editor.rs | 73 ++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 41 deletions(-) diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index edfb39b..dbe9410 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -4,7 +4,6 @@ use std::collections::BTreeMap; use std::collections::HashMap; -use std::collections::HashSet; use camino::Utf8Path; use camino::Utf8PathBuf; @@ -44,7 +43,7 @@ pub struct RepositoryEditor<'a> { artifact_version: ArtifactVersion, generate_installinator_document: bool, targets: HashMap>>, - artifacts: HashMap>, + artifacts: BTreeMap, metadata: BTreeMap, } @@ -56,7 +55,7 @@ impl<'a> RepositoryEditor<'a> { system_version, generate_installinator_document: true, targets: HashMap::new(), - artifacts: HashMap::new(), + artifacts: BTreeMap::new(), metadata: BTreeMap::new(), }) } @@ -253,10 +252,7 @@ impl<'a> RepositoryEditor<'a> { { for output in input.outputs()? { if let Some(artifact) = output.to_artifact_schema() { - self.artifacts - .entry(artifact.target_name.clone()) - .or_default() - .insert(artifact); + insert_artifact(&mut self.artifacts, artifact)?; } self.targets .entry(output.target_name) @@ -291,10 +287,9 @@ impl<'a> RepositoryEditor<'a> { ) -> Result { let filter = filter.to_tags().map_err(ErrorKind::ConvertKnownTagsToMap)?; - let removed = self.artifacts.extract_if(|_, artifacts| { - artifacts.retain(|artifact| artifact.tags != filter); - artifacts.is_empty() // extract_if: remove from map if true - }); + let removed = self + .artifacts + .extract_if(.., |_, artifact| artifact.tags == filter); for (target_name, _) in removed { self.targets.remove(&target_name); } @@ -356,10 +351,7 @@ impl<'a> RepositoryEditor<'a> { /// editor. pub fn import_repo(mut self, repo: &'a Repository) -> Result { for artifact in repo.to_artifact_schema()? { - self.artifacts - .entry(artifact.target_name.clone()) - .or_default() - .insert(artifact); + insert_artifact(&mut self.artifacts, artifact)?; } for (target_name, target) in repo.targets() { if target_name.raw() == ArtifactSetSchema::TARGET_NAME { @@ -381,19 +373,7 @@ impl<'a> RepositoryEditor<'a> { /// Finalize the artifacts and targets, returning an [`UnsignedRepository`]. pub async fn finish(self) -> Result, Error> { - // Un-nest `self.artifacts`, returning an error if we have multiple - // artifact definitions for a single target name. - let mut artifacts = self - .artifacts - .into_iter() - .filter_map(|(target_name, entries)| { - Some(if entries.len() > 1 { - Err(ErrorKind::TargetNameCollision { target_name }.into()) - } else { - Ok((target_name, entries.into_iter().next()?)) - }) - }) - .collect::, Error>>()?; + let mut artifacts = self.artifacts; // Ensure each set of tags is unique, except for known tag sets that // we expect to be non-unique. @@ -464,18 +444,12 @@ impl<'a> RepositoryEditor<'a> { }), self.artifact_version.clone(), )?; - if let Some(artifact) = output.to_artifact_schema() { - if let Some(existing) = artifacts.get(&artifact.target_name) { - if existing != &artifact { - return Err(ErrorKind::TargetNameCollision { - target_name: artifact.target_name, - } - .into()); - } - } else { - artifacts.insert(artifact.target_name.clone(), artifact); - } - } + let Some(artifact) = output.to_artifact_schema() else { + unreachable!( + "generate_installinator_document returns an artifact" + ); + }; + insert_artifact(&mut artifacts, artifact)?; targets.insert( output.target_name, output.source.into_target().await, @@ -549,6 +523,22 @@ pub(crate) fn generate_installinator_document( ) } +fn insert_artifact( + artifacts: &mut BTreeMap, + artifact: ArtifactSchema, +) -> Result<(), Error> { + if let Some(existing) = artifacts.get(&artifact.target_name) + && existing != &artifact + { + return Err(ErrorKind::TargetNameCollision { + target_name: artifact.target_name, + } + .into()); + } + artifacts.insert(artifact.target_name.clone(), artifact); + Ok(()) +} + #[cfg(test)] mod tests { use std::collections::BTreeMap; @@ -581,7 +571,8 @@ mod tests { .entry(target_name.clone()) .or_default() .push(BytesSource::new(Bytes::new()).into()); - editor.artifacts.entry(target_name.clone()).or_default().insert( + editor.artifacts.insert( + target_name.clone(), ArtifactSchema { target_name, version: ArtifactVersion::new_const("1.0.0"), From 7a653ceda63b8455f99cee6906af3f09a492b1be Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 11 Jun 2026 23:20:10 -0700 Subject: [PATCH 097/137] some more s/sign/rkth/ --- lib/src/edit/hubris_archive.rs | 12 ++++++------ lib/src/edit/input.rs | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/src/edit/hubris_archive.rs b/lib/src/edit/hubris_archive.rs index 4d9e59c..0d80abf 100644 --- a/lib/src/edit/hubris_archive.rs +++ b/lib/src/edit/hubris_archive.rs @@ -149,7 +149,7 @@ impl Input { ) -> Result { let data = CabooseData { board: &tags.rot_board, - sign: &tags.rot_rkth, + rkth: &tags.rot_rkth, commit: "this-is-a-fake-rot", version: interior_version.unwrap_or(&version), }; @@ -164,7 +164,7 @@ impl Input { ) -> Result { let data = CabooseData { board: &tags.rot_board, - sign: &tags.rot_rkth, + rkth: &tags.rot_rkth, commit: "this-is-a-fake-rot-bootloader", version: interior_version.unwrap_or(&version), }; @@ -179,7 +179,7 @@ impl Input { ) -> Result { let data = CabooseData { board: &tags.sp_board, - sign: &None, + rkth: &None, commit: "this-is-a-fake-sp", version: interior_version.unwrap_or(&version), }; @@ -190,7 +190,7 @@ impl Input { struct CabooseData<'a> { board: &'a str, - sign: &'a Option, + rkth: &'a Option, commit: &'static str, version: &'a ArtifactVersion, } @@ -202,8 +202,8 @@ impl CabooseData<'_> { .name(self.board) .git_commit(self.commit) .version(self.version.to_string()); - if let Some(sign) = self.sign { - builder = builder.sign(sign.as_str()); + if let Some(rkth) = self.rkth { + builder = builder.sign(rkth.as_str()); } let caboose = builder.build(); diff --git a/lib/src/edit/input.rs b/lib/src/edit/input.rs index fd8bbe9..b6dbd72 100644 --- a/lib/src/edit/input.rs +++ b/lib/src/edit/input.rs @@ -118,9 +118,9 @@ impl Input { } Input::Rot { source, tags, version } => { let target_name = format!( - "rot/{board}-{sign}-{version}-slot-{slot}.zip", + "rot/{board}-{rkth}-{version}-slot-{slot}.zip", board = tags.rot_board, - sign = tags + rkth = tags .rot_rkth .as_ref() .map_or("unsigned", RotKeyTableHash::as_str), @@ -130,9 +130,9 @@ impl Input { } Input::RotBootloader { source, tags, version } => { let target_name = format!( - "rot-bootloader/{board}-{sign}-{version}.zip", + "rot-bootloader/{board}-{rkth}-{version}.zip", board = tags.rot_board, - sign = tags + rkth = tags .rot_rkth .as_ref() .map_or("unsigned", RotKeyTableHash::as_str), From 1af3f9b0bff912ab0bedd37e923fe44f2b1520ad Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 11 Jun 2026 23:31:18 -0700 Subject: [PATCH 098/137] remove errant eprintln! --- lib/src/edit/measurement_corpus.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/src/edit/measurement_corpus.rs b/lib/src/edit/measurement_corpus.rs index 5bd2964..6f55be1 100644 --- a/lib/src/edit/measurement_corpus.rs +++ b/lib/src/edit/measurement_corpus.rs @@ -60,8 +60,7 @@ impl Input> { // we read the entire thing it'd still be a CoRIM manifest. None } - Err(error) => { - eprintln!("{error:?}"); + Err(_) => { return Ok(ControlFlow::Continue(input)); } }; From a02d5c06d87ed6e409855df8fb8fe92b21c1d6eb Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 11 Jun 2026 23:36:03 -0700 Subject: [PATCH 099/137] fix logic bug in read_os_tarball_metadata_blocking --- lib/src/edit/os_images.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/src/edit/os_images.rs b/lib/src/edit/os_images.rs index 3766ece..9bbb0e7 100644 --- a/lib/src/edit/os_images.rs +++ b/lib/src/edit/os_images.rs @@ -147,15 +147,12 @@ fn read_os_tarball_metadata_blocking( } let Some(extension) = path.extension() else { continue }; if extension != "txt" { - continue; + break; } let file_name = Path::new( path.file_name() .expect("a path with an extension must have a file name"), ); - if file_name == "zfs.img" { - break; - } let file_name = try_path!( Utf8PathBuf::try_from(file_name.to_owned()) .map_err(camino::FromPathBufError::into_io_error), From 34aebb84122b3263de90605f7df0f1fcb1b54d13 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 11 Jun 2026 23:41:54 -0700 Subject: [PATCH 100/137] stronger root types in edit::sign --- lib/src/edit/sign.rs | 55 +++++++++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/lib/src/edit/sign.rs b/lib/src/edit/sign.rs index d1e393a..1d5cf0f 100644 --- a/lib/src/edit/sign.rs +++ b/lib/src/edit/sign.rs @@ -36,9 +36,8 @@ pub(crate) const DEFAULT_VALIDITY: Duration = #[must_use] pub struct UnsignedRepository<'a> { targets: BTreeMap>, - root: Option>, + root: Option, keys: Vec>, - generate_root: bool, snapshot_version: NonZero, snapshot_expires: DateTime, targets_version: NonZero, @@ -47,6 +46,12 @@ pub struct UnsignedRepository<'a> { timestamp_expires: DateTime, } +#[derive(Debug)] +enum RequestedRoot { + Root(Vec), + Generate, +} + impl<'a> UnsignedRepository<'a> { pub(crate) fn from_targets(targets: BTreeMap>) -> Self { let now = Utc::now().trunc_subsecs(0); @@ -59,7 +64,6 @@ impl<'a> UnsignedRepository<'a> { targets, root: None, keys: Vec::new(), - generate_root: false, snapshot_version: version, snapshot_expires: expires, targets_version: version, @@ -70,11 +74,7 @@ impl<'a> UnsignedRepository<'a> { } pub fn root(self, root: impl AsRef<[u8]>) -> Self { - Self { - root: Some(root.as_ref().to_vec()), - generate_root: false, - ..self - } + Self { root: Some(RequestedRoot::Root(root.as_ref().to_vec())), ..self } } pub fn key(mut self, key: impl KeySource + 'static) -> Self { @@ -83,7 +83,7 @@ impl<'a> UnsignedRepository<'a> { } pub fn generate_root(self) -> Self { - Self { root: None, generate_root: true, ..self } + Self { root: Some(RequestedRoot::Generate), ..self } } pub fn snapshot_version(self, snapshot_version: NonZero) -> Self { @@ -111,22 +111,29 @@ impl<'a> UnsignedRepository<'a> { } pub async fn sign(mut self) -> Result, Error> { - let (root, consistent_snapshot) = if let Some(root) = self.root { - let parsed_root: Signed = serde_json::from_slice(&root) - .map_err(ErrorKind::ParseSigningRoot)?; - (root, parsed_root.signed.consistent_snapshot) - } else if self.generate_root { - if self.keys.is_empty() { - self.keys.push(Box::new(Ed25519Key::generate()?)); + let (root, consistent_snapshot) = match self.root { + Some(RequestedRoot::Root(root)) => { + let parsed_root: Signed = + serde_json::from_slice(&root) + .map_err(ErrorKind::ParseSigningRoot)?; + (root, parsed_root.signed.consistent_snapshot) } - let expires = self - .snapshot_expires - .min(self.targets_expires) - .min(self.timestamp_expires); - let root = crate::edit::generate_root(&self.keys, expires).await?; - (root.buffer().clone(), root.signed().signed.consistent_snapshot) - } else { - return Err(ErrorKind::NoSigningRoot.into()); + Some(RequestedRoot::Generate) => { + if self.keys.is_empty() { + self.keys.push(Box::new(Ed25519Key::generate()?)); + } + let expires = self + .snapshot_expires + .min(self.targets_expires) + .min(self.timestamp_expires); + let root = + crate::edit::generate_root(&self.keys, expires).await?; + ( + root.buffer().clone(), + root.signed().signed.consistent_snapshot, + ) + } + None => return Err(ErrorKind::NoSigningRoot.into()), }; let tempdir = tokio::task::spawn_blocking(camino_tempfile::tempdir) From 176d2821126cee727b199ece752dd7f47e2601fb Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 11 Jun 2026 23:57:02 -0700 Subject: [PATCH 101/137] return Err in zip_writer::write_task if other end is dropped --- lib/src/zip_writer.rs | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/lib/src/zip_writer.rs b/lib/src/zip_writer.rs index 999ff64..ecde03d 100644 --- a/lib/src/zip_writer.rs +++ b/lib/src/zip_writer.rs @@ -19,7 +19,7 @@ use tokio::task::JoinHandle; pub(crate) struct ZipWriter { task: JoinHandle>, - file_tx: mpsc::Sender, + file_tx: mpsc::Sender, } impl ZipWriter { @@ -67,7 +67,7 @@ impl ZipWriter { } pub(crate) async fn finish(self) -> Result { - drop(self.file_tx); + self.file_tx.send(ZipFileMessage::Finish).await.ok(); self.task.await.map_err(std::io::Error::from)? } } @@ -95,7 +95,7 @@ impl<'a, W> ZipFileBuilder<'a, W> { } pub(crate) async fn start(self) -> Result, SendError> { - self.writer.file_tx.send(self.inner).await?; + self.writer.file_tx.send(ZipFileMessage::File(self.inner)).await?; Ok(ZipDataWriter { _writer: self.writer, bytes_tx: self.bytes_tx }) } } @@ -115,6 +115,11 @@ impl ZipDataWriter<'_, W> { } } +enum ZipFileMessage { + File(ZipFile), + Finish, +} + struct ZipFile { name: String, compression: Compression, @@ -125,10 +130,23 @@ struct ZipFile { fn write_task( writer: W, - mut file_rx: mpsc::Receiver, + mut file_rx: mpsc::Receiver, ) -> Result { let mut archive = ZipArchiveWriter::new(writer); - while let Some(mut file) = file_rx.blocking_recv() { + loop { + let message = file_rx.blocking_recv().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "ZipWriter dropped before finish", + ) + })?; + let mut file = match message { + ZipFileMessage::File(file) => file, + ZipFileMessage::Finish => { + return archive.finish(); + } + }; + let mut builder = archive.new_file(&file.name); if file.compression != Compression::none() { builder = builder.compression_method(CompressionMethod::Deflate); @@ -149,7 +167,6 @@ fn write_task( encoder.finish()?; entry.finish(output)?; } - archive.finish() } enum ZipEncoder { From 1fa487848acd21fe288bb14e750a0edbb2415405 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Fri, 12 Jun 2026 00:00:01 -0700 Subject: [PATCH 102/137] don't panic in deflate_heuristic --- lib/src/edit/sign.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/src/edit/sign.rs b/lib/src/edit/sign.rs index 1d5cf0f..46ea4b3 100644 --- a/lib/src/edit/sign.rs +++ b/lib/src/edit/sign.rs @@ -286,12 +286,13 @@ fn deflate_heuristic(buf: &[u8]) -> Compression { // gzip, e.g. illumos zone tarball Compression::none() } else if buf.starts_with(b"\x78") - && let [x, y] = &buf[..2] + && let Some([x, y]) = &buf.get(..2) && u16::from_be_bytes([*x, *y]) % 31 == 0 { // zlib if y & 0xc0 == 0 { - // compression level 0 + // buf is zlib-compressed at level 0, which is not compressed at + // all, so we should compress this Compression::best() } else { Compression::none() From ea8e14bcfce37f529f017ae315a6036362fe13be Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Fri, 12 Jun 2026 00:26:16 -0700 Subject: [PATCH 103/137] fmt --- bin/src/build.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/src/build.rs b/bin/src/build.rs index 2247bfc..420de3f 100644 --- a/bin/src/build.rs +++ b/bin/src/build.rs @@ -33,8 +33,9 @@ impl Args { } else { RepositoryEditor::new(self.version) }?; - editor = editor - .set_generate_installinator_document(!self.no_installinator_document); + editor = editor.set_generate_installinator_document( + !self.no_installinator_document, + ); for path in self.artifacts { editor = editor.guess_artifact(path).await?; } From 44eff96be6f54a017d3175fb3b8ffe6cdcf41e14 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Fri, 12 Jun 2026 00:31:57 -0700 Subject: [PATCH 104/137] Update lib/src/edit/input.rs Co-authored-by: John Gallagher --- lib/src/edit/input.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/edit/input.rs b/lib/src/edit/input.rs index b6dbd72..5bae3c0 100644 --- a/lib/src/edit/input.rs +++ b/lib/src/edit/input.rs @@ -87,7 +87,7 @@ impl Input { version, } => { let base = Utf8PathBuf::from(format!("os-{os_variant}")); - let mut vec = Vec::new(); + let mut vec = Vec::with_capacity(3 + extra_targets.len()); vec.push(Output::new( base.join(COSMO_PHASE_1_PATH).into(), version.clone(), From eb21197220aa4c9e73a951dfc150bf91f1d0c766 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Fri, 12 Jun 2026 00:32:55 -0700 Subject: [PATCH 105/137] () = result??; --- lib/src/repo.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/src/repo.rs b/lib/src/repo.rs index ad65b2d..611b5c9 100644 --- a/lib/src/repo.rs +++ b/lib/src/repo.rs @@ -336,7 +336,7 @@ impl Repository { } Err(TryAcquireError::NoPermits) => { if let Some(result) = set.join_next().await { - result??; + () = result??; } Arc::clone(&semaphore) .acquire_owned() @@ -355,7 +355,7 @@ impl Repository { } while let Some(result) = set.join_next().await { - result??; + () = result??; } Ok(()) From 68bf84e1e093d007c305f862a4b7aff892462f74 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 15 Jun 2026 09:44:59 -0700 Subject: [PATCH 106/137] split ErrorKind::ReadZip --- lib/src/error.rs | 19 ++++++++++++++++--- lib/src/zip_transport.rs | 10 +++++----- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/lib/src/error.rs b/lib/src/error.rs index d8f5648..7dd20fe 100644 --- a/lib/src/error.rs +++ b/lib/src/error.rs @@ -60,10 +60,21 @@ pub enum ErrorKind { ToughKey(#[source] Box), #[error( - "failed to read zip archive{archive_path}", + "failed to read end of central directory of zip archive{archive_path}", archive_path = SpacePath(archive_path), )] - ReadZip { source: rawzip::Error, archive_path: Option }, + ReadZipEocd { source: rawzip::Error, archive_path: Option }, + #[error( + "failed to read central directory file header of \ + zip archive{archive_path}", + archive_path = SpacePath(archive_path), + )] + ReadZipCdfh { source: rawzip::Error, archive_path: Option }, + #[error( + "failed to read local file header of zip archive{archive_path}", + archive_path = SpacePath(archive_path), + )] + ReadZipLocal { source: rawzip::Error, archive_path: Option }, #[error( "failed to write zip archive{archive_path}", archive_path = SpacePath(archive_path), @@ -342,7 +353,9 @@ impl ErrorKind { true } - ErrorKind::ReadZip { source, .. } => { + ErrorKind::ReadZipEocd { source, .. } + | ErrorKind::ReadZipCdfh { source, .. } + | ErrorKind::ReadZipLocal { source, .. } => { // All of rawzip's errors are related to broken zip files, // except for IO errors. !matches!(source.kind(), rawzip::ErrorKind::IO(_)) diff --git a/lib/src/zip_transport.rs b/lib/src/zip_transport.rs index df71109..4e188ed 100644 --- a/lib/src/zip_transport.rs +++ b/lib/src/zip_transport.rs @@ -133,7 +133,7 @@ impl + Debug + Send + Sync + 'static> ZipTransport> { pub fn from_slice(data: T, log: &Logger) -> Result { let archive = ZipArchive::with_max_search_space(EOCD_MAX_SEARCH_SPACE) .locate_in_slice(data) - .map_err(|(_, source)| ErrorKind::ReadZip { + .map_err(|(_, source)| ErrorKind::ReadZipEocd { source, archive_path: None, })?; @@ -152,7 +152,7 @@ impl ZipTransport { ZipArchive::with_max_search_space(EOCD_MAX_SEARCH_SPACE) .locate_in_file(file, &mut buffer) .map_err(|(_, error)| error), - ReadZip, + ReadZipEocd, archive_path ); Self::from_impl_blocking(archive, archive_path, Some(buffer), log) @@ -177,7 +177,7 @@ impl ZipTransport { let mut records = archive.entries(&mut buffer); while let Some(record) = - try_archive_path!(records.next_entry(), ReadZip, archive_path) + try_archive_path!(records.next_entry(), ReadZipCdfh, archive_path) { if usize64!(all_entries.len()) >= archive.entries_hint() { return Err(ErrorKind::ZipEntryCount { @@ -213,12 +213,12 @@ impl ZipTransport { for RawEntry { raw_path, wayfinder } in all_entries { let entry = try_archive_path!( archive.get_entry(wayfinder), - ReadZip, + ReadZipLocal, archive_path ); let header = try_archive_path!( entry.local_header(&mut buffer), - ReadZip, + ReadZipLocal, archive_path ); // Check that the path in the central directory matches the header From abccf73f416a39fbc915570640a723617be1275e Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 15 Jun 2026 10:53:36 -0700 Subject: [PATCH 107/137] un-backwards the streams (remove stream::try_unfold) --- lib/src/edit/source.rs | 55 ++++++++++++++--------- lib/src/repo.rs | 15 ++++++- lib/src/repo/v1.rs | 97 +++++++++++++++++++++++++++------------- lib/src/zip_transport.rs | 91 +++++++++++++++++++------------------ 4 files changed, 161 insertions(+), 97 deletions(-) diff --git a/lib/src/edit/source.rs b/lib/src/edit/source.rs index 57884d8..33f0f11 100644 --- a/lib/src/edit/source.rs +++ b/lib/src/edit/source.rs @@ -23,6 +23,7 @@ use serde::Serialize; use sha2::Digest; use sha2::Sha256; use tokio::fs::File; +use tokio::sync::mpsc; use tufaceous_artifact::ArtifactHash; use crate::Repository; @@ -274,29 +275,41 @@ impl FileSource { } pub(crate) fn stream(&self) -> impl Stream> { - stream::try_unfold( - (self.inner.clone(), BytesMut::new(), 0), - async |(inner, mut buf, mut offset)| { - tokio::task::spawn_blocking(move || { - if buf.capacity() == 0 { - buf.reserve(8192); + let (tx, mut rx) = mpsc::channel::>(1); + let inner = self.inner.clone(); + tokio::task::spawn_blocking(move || { + let mut buf = BytesMut::with_capacity(8192); + let mut offset = 0; + loop { + if buf.capacity() == 0 { + buf.reserve(8192); + } + buf.resize(buf.capacity(), 0); + match inner.file.read_at(&mut buf, offset) { + Ok(n) => { + buf.truncate(n); } - buf.resize(buf.capacity(), 0); - let n = try_path!( - inner.file.read_at(&mut buf, offset), - ReadFile, - inner.path.clone() - ); - if n == 0 { - return Ok(None); + Err(source) => { + let err = ErrorKind::ReadFile { + source, + path: Some(inner.path.clone()), + }; + tx.blocking_send(Err(err.into())).ok(); + return; } - offset += usize64!(n); - buf.truncate(n); - Ok(Some((buf.split().freeze(), (inner, buf, offset)))) - }) - .await? - }, - ) + } + + let bytes = buf.split().freeze(); + if bytes.is_empty() { + return; + } + offset += usize64!(bytes.len()); + if tx.blocking_send(Ok(bytes)).is_err() { + return; + } + } + }); + futures_util::stream::poll_fn(move |cx| rx.poll_recv(cx)) } } diff --git a/lib/src/repo.rs b/lib/src/repo.rs index 611b5c9..279ba42 100644 --- a/lib/src/repo.rs +++ b/lib/src/repo.rs @@ -45,6 +45,7 @@ pub type TargetStream = /// A loaded TUF repository. #[derive(Debug, Clone)] pub struct Repository { + log: Logger, inner: tough::Repository, system_version: Version, trust_root: Vec, @@ -121,6 +122,7 @@ impl Repository { && let Some(partial) = v1::from_loaded(&repo, log).await? { return Ok(Repository { + log: log.clone(), inner: repo, trust_root, system_version: partial.system_version, @@ -151,6 +153,7 @@ impl Repository { }) .collect(); Ok(Repository { + log: log.clone(), inner: repo, trust_root, system_version, @@ -240,13 +243,21 @@ impl Repository { ArtifactData::Target { target_name } => { self.read_target(target_name).await } - ArtifactData::V1Unpacked { file, .. } => { + ArtifactData::V1Unpacked { + file, + original_target_name, + inner_path, + } => { let unpacked = v1::UnpackedArtifact { file: file.clone(), hash: artifact.hash, length: artifact.length, }; - Ok(Box::pin(unpacked.stream())) + Ok(Box::pin(unpacked.stream( + self.log.clone(), + original_target_name.clone(), + inner_path.clone(), + ))) } } } diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs index 5052c45..2fe295f 100644 --- a/lib/src/repo/v1.rs +++ b/lib/src/repo/v1.rs @@ -20,7 +20,6 @@ use flate2::bufread::GzDecoder; use futures_util::Stream; use futures_util::TryStreamExt; use futures_util::pin_mut; -use futures_util::stream; use hubtools::Caboose; use hubtools::RawHubrisArchive; use rawzip::FileReader; @@ -31,6 +30,7 @@ use serde::Deserialize; use sha2::Digest; use sha2::Sha256; use slog::Logger; +use slog::error; use slog::info; use slog::warn; use tokio::sync::mpsc; @@ -274,45 +274,82 @@ impl UnpackedArtifact { pub(crate) fn stream( self, + log: Logger, + original_target_name: String, + inner_path: Utf8PathBuf, ) -> impl Stream> + 'static { - stream::try_unfold( - (self, BytesMut::new(), Sha256::new(), 0), - async |(this, mut buf, mut hasher, mut bytes_read)| { + let (tx, mut rx) = mpsc::channel::>(1); + let task = tokio::task::spawn_blocking(move || { + type SendError = mpsc::error::SendError>; + + let mut buf = BytesMut::with_capacity(8192); + let mut hasher = Sha256::new(); + let mut bytes_read = 0; + loop { if buf.capacity() == 0 { buf.reserve(8192); } buf.resize(buf.capacity(), 0); - let (this, mut buf) = tokio::task::spawn_blocking(move || { - let n = this.file.read_at(&mut buf, bytes_read).map_err( - |source| ErrorKind::ReadFile { source, path: None }, - )?; - buf.truncate(n); - Ok::<_, Error>((this, buf)) - }) - .await??; + match self.file.read_at(&mut buf, bytes_read) { + Ok(n) => { + buf.truncate(n); + } + Err(source) => { + let err = ErrorKind::ReadFile { source, path: None }; + tx.blocking_send(Err(err.into()))?; + return Ok::<_, SendError>(()); + } + } + let bytes = buf.split().freeze(); if bytes.is_empty() { - let msg = if this.hash != ArtifactHash(hasher.finalize().0) - { - "invalid checksum" - } else if this.length != bytes_read { - "invalid length" - } else { - return Ok(None); - }; - let source = std::io::Error::new( - std::io::ErrorKind::InvalidData, - msg, - ); - return Err( - ErrorKind::ReadFile { source, path: None }.into() - ); + break; } hasher.update(&bytes); bytes_read += usize64!(bytes.len()); - Ok(Some((bytes, (this, buf, hasher, bytes_read)))) - }, - ) + tx.blocking_send(Ok(bytes))?; + } + + let msg = if self.hash != ArtifactHash(hasher.finalize().into()) { + "invalid checksum" + } else if self.length != bytes_read { + "invalid length" + } else { + // correct checksum and length + return Ok::<_, SendError>(()); + }; + let source = + std::io::Error::new(std::io::ErrorKind::InvalidData, msg); + tx.blocking_send(Err( + ErrorKind::ReadFile { source, path: None }.into() + )) + }); + + tokio::task::spawn(async move { + if let Ok(Err(send_error)) = task.await { + match send_error.0 { + Ok(_) => { + error!( + log, + "unpacked file reader hung up mid-stream"; + "original_target_name" => original_target_name, + "inner_path" => inner_path.as_str(), + ); + } + Err(err) => { + error!( + log, + "unpacked file reader hung up mid-stream \ + before receiving error"; + "original_target_name" => original_target_name, + "inner_path" => inner_path.as_str(), + "err" => &crate::util::error_chain(&err), + ); + } + } + } + }); + futures_util::stream::poll_fn(move |cx| rx.poll_recv(cx)) } } diff --git a/lib/src/zip_transport.rs b/lib/src/zip_transport.rs index 4e188ed..b0779e9 100644 --- a/lib/src/zip_transport.rs +++ b/lib/src/zip_transport.rs @@ -66,8 +66,6 @@ use bytes::BytesMut; use camino::Utf8PathBuf; use flate2::read::DeflateDecoder; use futures_util::Stream; -use futures_util::TryFutureExt; -use futures_util::stream; use rawzip::CompressionMethod; use rawzip::FileReader; use rawzip::ReaderAt; @@ -75,9 +73,9 @@ use rawzip::ZipArchive; use rawzip::ZipArchiveEntryWayfinder; use rawzip::ZipFileHeaderRecord; use slog::Logger; +use slog::error; use slog::warn; use tokio::sync::mpsc; -use tokio::sync::oneshot; use tough::Transport; use tough::TransportError; use tough::TransportErrorKind; @@ -114,6 +112,7 @@ const EOCD_MAX_SEARCH_SPACE: u64 = 22; #[derive(Debug)] pub struct ZipTransport { inner: Arc>, + log: Logger, } #[derive(Debug)] @@ -125,7 +124,7 @@ struct Inner { // Manually implemented, as the derive macro adds an unnecessary `T: Clone`. impl Clone for ZipTransport { fn clone(&self) -> Self { - Self { inner: self.inner.clone() } + Self { inner: self.inner.clone(), log: self.log.clone() } } } @@ -262,7 +261,10 @@ impl ZipTransport { } } - Ok(Self { inner: Arc::new(Inner { archive, entries }) }) + Ok(Self { + inner: Arc::new(Inner { archive, entries }), + log: log.clone(), + }) } fn stream( @@ -272,11 +274,12 @@ impl ZipTransport { ) -> impl Stream> { // This could be a lot nicer but the ZIP entry reader can't cross in/out // of `spawn_blocking` because the type has an associated lifetime. - let (outer_tx, mut outer_rx) = mpsc::channel::< - oneshot::Sender>, - >(1); + let (tx, mut rx) = mpsc::channel::>(1); + let cloned_url = url.clone(); let task = tokio::task::spawn_blocking(move || { - let Some(inner_tx) = outer_rx.blocking_recv() else { return }; + type SendError = + mpsc::error::SendError>; + let mut reader = match self .inner .archive @@ -293,55 +296,55 @@ impl ZipTransport { }) { Ok(reader) => reader, Err(error) => { - inner_tx.send(Err(error)).ok(); - return; + tx.blocking_send(Err(error.into_tough_error(url)))?; + return Ok::<_, SendError>(()); } }; - let mut buf = BytesMut::new(); - let mut first_tx = Some(inner_tx); - while let Some(inner_tx) = - first_tx.take().or_else(|| outer_rx.blocking_recv()) - { + let mut buf = BytesMut::with_capacity(8192); + loop { if buf.capacity() == 0 { buf.reserve(8192); } buf.resize(buf.capacity(), 0); - let result = match reader.read(&mut buf) { - Ok(0) => break, + match reader.read(&mut buf) { + Ok(0) => return Ok::<_, SendError>(()), Ok(n) => { buf.truncate(n); - Ok(buf.split().freeze()) + tx.blocking_send(Ok(buf.split().freeze()))?; + } + Err(error) => { + tx.blocking_send(Err(ZipTransportError::from(error) + .into_tough_error(url)))?; + return Ok::<_, SendError>(()); } - Err(error) => Err(error.into()), - }; - let Ok(()) = inner_tx.send(result) else { break }; + } } }); - stream::try_unfold( - (outer_tx, task, url), - async |(outer_tx, task, url)| { - let (inner_tx, inner_rx) = oneshot::channel(); - match outer_tx - .send(inner_tx) - .map_err(|_send_error| ()) - .and_then(|()| inner_rx.map_err(|_recv_error| ())) - .await - { - Ok(Ok(bytes)) => Ok(Some((bytes, (outer_tx, task, url)))), - Ok(Err(error)) => Err(error.into_tough_error(url)), - Err(()) => { - // The task hung up. If we're still here, then the task - // reached EOF. await the task and return. - match task.await { - Ok(()) => Ok(None), - Err(error) => Err(ZipTransportError::from(error) - .into_tough_error(url)), - } + + tokio::task::spawn(async move { + if let Ok(Err(send_error)) = task.await { + match send_error.0 { + Ok(_) => { + error!( + self.log, + "zip file reader hung up mid-stream"; + "url" => &cloned_url.to_string(), + ); + } + Err(err) => { + error!( + self.log, + "zip file reader hung up mid-stream \ + before receiving error"; + "url" => &cloned_url.to_string(), + "err" => &crate::util::error_chain(&err), + ); } } - }, - ) + } + }); + futures_util::stream::poll_fn(move |cx| rx.poll_recv(cx)) } } From 479e433c59ee035cca290fc781e7dca460f4b8b9 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 15 Jun 2026 12:53:06 -0700 Subject: [PATCH 108/137] redo trust store options, implement in bin --- bin/src/edit.rs | 15 ++-- bin/src/list_targets.rs | 16 ++-- bin/src/load.rs | 146 ++++++++++++++++++++++++++++++++++ bin/src/main.rs | 1 + bin/src/show_target.rs | 16 ++-- bin/src/sign.rs | 45 +++++++---- bin/src/verify.rs | 16 ++-- lib/src/edit/editor.rs | 13 +-- lib/src/edit/sign.rs | 15 ++-- lib/src/loader.rs | 72 ++++++++++------- lib/src/repo.rs | 7 +- lib/tests/check.rs | 8 +- lib/tests/tests.rs | 49 +++++------- lib/tests/v1_compatibility.rs | 5 +- 14 files changed, 294 insertions(+), 130 deletions(-) create mode 100644 bin/src/load.rs diff --git a/bin/src/edit.rs b/bin/src/edit.rs index a431229..c4d59f8 100644 --- a/bin/src/edit.rs +++ b/bin/src/edit.rs @@ -7,18 +7,18 @@ use camino::Utf8PathBuf; use chrono::Utc; use clap::Parser; use semver::Version; -use tufaceous::ExpirationEnforcement; -use tufaceous::RepositoryLoader; -use tufaceous::TrustStoreBehavior; use tufaceous::edit::RepositoryEditor; use tufaceous_artifact::KnownArtifactTags; +use crate::load::LoadOptions; use crate::sign::SignOptions; #[derive(Debug, Parser)] pub struct Args { #[arg(short = 'a', long, num_args(1..))] add_artifacts: Vec, + #[clap(flatten)] + load_options: LoadOptions, #[clap(long)] no_installinator_document: bool, #[clap(short = 'o', long)] @@ -34,11 +34,10 @@ pub struct Args { impl Args { pub async fn run(self) -> Result<()> { - let repo = RepositoryLoader::new() - // TODO after v2 merge: Create a LoadOptions struct for configuring - // expiration enforcement and trust store behavior. - .expiration_enforcement(ExpirationEnforcement::Unsafe) - .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + let repo = self + .load_options + .loader() + .await? .load_zip_path(self.repo.clone(), &crate::LOG) .await?; let mut editor = RepositoryEditor::from_repo(&repo)? diff --git a/bin/src/list_targets.rs b/bin/src/list_targets.rs index 14b1956..393657d 100644 --- a/bin/src/list_targets.rs +++ b/bin/src/list_targets.rs @@ -7,22 +7,22 @@ use std::collections::BTreeSet; use anyhow::Result; use camino::Utf8PathBuf; use clap::Parser; -use tufaceous::ExpirationEnforcement; -use tufaceous::RepositoryLoader; -use tufaceous::TrustStoreBehavior; + +use crate::load::LoadOptions; #[derive(Debug, Parser)] pub struct Args { + #[clap(flatten)] + load_options: LoadOptions, repo: Utf8PathBuf, } impl Args { pub async fn run(self) -> Result<()> { - let repo = RepositoryLoader::new() - // TODO after v2 merge: Create a LoadOptions struct for configuring - // expiration enforcement and trust store behavior. - .expiration_enforcement(ExpirationEnforcement::Unsafe) - .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + let repo = self + .load_options + .loader() + .await? .load_zip_path(self.repo.clone(), &crate::LOG) .await?; let target_names = repo.targets().keys().collect::>(); diff --git a/bin/src/load.rs b/bin/src/load.rs new file mode 100644 index 0000000..fe62b72 --- /dev/null +++ b/bin/src/load.rs @@ -0,0 +1,146 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use camino::Utf8PathBuf; +use clap::Args; +use clap::builder::ArgPredicate; +use tufaceous::ExpirationEnforcement; +use tufaceous::RepositoryLoader; +use tufaceous::error::Error; +use tufaceous::error::ErrorKind; + +#[derive(Debug, Args)] +#[cfg_attr(test, derive(PartialEq))] +pub struct LoadOptions { + /// Allow loading a repository with an expired signature. + #[clap( + long, + default_value_if("force_load", ArgPredicate::IsPresent, "true") + )] + allow_expired: bool, + + /// Blindly trust whatever root is contained in the repository. + /// + /// This reads the trust root by fetching metadata/1.root.json from the + /// repository, then proceeds with normal verification. The repository must + /// be validly signed, but with this set we do not care who signed it. + #[clap( + long, + default_value_if("force_load", ArgPredicate::IsPresent, "true") + )] + blindly_trust: bool, + + /// Shorthand for --allow-expired --blindly-trust. + #[clap(short('f'), long)] + force_load: bool, + + /// Provide a trust root that can be used to verify the repository. + /// + /// This flag is required unless --blindly-trust or --force-load is set. + #[clap( + short('r'), + long("trust-roots"), + conflicts_with("blindly_trust"), + required_unless_present_any(["blindly_trust", "force_load"]) + )] + trust_roots: Vec, +} + +impl LoadOptions { + pub async fn loader(self) -> Result { + let mut loader = RepositoryLoader::new(); + for trust_root in self.trust_roots { + let root = + tokio::fs::read(&trust_root).await.map_err(|source| { + ErrorKind::ReadFile { source, path: Some(trust_root) } + })?; + loader = loader.trust_root(root); + } + if self.allow_expired { + loader = + loader.expiration_enforcement(ExpirationEnforcement::Unsafe); + } + if self.blindly_trust { + loader = loader.unsafe_blindly_trust_repo(); + } + Ok(loader) + } +} + +#[cfg(test)] +mod tests { + use clap::Parser; + + use crate::load::LoadOptions; + + #[derive(Debug, Parser)] + struct Args { + #[clap(flatten)] + inner: LoadOptions, + } + + #[test] + fn test_force_load() { + assert_eq!( + Args::try_parse_from(["", "-f"]).unwrap().inner, + LoadOptions { + allow_expired: true, + blindly_trust: true, + force_load: true, + trust_roots: vec![], + } + ); + // ... but they default to false if --force-load is not used: + assert_eq!( + Args::try_parse_from(["", "-r", "/dev/null"]).unwrap().inner, + LoadOptions { + allow_expired: false, + blindly_trust: false, + force_load: false, + trust_roots: vec!["/dev/null".into()], + } + ); + // --force-load doesn't conflict with --trust-roots + assert_eq!( + Args::try_parse_from(["", "-f", "-r", "/dev/null"]).unwrap().inner, + LoadOptions { + allow_expired: true, + blindly_trust: true, + force_load: true, + trust_roots: vec!["/dev/null".into()], + } + ); + } + + #[test] + fn test_trust_roots_required() { + let error = Args::try_parse_from([""]).unwrap_err().to_string(); + assert!( + error.contains( + "the following required arguments were not provided:" + ) + ); + assert!(error.contains("--trust-roots ")); + + // ... but not if --blindly-trust or --force-load is set! + Args::try_parse_from(["", "--blindly-trust"]).unwrap(); + Args::try_parse_from(["", "--force-load"]).unwrap(); + } + + #[test] + fn test_trust_roots_conflicts() { + let error = Args::try_parse_from([ + "", + "--trust-roots", + "/dev/null", + "--blindly-trust", + ]) + .unwrap_err() + .to_string(); + assert!(error.contains( + "the argument '--trust-roots ' \ + cannot be used with '--blindly-trust'" + )); + } +} diff --git a/bin/src/main.rs b/bin/src/main.rs index fa07b1b..6aba310 100644 --- a/bin/src/main.rs +++ b/bin/src/main.rs @@ -5,6 +5,7 @@ mod build; mod edit; mod list_targets; +mod load; mod show_target; mod sign; mod verify; diff --git a/bin/src/show_target.rs b/bin/src/show_target.rs index ae3888b..6c1032e 100644 --- a/bin/src/show_target.rs +++ b/bin/src/show_target.rs @@ -9,23 +9,23 @@ use anyhow::Result; use camino::Utf8PathBuf; use clap::Parser; use futures_util::TryStreamExt; -use tufaceous::ExpirationEnforcement; -use tufaceous::RepositoryLoader; -use tufaceous::TrustStoreBehavior; + +use crate::load::LoadOptions; #[derive(Debug, Parser)] pub struct Args { + #[clap(flatten)] + load_options: LoadOptions, repo: Utf8PathBuf, target_name: String, } impl Args { pub async fn run(self) -> Result<()> { - let repo = RepositoryLoader::new() - // TODO after v2 merge: Create a LoadOptions struct for configuring - // expiration enforcement and trust store behavior. - .expiration_enforcement(ExpirationEnforcement::Unsafe) - .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + let repo = self + .load_options + .loader() + .await? .load_zip_path(self.repo.clone(), &crate::LOG) .await?; let mut stream = repo.read_target(&self.target_name).await?; diff --git a/bin/src/sign.rs b/bin/src/sign.rs index ed9c4c9..e0a43bc 100644 --- a/bin/src/sign.rs +++ b/bin/src/sign.rs @@ -4,20 +4,25 @@ use anyhow::Result; use camino::Utf8PathBuf; -use clap::Parser; +use clap::Args; use tough::key_source::LocalKeySource; use tufaceous::edit::SignedRepository; use tufaceous::edit::UnsignedRepository; use tufaceous::error::Error; use tufaceous::error::ErrorKind; -#[derive(Debug, Parser)] +#[derive(Debug, Args)] #[cfg_attr(test, derive(PartialEq))] pub struct SignOptions { + /// Refuse to generate a signing root. #[clap(long)] no_generate_root: bool, + + /// Path to the signing root [default: generate a root] #[clap(long, required_if_eq("no_generate_root", "true"))] - root: Option, + signing_root: Option, + + /// Path to a key listed in the signing root. #[clap(long)] key: Vec, } @@ -30,7 +35,7 @@ impl SignOptions { if !self.no_generate_root { unsigned = unsigned.generate_root(); } - if let Some(path) = self.root { + if let Some(path) = self.signing_root { let root = match tokio::fs::read(&path).await { Ok(root) => root, Err(source) => { @@ -57,39 +62,47 @@ mod tests { use crate::sign::SignOptions; + #[derive(Parser)] + struct Args { + #[clap(flatten)] + inner: SignOptions, + } + #[test] fn test_flags() { assert_eq!( - SignOptions::try_parse_from([""]).unwrap(), - SignOptions { no_generate_root: false, root: None, key: vec![] } - ); - assert!( - SignOptions::try_parse_from(["", "--no-generate-root"]).is_err() + Args::try_parse_from([""]).unwrap().inner, + SignOptions { + no_generate_root: false, + signing_root: None, + key: vec![] + } ); - let mut args = vec!["", "--root", "root.json"]; + assert!(Args::try_parse_from(["", "--no-generate-root"]).is_err()); + let mut args = vec!["", "--signing-root", "root.json"]; assert_eq!( - SignOptions::try_parse_from(&args).unwrap(), + Args::try_parse_from(&args).unwrap().inner, SignOptions { no_generate_root: false, - root: Some("root.json".into()), + signing_root: Some("root.json".into()), key: vec![], } ); args.extend(["--key", "key.pem"]); assert_eq!( - SignOptions::try_parse_from(&args).unwrap(), + Args::try_parse_from(&args).unwrap().inner, SignOptions { no_generate_root: false, - root: Some("root.json".into()), + signing_root: Some("root.json".into()), key: vec!["key.pem".into()], } ); args.extend(["--key", "key2.pem"]); assert_eq!( - SignOptions::try_parse_from(&args).unwrap(), + Args::try_parse_from(&args).unwrap().inner, SignOptions { no_generate_root: false, - root: Some("root.json".into()), + signing_root: Some("root.json".into()), key: vec!["key.pem".into(), "key2.pem".into()], } ); diff --git a/bin/src/verify.rs b/bin/src/verify.rs index f4cb29f..57eede8 100644 --- a/bin/src/verify.rs +++ b/bin/src/verify.rs @@ -9,13 +9,14 @@ use anyhow::Result; use anyhow::ensure; use camino::Utf8PathBuf; use clap::Parser; -use tough::ExpirationEnforcement; use tufaceous::CheckProblem; -use tufaceous::RepositoryLoader; -use tufaceous::TrustStoreBehavior; + +use crate::load::LoadOptions; #[derive(Debug, Parser)] pub struct Args { + #[clap(flatten)] + load_options: LoadOptions, /// Path to the repository repo: Utf8PathBuf, /// Number of threads to use while verifying targets @@ -31,11 +32,10 @@ fn default_threads() -> usize { impl Args { pub async fn run(self) -> Result<()> { - let repo = RepositoryLoader::new() - // TODO after v2 merge: Create a LoadOptions struct for configuring - // expiration enforcement and trust store behavior. - .expiration_enforcement(ExpirationEnforcement::Unsafe) - .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + let repo = self + .load_options + .loader() + .await? .v1_compatibility(true) .load_zip_path(self.repo.clone(), &crate::LOG) .await?; diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index dbe9410..6134b6c 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -550,7 +550,6 @@ mod tests { use tufaceous_artifact::ArtifactVersion; use crate::RepositoryLoader; - use crate::TrustStoreBehavior; use crate::edit::RepositoryEditor; use crate::edit::source::BytesSource; use crate::error::Error; @@ -580,16 +579,10 @@ mod tests { }, ); - let zip = editor - .finish() - .await? - .generate_root() - .sign() - .await? - .write_zip(Vec::new(), Utc::now()) - .await?; + let signed = editor.finish().await?.generate_root().sign().await?; + let zip = signed.write_zip(Vec::new(), Utc::now()).await?; let repo = RepositoryLoader::new() - .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .trust_root(signed.root()) .load_zip_buffer(zip, &log) .await?; let artifacts = repo.artifacts().iter().collect::>(); diff --git a/lib/src/edit/sign.rs b/lib/src/edit/sign.rs index 46ea4b3..06ba974 100644 --- a/lib/src/edit/sign.rs +++ b/lib/src/edit/sign.rs @@ -198,12 +198,13 @@ impl<'a> UnsignedRepository<'a> { ); } - Ok(SignedRepository { sources }) + Ok(SignedRepository { root, sources }) } } #[derive(Debug)] pub struct SignedRepository<'a> { + root: Vec, sources: BTreeMap<(FilePrefix, String), TargetSource<'a>>, } @@ -214,8 +215,12 @@ enum FilePrefix { } impl SignedRepository<'_> { + pub fn root(&self) -> &[u8] { + &self.root + } + pub async fn write_zip( - &mut self, + &self, writer: W, modification_time: DateTime, ) -> Result { @@ -224,7 +229,7 @@ impl SignedRepository<'_> { } pub async fn write_zip_file( - &mut self, + &self, path: impl AsRef, modification_time: DateTime, ) -> Result<(), Error> { @@ -238,14 +243,14 @@ impl SignedRepository<'_> { } async fn write_zip_impl( - &mut self, + &self, mut writer: ZipWriter, modification_time: DateTime, archive_path: Option<&Utf8Path>, ) -> Result { let last_modified = UtcDateTime::from_unix(modification_time.timestamp()); - 'outer: for ((prefix, name), source) in &mut self.sources { + 'outer: for ((prefix, name), source) in &self.sources { let prefix = Utf8Path::new(match prefix { FilePrefix::Metadata => "repo/metadata", FilePrefix::Targets => "repo/targets", diff --git a/lib/src/loader.rs b/lib/src/loader.rs index dd2f36e..408bc9e 100644 --- a/lib/src/loader.rs +++ b/lib/src/loader.rs @@ -25,21 +25,6 @@ use crate::error::Error; use crate::error::ErrorKind; use crate::error::try_path; -/// Controls how repository trust is established. -/// -/// Used in [`RepositoryLoader::trust_store_behavior`]. -#[derive(Debug, Clone, Copy, Default)] -pub enum TrustStoreBehavior { - /// Verification by the provided trust roots is required. - /// - /// This is the default behavior. - #[default] - Required, - /// The trust root is established by fetching `metadata/1.root.json` from - /// the repository. Verification then proceeds normally. - UnsafeBlindFaith, -} - #[derive(Debug, Clone, Default)] #[must_use] pub struct RepositoryLoader { @@ -48,11 +33,22 @@ pub struct RepositoryLoader { limits: Limits, metadata_base_url: Option, targets_base_url: Option, - trust_roots: Vec>, - trust_store_behavior: TrustStoreBehavior, + trust_store: TrustStore, v1_compatibility: bool, } +#[derive(Debug, Clone)] +enum TrustStore { + UnsafeBlindTrust, + Store(Vec>), +} + +impl Default for TrustStore { + fn default() -> Self { + Self::Store(Vec::new()) + } +} + impl RepositoryLoader { /// Constructs a new `RepositoryLoader`. /// @@ -95,7 +91,15 @@ impl RepositoryLoader { /// Add a trusted root role to the trust store. pub fn trust_root(mut self, trust_root: impl AsRef<[u8]>) -> Self { - self.trust_roots.push(trust_root.as_ref().into()); + match self.trust_store { + TrustStore::Store(ref mut store) => { + store.push(trust_root.as_ref().into()); + } + TrustStore::UnsafeBlindTrust => { + self.trust_store = + TrustStore::Store(vec![trust_root.as_ref().into()]); + } + } self } @@ -105,16 +109,28 @@ impl RepositoryLoader { trust_roots: impl IntoIterator>, ) -> Self { let iter = trust_roots.into_iter().map(|root| root.as_ref().into()); - self.trust_roots.extend(iter); + match self.trust_store { + TrustStore::Store(ref mut store) => { + store.extend(iter); + } + TrustStore::UnsafeBlindTrust => { + self.trust_store = TrustStore::Store(iter.collect()); + } + } self } - /// Set how repository trust is established. - pub fn trust_store_behavior( - self, - trust_store_behavior: TrustStoreBehavior, - ) -> Self { - Self { trust_store_behavior, ..self } + /// Blindly trust whatever root is contained in the repository. + /// + /// This reads the trust root by fetching `metadata/1.root.json` from the + /// repository, then proceeds with normal verification. The repository must + /// be validly signed, but with this set we do not care who signed it. + /// + /// Calling this method removes any trust roots added via + /// [`Self::trust_root`] and [`Self::trust_roots`], and calling either of + /// those methods to add a trust root reverts the effect of this method. + pub fn unsafe_blindly_trust_repo(self) -> Self { + Self { trust_store: TrustStore::UnsafeBlindTrust, ..self } } /// Enable compatibility with v1-format repositories. @@ -282,9 +298,9 @@ impl RepositoryLoader { return Err(ErrorKind::TargetsBaseUrlUnset.into()); }; - let trust_roots = match self.trust_store_behavior { - TrustStoreBehavior::Required => self.trust_roots, - TrustStoreBehavior::UnsafeBlindFaith => { + let trust_roots = match self.trust_store { + TrustStore::Store(trust_roots) => trust_roots, + TrustStore::UnsafeBlindTrust => { let root_path = "1.root.json"; let root_url = metadata_base_url.join(root_path).map_err(|source| { diff --git a/lib/src/repo.rs b/lib/src/repo.rs index 279ba42..b1faed0 100644 --- a/lib/src/repo.rs +++ b/lib/src/repo.rs @@ -95,16 +95,15 @@ impl Repository { system_version: Version, log: &Logger, ) -> Result { - let zip = crate::edit::RepositoryEditor::fake(system_version)? + let signed = crate::edit::RepositoryEditor::fake(system_version)? .finish() .await? .generate_root() .sign() - .await? - .write_zip(Vec::new(), chrono::Utc::now()) .await?; + let zip = signed.write_zip(Vec::new(), chrono::Utc::now()).await?; RepositoryLoader::new() - .trust_store_behavior(crate::TrustStoreBehavior::UnsafeBlindFaith) + .trust_root(signed.root()) .load_zip_buffer(zip, log) .await } diff --git a/lib/tests/check.rs b/lib/tests/check.rs index af0c582..91694a4 100644 --- a/lib/tests/check.rs +++ b/lib/tests/check.rs @@ -7,7 +7,6 @@ use semver::Version; use tufaceous::CheckProblem; use tufaceous::Repository; use tufaceous::RepositoryLoader; -use tufaceous::TrustStoreBehavior; use tufaceous::edit::RepositoryEditor; use tufaceous::error::Error; use tufaceous_artifact::KnownArtifactTags; @@ -26,17 +25,16 @@ async fn fake_checks_out() -> Result<(), Error> { #[tokio::test] async fn missing_installinator() -> Result<(), Error> { let log = slog::Logger::root(slog::Discard, slog::o!()); - let zip = RepositoryEditor::fake(V1)? + let signed = RepositoryEditor::fake(V1)? .set_generate_installinator_document(false) .finish() .await? .generate_root() .sign() - .await? - .write_zip(Vec::new(), Utc::now()) .await?; + let zip = signed.write_zip(Vec::new(), Utc::now()).await?; let repo = RepositoryLoader::new() - .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .trust_root(signed.root()) .load_zip_buffer(zip, &log) .await?; let problems = repo.check_problems().await; diff --git a/lib/tests/tests.rs b/lib/tests/tests.rs index e94b395..4f1f5e6 100644 --- a/lib/tests/tests.rs +++ b/lib/tests/tests.rs @@ -13,7 +13,6 @@ use futures_util::TryStreamExt; use futures_util::stream; use semver::Version; use tufaceous::RepositoryLoader; -use tufaceous::TrustStoreBehavior; use tufaceous::edit::RepositoryEditor; use tufaceous::error::Error; use tufaceous_artifact::KnownArtifactTags; @@ -24,16 +23,15 @@ const V2: Version = Version::new(2, 0, 0); #[tokio::test] async fn it_works() -> Result<(), Error> { let log = slog::Logger::root(slog::Discard, slog::o!()); - let zip = RepositoryEditor::fake(V1)? + let signed = RepositoryEditor::fake(V1)? .finish() .await? .generate_root() .sign() - .await? - .write_zip(Vec::new(), Utc::now()) .await?; + let zip = signed.write_zip(Vec::new(), Utc::now()).await?; let repo = RepositoryLoader::new() - .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .trust_root(signed.root()) .load_zip_buffer(zip, &log) .await?; for artifact in repo.artifacts() { @@ -49,16 +47,15 @@ async fn it_works() -> Result<(), Error> { #[tokio::test] async fn verify_targets() -> Result<(), Error> { let log = slog::Logger::root(slog::Discard, slog::o!()); - let mut zip = RepositoryEditor::fake(V1)? + let signed = RepositoryEditor::fake(V1)? .finish() .await? .generate_root() .sign() - .await? - .write_zip(Vec::new(), Utc::now()) .await?; + let mut zip = signed.write_zip(Vec::new(), Utc::now()).await?; let repo = RepositoryLoader::new() - .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .trust_root(signed.root()) .load_zip_buffer(zip.clone(), &log) .await?; let repo = Arc::new(repo); @@ -70,7 +67,7 @@ async fn verify_targets() -> Result<(), Error> { .expect("b\"hubris\" not found in archive"); zip[pos] = b'H'; // flip! that! bit! let repo = RepositoryLoader::new() - .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .trust_root(signed.root()) .load_zip_buffer(zip.clone(), &log) .await?; let repo = Arc::new(repo); @@ -84,17 +81,16 @@ async fn verify_targets() -> Result<(), Error> { #[tokio::test] async fn no_artifacts() -> Result<(), Error> { let log = slog::Logger::root(slog::Discard, slog::o!()); - let zip = RepositoryEditor::new(V1)? + let signed = RepositoryEditor::new(V1)? .set_generate_installinator_document(false) .finish() .await? .generate_root() .sign() - .await? - .write_zip(Vec::new(), Utc::now()) .await?; + let zip = signed.write_zip(Vec::new(), Utc::now()).await?; let repo = RepositoryLoader::new() - .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .trust_root(signed.root()) .load_zip_buffer(zip, &log) .await?; assert_eq!(repo.artifacts().len(), 0); @@ -105,32 +101,30 @@ async fn no_artifacts() -> Result<(), Error> { #[tokio::test] async fn inconsistent_fake() -> Result<(), Error> { let log = slog::Logger::root(slog::Discard, slog::o!()); - let zip = RepositoryEditor::fake(V1)? + let signed = RepositoryEditor::fake(V1)? .finish() .await? .generate_root() .sign() - .await? - .write_zip(Vec::new(), Utc::now()) .await?; + let zip = signed.write_zip(Vec::new(), Utc::now()).await?; let repo1 = RepositoryLoader::new() - .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .trust_root(signed.root()) .load_zip_buffer(zip, &log) .await?; let artifact_v1 = V1.to_string().parse()?; let artifact_v2 = V2.to_string().parse()?; - let zip = + let signed = RepositoryEditor::inconsistent_fake(V2, &artifact_v2, &artifact_v1)? .finish() .await? .generate_root() .sign() - .await? - .write_zip(Vec::new(), Utc::now()) .await?; + let zip = signed.write_zip(Vec::new(), Utc::now()).await?; let repo2 = RepositoryLoader::new() - .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .trust_root(signed.root()) .load_zip_buffer(zip, &log) .await?; @@ -160,12 +154,13 @@ async fn inconsistent_fake() -> Result<(), Error> { #[tokio::test] async fn compute_archive_sha256() -> Result<(), Error> { let log = slog::Logger::root(slog::Discard, slog::o!()); - let zip = RepositoryEditor::fake(V1)? + let signed = RepositoryEditor::fake(V1)? .finish() .await? .generate_root() .sign() - .await? + .await?; + let zip = signed .write_zip(BytesMut::new().writer(), Utc::now()) .await? .into_inner() @@ -173,7 +168,7 @@ async fn compute_archive_sha256() -> Result<(), Error> { for should_compute in [false, true] { let repo = RepositoryLoader::new() .compute_archive_sha256(should_compute) - .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .trust_root(signed.root()) .load_zip_buffer(zip.clone(), &log) .await?; assert_eq!(repo.archive_sha256().is_some(), should_compute); @@ -182,14 +177,14 @@ async fn compute_archive_sha256() -> Result<(), Error> { file.write_all(&zip).unwrap(); let repo = RepositoryLoader::new() .compute_archive_sha256(should_compute) - .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .trust_root(signed.root()) .load_zip_file(file, None, &log) .await?; assert_eq!(repo.archive_sha256().is_some(), should_compute); let repo = RepositoryLoader::new() .compute_archive_sha256(should_compute) - .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .trust_root(signed.root()) .load_zip_stream( stream::once(async { Ok::<_, Infallible>(zip.clone()) }), None, diff --git a/lib/tests/v1_compatibility.rs b/lib/tests/v1_compatibility.rs index 865ad33..7d612ee 100644 --- a/lib/tests/v1_compatibility.rs +++ b/lib/tests/v1_compatibility.rs @@ -5,7 +5,6 @@ use camino::Utf8Path; use tufaceous::ExpirationEnforcement; use tufaceous::RepositoryLoader; -use tufaceous::TrustStoreBehavior; use tufaceous::error::Error; use tufaceous_artifact::ArtifactSet; use tufaceous_artifact::KnownArtifactTags; @@ -28,7 +27,7 @@ async fn v1_fake() -> Result<(), Error> { let repo = RepositoryLoader::new() .expiration_enforcement(ExpirationEnforcement::Unsafe) - .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .unsafe_blindly_trust_repo() .v1_compatibility(true) .load_zip_path(path, &log) .await?; @@ -90,7 +89,7 @@ async fn v1_fake_without_compat() { let _ = RepositoryLoader::new() .expiration_enforcement(ExpirationEnforcement::Unsafe) - .trust_store_behavior(TrustStoreBehavior::UnsafeBlindFaith) + .unsafe_blindly_trust_repo() .v1_compatibility(false) .load_zip_path(path, &log) .await From 1a4bb85923401079a712d7d112fc6381022e93fb Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 15 Jun 2026 13:04:35 -0700 Subject: [PATCH 109/137] bin docs pass --- bin/src/build.rs | 14 +++++++++++--- bin/src/edit.rs | 19 ++++++++++++++++--- bin/src/list_targets.rs | 1 + bin/src/load.rs | 4 ++-- bin/src/main.rs | 3 +++ bin/src/show_target.rs | 5 +++++ bin/src/sign.rs | 4 ++-- bin/src/verify.rs | 2 +- 8 files changed, 41 insertions(+), 11 deletions(-) diff --git a/bin/src/build.rs b/bin/src/build.rs index 420de3f..7cc8ecd 100644 --- a/bin/src/build.rs +++ b/bin/src/build.rs @@ -13,25 +13,33 @@ use crate::sign::SignOptions; #[derive(Debug, Parser)] pub struct Args { + /// Artifacts to add to the repository + /// + /// For OS images, use the directory generated by helios-build that contains + /// zfs.img, gimlet.rom, etc. artifacts: Vec, + /// Add a standard set of fake artifacts #[clap(long)] fake: bool, + /// Do not generate an Installinator document #[clap(long)] no_installinator_document: bool, + /// ZIP archive output path #[clap(short = 'o', long)] output: Utf8PathBuf, #[clap(flatten)] sign_options: SignOptions, + /// The system version [required] #[clap(short = 'V', long)] - version: Version, + system_version: Version, } impl Args { pub async fn run(self) -> Result<()> { let mut editor = if self.fake { - RepositoryEditor::fake(self.version) + RepositoryEditor::fake(self.system_version) } else { - RepositoryEditor::new(self.version) + RepositoryEditor::new(self.system_version) }?; editor = editor.set_generate_installinator_document( !self.no_installinator_document, diff --git a/bin/src/edit.rs b/bin/src/edit.rs index c4d59f8..79bed69 100644 --- a/bin/src/edit.rs +++ b/bin/src/edit.rs @@ -15,21 +15,34 @@ use crate::sign::SignOptions; #[derive(Debug, Parser)] pub struct Args { + /// Artifacts to add to the repository + /// + /// For OS images, use the directory generated by helios-build that contains + /// zfs.img, gimlet.rom, etc. #[arg(short = 'a', long, num_args(1..))] add_artifacts: Vec, #[clap(flatten)] load_options: LoadOptions, + /// Do not generate a new Installinator document, and leave the existing + /// Installinator document if one exists #[clap(long)] no_installinator_document: bool, + /// ZIP archive output path [default: rewrite input repository] #[clap(short = 'o', long)] output: Option, + /// Targets to remove from the repository + /// + /// These are the paths under "repo/targets/" in a ZIP archive. The + /// list-targets command lists the target names. #[arg(short = 'd', long, num_args(1..))] remove_targets: Vec, + /// Input repository path repo: Utf8PathBuf, #[clap(flatten)] sign_options: SignOptions, + /// Edit the system version #[clap(short = 'V', long)] - version: Option, + system_version: Option, } impl Args { @@ -50,8 +63,8 @@ impl Args { .remove_artifacts(&KnownArtifactTags::InstallinatorDocument)?; } - if let Some(version) = self.version { - editor = editor.set_system_version(version)?; + if let Some(system_version) = self.system_version { + editor = editor.set_system_version(system_version)?; } for target_name in &self.remove_targets { editor = editor.remove_target(target_name); diff --git a/bin/src/list_targets.rs b/bin/src/list_targets.rs index 393657d..6ad8568 100644 --- a/bin/src/list_targets.rs +++ b/bin/src/list_targets.rs @@ -14,6 +14,7 @@ use crate::load::LoadOptions; pub struct Args { #[clap(flatten)] load_options: LoadOptions, + /// Input repository path repo: Utf8PathBuf, } diff --git a/bin/src/load.rs b/bin/src/load.rs index fe62b72..77bfc58 100644 --- a/bin/src/load.rs +++ b/bin/src/load.rs @@ -20,7 +20,7 @@ pub struct LoadOptions { )] allow_expired: bool, - /// Blindly trust whatever root is contained in the repository. + /// Blindly trust whatever root is contained in the repository /// /// This reads the trust root by fetching metadata/1.root.json from the /// repository, then proceeds with normal verification. The repository must @@ -35,7 +35,7 @@ pub struct LoadOptions { #[clap(short('f'), long)] force_load: bool, - /// Provide a trust root that can be used to verify the repository. + /// Trust roots to verify the repository /// /// This flag is required unless --blindly-trust or --force-load is set. #[clap( diff --git a/bin/src/main.rs b/bin/src/main.rs index 6aba310..34812fe 100644 --- a/bin/src/main.rs +++ b/bin/src/main.rs @@ -38,6 +38,9 @@ enum Command { /// Build a new repository from a set of artifacts Build(build::Args), /// Add artifacts to and remove targets from a repository + /// + /// Calling this with no options will re-sign the repository with a + /// newly-generated root and overwrite it. Edit(edit::Args), /// List the targets in a repository #[clap(alias = "ls")] diff --git a/bin/src/show_target.rs b/bin/src/show_target.rs index 6c1032e..b633198 100644 --- a/bin/src/show_target.rs +++ b/bin/src/show_target.rs @@ -16,7 +16,12 @@ use crate::load::LoadOptions; pub struct Args { #[clap(flatten)] load_options: LoadOptions, + /// Input repository path repo: Utf8PathBuf, + /// Targets to output + /// + /// This is the path under "repo/targets/" in a ZIP archive. The + /// list-targets command lists the target names. target_name: String, } diff --git a/bin/src/sign.rs b/bin/src/sign.rs index e0a43bc..91326bf 100644 --- a/bin/src/sign.rs +++ b/bin/src/sign.rs @@ -14,7 +14,7 @@ use tufaceous::error::ErrorKind; #[derive(Debug, Args)] #[cfg_attr(test, derive(PartialEq))] pub struct SignOptions { - /// Refuse to generate a signing root. + /// Refuse to generate a signing root #[clap(long)] no_generate_root: bool, @@ -22,7 +22,7 @@ pub struct SignOptions { #[clap(long, required_if_eq("no_generate_root", "true"))] signing_root: Option, - /// Path to a key listed in the signing root. + /// Private signing key listed in the signing root #[clap(long)] key: Vec, } diff --git a/bin/src/verify.rs b/bin/src/verify.rs index 57eede8..155df85 100644 --- a/bin/src/verify.rs +++ b/bin/src/verify.rs @@ -17,7 +17,7 @@ use crate::load::LoadOptions; pub struct Args { #[clap(flatten)] load_options: LoadOptions, - /// Path to the repository + /// Input repository path repo: Utf8PathBuf, /// Number of threads to use while verifying targets #[clap(short = 'j', alias = "jobs", default_value_t = default_threads())] From 77d4820aa85e07d1c78e7f55883668025bfbd659 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 15 Jun 2026 13:14:12 -0700 Subject: [PATCH 110/137] require compression param in ZipWriter::new_file --- lib/src/edit/sign.rs | 3 +-- lib/src/zip_writer.rs | 8 ++------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/lib/src/edit/sign.rs b/lib/src/edit/sign.rs index 06ba974..6b38f47 100644 --- a/lib/src/edit/sign.rs +++ b/lib/src/edit/sign.rs @@ -261,8 +261,7 @@ impl SignedRepository<'_> { .as_deref() .map_or(Compression::none(), deflate_heuristic); let Ok(mut entry) = writer - .new_file(prefix.join(name)) - .compression(compression) + .new_file(prefix.join(name), compression) .last_modified(last_modified) .unix_permissions(0o644) .start() diff --git a/lib/src/zip_writer.rs b/lib/src/zip_writer.rs index ecde03d..98ba64c 100644 --- a/lib/src/zip_writer.rs +++ b/lib/src/zip_writer.rs @@ -51,13 +51,14 @@ impl ZipWriter { pub(crate) fn new_file( &mut self, name: Utf8PathBuf, + compression: Compression, ) -> ZipFileBuilder<'_, W> { let (bytes_tx, bytes_rx) = mpsc::channel(1); ZipFileBuilder { writer: self, inner: ZipFile { name: name.into(), - compression: Compression::none(), + compression, last_modified: None, unix_permissions: None, bytes_rx, @@ -79,11 +80,6 @@ pub(crate) struct ZipFileBuilder<'a, W> { } impl<'a, W> ZipFileBuilder<'a, W> { - pub(crate) fn compression(mut self, compression: Compression) -> Self { - self.inner.compression = compression; - self - } - pub(crate) fn last_modified(mut self, last_modified: ZipDateTime) -> Self { self.inner.last_modified = Some(last_modified); self From 80e93ab33f655bdc430e4bc1e77a9cadee60c822 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 15 Jun 2026 13:18:07 -0700 Subject: [PATCH 111/137] fix doc test after 479e433 --- lib/src/repo.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/src/repo.rs b/lib/src/repo.rs index b1faed0..89ed65d 100644 --- a/lib/src/repo.rs +++ b/lib/src/repo.rs @@ -75,17 +75,16 @@ impl Repository { /// # let system_version = const { semver::Version::new(1, 0, 0) }; /// # let log = slog::Logger::root(slog::Discard, slog::o!()); /// # let log = &log; - /// let zip = tufaceous::edit::RepositoryEditor::fake(system_version)? + /// let signed = tufaceous::edit::RepositoryEditor::fake(system_version)? /// .finish() /// .await? /// .generate_root() /// .sign() - /// .await? - /// .write_zip(Vec::new(), chrono::Utc::now()) /// .await?; + /// let zip = signed.write_zip(Vec::new(), chrono::Utc::now()).await?; /// # Ok::<_, tufaceous::error::Error>( /// tufaceous::RepositoryLoader::new() - /// .trust_store_behavior(tufaceous::TrustStoreBehavior::UnsafeBlindFaith) + /// .trust_root(signed.root()) /// .load_zip_buffer(zip, log) /// .await? /// # ) From ec5bb32f93258fad43cbc3e45549c950847607ce Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 15 Jun 2026 19:43:42 -0700 Subject: [PATCH 112/137] simplify the zip writer, moving it to edit::sign --- lib/src/edit/sign.rs | 219 +++++++++++++++++++++++++++++++++++++----- lib/src/lib.rs | 1 - lib/src/zip_writer.rs | 212 ---------------------------------------- 3 files changed, 194 insertions(+), 238 deletions(-) delete mode 100644 lib/src/zip_writer.rs diff --git a/lib/src/edit/sign.rs b/lib/src/edit/sign.rs index 6b38f47..f89e293 100644 --- a/lib/src/edit/sign.rs +++ b/lib/src/edit/sign.rs @@ -8,13 +8,23 @@ use std::io::Write; use std::num::NonZero; use std::time::Duration; +use atomicwrites::AtomicFile; +use atomicwrites::OverwriteBehavior; +use bytes::Bytes; use camino::Utf8Path; +use camino::Utf8PathBuf; use chrono::DateTime; use chrono::SubsecRound; use chrono::Utc; use flate2::Compression; +use flate2::write::DeflateEncoder; +use futures_util::StreamExt; use futures_util::TryStreamExt; +use rawzip::CompressionMethod; +use rawzip::ZipArchiveWriter; use rawzip::time::UtcDateTime; +use tokio::sync::mpsc; +use tokio::task::JoinHandle; use tough::key_source::KeySource; use tough::schema::Root; use tough::schema::Signed; @@ -27,7 +37,6 @@ use crate::edit::source::TargetSource; use crate::error::Error; use crate::error::ErrorKind; use crate::error::try_path; -use crate::zip_writer::ZipWriter; pub(crate) const DEFAULT_VALIDITY: Duration = Duration::from_secs(60 * 60 * 24 * 7 /* 1 week */); @@ -224,8 +233,11 @@ impl SignedRepository<'_> { writer: W, modification_time: DateTime, ) -> Result { - self.write_zip_impl(ZipWriter::new(writer), modification_time, None) - .await + let (tx, rx) = mpsc::channel(1); + let task = tokio::task::spawn_blocking(move || { + blocking_write_task(writer, rx, modification_time) + }); + self.write_zip_impl(tx, task, None).await } pub async fn write_zip_file( @@ -234,22 +246,23 @@ impl SignedRepository<'_> { modification_time: DateTime, ) -> Result<(), Error> { let path = path.as_ref(); - self.write_zip_impl( - ZipWriter::create(path), - modification_time, - Some(path), - ) - .await + let file = AtomicFile::new(path, OverwriteBehavior::AllowOverwrite); + let (tx, rx) = mpsc::channel(1); + let task = tokio::task::spawn_blocking(move || { + file.write(|file| { + blocking_write_task(file, rx, modification_time).map(|_| ()) + })?; + Ok(()) + }); + self.write_zip_impl(tx, task, Some(path)).await } async fn write_zip_impl( &self, - mut writer: ZipWriter, - modification_time: DateTime, + tx: mpsc::Sender, + task: JoinHandle>, archive_path: Option<&Utf8Path>, ) -> Result { - let last_modified = - UtcDateTime::from_unix(modification_time.timestamp()); 'outer: for ((prefix, name), source) in &self.sources { let prefix = Utf8Path::new(match prefix { FilePrefix::Metadata => "repo/metadata", @@ -260,24 +273,49 @@ impl SignedRepository<'_> { let compression = first_chunk .as_deref() .map_or(Compression::none(), deflate_heuristic); - let Ok(mut entry) = writer - .new_file(prefix.join(name), compression) - .last_modified(last_modified) - .unix_permissions(0o644) - .start() + + let Ok(()) = tx + .send(ZipWriterMessage::StartFile { + name: prefix.join(name), + compression, + }) .await else { break 'outer; }; - let Some(first_chunk) = first_chunk else { continue }; - let Ok(()) = entry.write(first_chunk).await else { break }; - while let Some(item) = stream.try_next().await? { - let Ok(()) = entry.write(item).await else { break 'outer }; + + let mut stream = + futures_util::stream::iter(first_chunk).map(Ok).chain(stream); + while let Some(chunk) = stream.try_next().await? { + let Ok(()) = + tx.send(ZipWriterMessage::WriteFileBytes(chunk)).await + else { + break 'outer; + }; } + let Ok(()) = tx.send(ZipWriterMessage::FinishFile).await else { + break 'outer; + }; } - writer.finish().await.map_err(|source| { - let archive_path = archive_path.map(Utf8Path::to_owned); - ErrorKind::WriteZip { source, archive_path }.into() + tx.send(ZipWriterMessage::FinishArchive).await.ok(); + task.await?.map_err(|error| { + let source = match error { + ZipWriterError::Zip(error) => error, + err @ (ZipWriterError::ExpectedFile + | ZipWriterError::ExpectedBytes) => { + std::io::Error::new(std::io::ErrorKind::InvalidInput, err) + .into() + } + err @ ZipWriterError::UnexpectedEof => { + std::io::Error::new(std::io::ErrorKind::UnexpectedEof, err) + .into() + } + }; + ErrorKind::WriteZip { + source, + archive_path: archive_path.map(Utf8Path::to_owned), + } + .into() }) } } @@ -319,3 +357,134 @@ fn deflate_heuristic(buf: &[u8]) -> Compression { Compression::best() } } + +enum ZipWriterMessage { + FinishArchive, + StartFile { name: Utf8PathBuf, compression: Compression }, + WriteFileBytes(Bytes), + FinishFile, +} + +#[derive(Debug, thiserror::Error)] +enum ZipWriterError { + #[error(transparent)] + Zip(#[from] rawzip::Error), + #[error("state machine violation: expected file start or archive finish")] + ExpectedFile, + #[error("state machine violation: expected file bytes or file finish")] + ExpectedBytes, + #[error("state machine violation: unexpected end of message stream")] + UnexpectedEof, +} + +impl From for ZipWriterError { + fn from(error: std::io::Error) -> Self { + ZipWriterError::Zip(error.into()) + } +} + +impl From> for ZipWriterError { + fn from(error: atomicwrites::Error) -> Self { + match error { + atomicwrites::Error::User(error) => error, + atomicwrites::Error::Internal(error) => error.into(), + } + } +} + +fn blocking_write_task( + writer: W, + mut rx: mpsc::Receiver, + modification_time: DateTime, +) -> Result { + let mut archive = ZipArchiveWriter::new(writer); + loop { + let message = + rx.blocking_recv().ok_or(ZipWriterError::UnexpectedEof)?; + let (name, compression) = match message { + ZipWriterMessage::FinishArchive => { + return Ok(archive.finish()?); + } + ZipWriterMessage::StartFile { name, compression } => { + (name, compression) + } + ZipWriterMessage::WriteFileBytes(_) + | ZipWriterMessage::FinishFile => { + return Err(ZipWriterError::ExpectedFile); + } + }; + + let (mut entry, config) = archive + .new_file(name.as_str()) + .compression_method(if compression == Compression::none() { + CompressionMethod::Store + } else { + CompressionMethod::Deflate + }) + .last_modified(UtcDateTime::from_unix( + modification_time.timestamp(), + )) + .unix_permissions(0o644) + .start()?; + let encoder = ZipEncoder::new(&mut entry, compression); + let mut writer = config.wrap(encoder); + + loop { + let message = + rx.blocking_recv().ok_or(ZipWriterError::UnexpectedEof)?; + match message { + ZipWriterMessage::WriteFileBytes(bytes) => { + writer.write_all(&bytes)?; + } + ZipWriterMessage::FinishFile => { + let (encoder, output) = writer.finish()?; + encoder.finish()?; + entry.finish(output)?; + break; + } + ZipWriterMessage::FinishArchive + | ZipWriterMessage::StartFile { .. } => { + return Err(ZipWriterError::ExpectedBytes); + } + } + } + } +} + +enum ZipEncoder { + Store(W), + Deflate(DeflateEncoder), +} + +impl ZipEncoder { + fn new(writer: W, compression: Compression) -> Self { + if compression == Compression::none() { + Self::Store(writer) + } else { + Self::Deflate(DeflateEncoder::new(writer, compression)) + } + } + + fn finish(self) -> std::io::Result { + match self { + Self::Store(writer) => Ok(writer), + Self::Deflate(encoder) => encoder.finish(), + } + } +} + +impl Write for ZipEncoder { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + match self { + ZipEncoder::Store(writer) => writer.write(buf), + ZipEncoder::Deflate(writer) => writer.write(buf), + } + } + + fn flush(&mut self) -> std::io::Result<()> { + match self { + ZipEncoder::Store(writer) => writer.flush(), + ZipEncoder::Deflate(writer) => writer.flush(), + } + } +} diff --git a/lib/src/lib.rs b/lib/src/lib.rs index bc6e2a4..92c0ac8 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -25,7 +25,6 @@ mod repo; mod schema; mod util; mod zip_transport; -mod zip_writer; pub use loader::*; pub use repo::*; diff --git a/lib/src/zip_writer.rs b/lib/src/zip_writer.rs deleted file mode 100644 index 98ba64c..0000000 --- a/lib/src/zip_writer.rs +++ /dev/null @@ -1,212 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use std::io::Write; - -use atomicwrites::AtomicFile; -use atomicwrites::OverwriteBehavior; -use bytes::Bytes; -use camino::Utf8Path; -use camino::Utf8PathBuf; -use flate2::Compression; -use flate2::write::DeflateEncoder; -use rawzip::CompressionMethod; -use rawzip::ZipArchiveWriter; -use rawzip::time::ZipDateTime; -use tokio::sync::mpsc; -use tokio::task::JoinHandle; - -pub(crate) struct ZipWriter { - task: JoinHandle>, - file_tx: mpsc::Sender, -} - -impl ZipWriter { - pub(crate) fn new(writer: W) -> Self { - let (file_tx, file_rx) = mpsc::channel(1); - let task = - tokio::task::spawn_blocking(move || write_task(writer, file_rx)); - Self { task, file_tx } - } -} - -impl ZipWriter<()> { - pub(crate) fn create(path: &Utf8Path) -> Self { - let file = AtomicFile::new(path, OverwriteBehavior::AllowOverwrite); - let (file_tx, file_rx) = mpsc::channel(1); - let task = tokio::task::spawn_blocking(move || { - file.write(|file| write_task(file, file_rx).map(|_| ())).map_err( - |error| match error { - atomicwrites::Error::User(error) => error, - atomicwrites::Error::Internal(error) => error.into(), - }, - ) - }); - Self { task, file_tx } - } -} - -impl ZipWriter { - pub(crate) fn new_file( - &mut self, - name: Utf8PathBuf, - compression: Compression, - ) -> ZipFileBuilder<'_, W> { - let (bytes_tx, bytes_rx) = mpsc::channel(1); - ZipFileBuilder { - writer: self, - inner: ZipFile { - name: name.into(), - compression, - last_modified: None, - unix_permissions: None, - bytes_rx, - }, - bytes_tx, - } - } - - pub(crate) async fn finish(self) -> Result { - self.file_tx.send(ZipFileMessage::Finish).await.ok(); - self.task.await.map_err(std::io::Error::from)? - } -} - -pub(crate) struct ZipFileBuilder<'a, W> { - writer: &'a mut ZipWriter, - inner: ZipFile, - bytes_tx: mpsc::Sender, -} - -impl<'a, W> ZipFileBuilder<'a, W> { - pub(crate) fn last_modified(mut self, last_modified: ZipDateTime) -> Self { - self.inner.last_modified = Some(last_modified); - self - } - - pub(crate) fn unix_permissions(mut self, unix_permissions: u32) -> Self { - self.inner.unix_permissions = Some(unix_permissions); - self - } - - pub(crate) async fn start(self) -> Result, SendError> { - self.writer.file_tx.send(ZipFileMessage::File(self.inner)).await?; - Ok(ZipDataWriter { _writer: self.writer, bytes_tx: self.bytes_tx }) - } -} - -pub(crate) struct ZipDataWriter<'a, W> { - _writer: &'a mut ZipWriter, - bytes_tx: mpsc::Sender, -} - -impl ZipDataWriter<'_, W> { - pub(crate) async fn write( - &mut self, - bytes: Bytes, - ) -> Result<(), SendError> { - self.bytes_tx.send(bytes).await?; - Ok(()) - } -} - -enum ZipFileMessage { - File(ZipFile), - Finish, -} - -struct ZipFile { - name: String, - compression: Compression, - last_modified: Option, - unix_permissions: Option, - bytes_rx: mpsc::Receiver, -} - -fn write_task( - writer: W, - mut file_rx: mpsc::Receiver, -) -> Result { - let mut archive = ZipArchiveWriter::new(writer); - loop { - let message = file_rx.blocking_recv().ok_or_else(|| { - std::io::Error::new( - std::io::ErrorKind::UnexpectedEof, - "ZipWriter dropped before finish", - ) - })?; - let mut file = match message { - ZipFileMessage::File(file) => file, - ZipFileMessage::Finish => { - return archive.finish(); - } - }; - - let mut builder = archive.new_file(&file.name); - if file.compression != Compression::none() { - builder = builder.compression_method(CompressionMethod::Deflate); - } - if let Some(last_modified) = file.last_modified { - builder = builder.last_modified(last_modified); - } - if let Some(unix_permissions) = file.unix_permissions { - builder = builder.unix_permissions(unix_permissions); - } - let (mut entry, config) = builder.start()?; - let encoder = ZipEncoder::new(&mut entry, file.compression); - let mut writer = config.wrap(encoder); - while let Some(bytes) = file.bytes_rx.blocking_recv() { - writer.write_all(&bytes)?; - } - let (encoder, output) = writer.finish()?; - encoder.finish()?; - entry.finish(output)?; - } -} - -enum ZipEncoder { - Store(W), - Deflate(DeflateEncoder), -} - -impl ZipEncoder { - fn new(writer: W, compression: Compression) -> Self { - if compression == Compression::none() { - Self::Store(writer) - } else { - Self::Deflate(DeflateEncoder::new(writer, compression)) - } - } - - fn finish(self) -> std::io::Result { - match self { - Self::Store(writer) => Ok(writer), - Self::Deflate(encoder) => encoder.finish(), - } - } -} - -impl Write for ZipEncoder { - fn write(&mut self, buf: &[u8]) -> std::io::Result { - match self { - ZipEncoder::Store(writer) => writer.write(buf), - ZipEncoder::Deflate(writer) => writer.write(buf), - } - } - - fn flush(&mut self) -> std::io::Result<()> { - match self { - ZipEncoder::Store(writer) => writer.flush(), - ZipEncoder::Deflate(writer) => writer.flush(), - } - } -} - -pub(crate) struct SendError; - -impl From> for SendError { - fn from(_value: mpsc::error::SendError) -> Self { - Self - } -} From 350530a49faa4bc76426601bc59ba324f28e64b4 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 15 Jun 2026 19:44:57 -0700 Subject: [PATCH 113/137] bleh --- bin/src/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/src/build.rs b/bin/src/build.rs index 7cc8ecd..11ab4e6 100644 --- a/bin/src/build.rs +++ b/bin/src/build.rs @@ -29,7 +29,7 @@ pub struct Args { output: Utf8PathBuf, #[clap(flatten)] sign_options: SignOptions, - /// The system version [required] + /// The system version (required) #[clap(short = 'V', long)] system_version: Version, } From 0e0a7e4410d4a4162dbe73a18a0d9e4a26375144 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 15 Jun 2026 20:19:37 -0700 Subject: [PATCH 114/137] generate a more-realistic fake OS phase 2 image --- lib/src/edit/os_images.rs | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/lib/src/edit/os_images.rs b/lib/src/edit/os_images.rs index 9bbb0e7..e0a60df 100644 --- a/lib/src/edit/os_images.rs +++ b/lib/src/edit/os_images.rs @@ -190,12 +190,41 @@ impl Input { ), MIB, ); - let phase_2 = BytesSource::fake_padded( - format!( - "{os_variant} OS phase 2 image version {interior_version}\n" - ), - 4 * MIB, + + let mut phase_2_bytes = BytesMut::with_capacity(4096); + phase_2_bytes.put(OXIDE_BOOT_MAGIC.as_slice()); // uint32_t odh_magic; + phase_2_bytes.put_u32_le(2); // uint32_t odh_version; + // The only defined ODH_FLAG is: + // #define ODH_FLAG_COMPRESSED 0x1 + // but we are not compressing any fake images. (Normally the host image + // is "uncompressed" -- a raw ZFS image with compressed contents -- and + // the recovery image is compressed.) + phase_2_bytes.put_u64_le(0); // uint64_t odh_flags; + phase_2_bytes.put_u64_le(4 * MIB); // uint64_t odh_data_size; + phase_2_bytes.put_u64_le(4 * MIB); // uint64_t odh_image_size; + phase_2_bytes.put_u64_le(1 << 32); // uint64_t odh_target_size; + // #define OXBOOT_CSUMLEN_SHA256 32 + // uint8_t odh_sha256[OXBOOT_CSUMLEN_SHA256]; + phase_2_bytes.put( + // head -c $((4 * 1024 * 1024)) "ci".as_bytes(), + OsVariant::Recovery => "recovery".as_bytes(), + }); + phase_2_bytes.put(" fake123/789fake 1986-12-28 01:23".as_bytes()); + // rest of header is zeroes, which will be written out by `fake_padded` + let phase_2 = BytesSource::fake_padded(phase_2_bytes, 4096 + 4 * MIB); let mut extra_targets = BTreeMap::new(); extra_targets.insert( From 47feeadf02be13501b6feef06f94825601df6f5b Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 15 Jun 2026 21:21:52 -0700 Subject: [PATCH 115/137] attempt to explain Source/Input/Output --- lib/src/edit/input.rs | 21 +++++++++++++++++++++ lib/src/edit/source.rs | 21 +++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/lib/src/edit/input.rs b/lib/src/edit/input.rs index 5bae3c0..39cb30c 100644 --- a/lib/src/edit/input.rs +++ b/lib/src/edit/input.rs @@ -2,6 +2,27 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +//! Indirection layer for logical sets of artifacts/targets. +//! +//! In the usual case, a file in a repository ("target" in TUF terms) is an +//! artifact that can be thought of as logically independent. The exception to +//! this general rule is OS images, which logically couple several phase 1 ROMs, +//! a phase 2 ZFS image, and extra non-artifact targets that are kept around for +//! archival purposes. +//! +//! An [`Input`] is a logical set of targets and their metadata, which can be +//! cheaply converted into a set of [`Output`]s. Each `Output` is the +//! [source data][crate::edit::source], the target name within the repository, +//! and if the target is an artifact, its version and tags. +//! +//! This indirection exists to make the behavior in the rest of the editor +//! module more consistent, as well as provide a single place where the +//! generated target names for artifacts are defined. This keeps target names +//! consistent across all real and fake repositories. Additionally, because +//! none of these methods are `async`, it is possible to generate a list of fake +//! artifacts in non-async code that will always be consistent with the set of +//! artifacts in a generated fake repository. + use std::collections::BTreeMap; use camino::Utf8PathBuf; diff --git a/lib/src/edit/source.rs b/lib/src/edit/source.rs index 33f0f11..b5f1e9c 100644 --- a/lib/src/edit/source.rs +++ b/lib/src/edit/source.rs @@ -2,6 +2,22 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +//! Indirection layer for reading target data from varying sources. +//! +//! When generating a repository, the data for each target could come from +//! one of several places. In the normal case it comes from a file on disk +//! ([`FileSource`]). If it is particularly small or is fake it might be an +//! in-memory representation ([`BytesSource`]). If we're editing an existing +//! repository, all of the previous repository's artifacts are treated as a +//! pointer to the open repository ([`RepositorySource`]). +//! +//! [`TargetSource`] is an enum that covers all three of these possible sources, +//! and has static dispatch for reading the underlying source as a `Stream`. +//! +//! Each of the three concrete sources has an `into_target` method which returns +//! a [`Target`], which is the `TargetSource` along with the target's length and +//! SHA-256 checksum. + use std::convert::Infallible; use std::pin::Pin; use std::sync::Arc; @@ -46,6 +62,11 @@ pub(crate) enum TargetSource<'a> { } impl TargetSource<'_> { + /// A relative indication of how computationally expensive it is to read + /// from this source, given two sources of the data. + /// + /// For example, reading from a file on disk is preferred from reading a + /// target out of an opened repository. pub(crate) fn cost(&self) -> usize { match self { TargetSource::Bytes(BytesSource { fake_length: None, .. }) => 0, From 4078d3fb76df4f1172b66549b17319ede5933ad2 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 15 Jun 2026 21:23:53 -0700 Subject: [PATCH 116/137] explicitly tag source errors as #[source] --- artifact/src/hubris.rs | 1 + lib/src/error.rs | 92 +++++++++++++++++++++++++++++++++------- lib/src/zip_transport.rs | 7 ++- 3 files changed, 84 insertions(+), 16 deletions(-) diff --git a/artifact/src/hubris.rs b/artifact/src/hubris.rs index 9750d81..db59d26 100644 --- a/artifact/src/hubris.rs +++ b/artifact/src/hubris.rs @@ -142,6 +142,7 @@ pub enum ReadCabooseError { /// The tag that was not valid UTF-8. tag: &'static str, /// The underlying error. + #[source] source: std::str::Utf8Error, }, } diff --git a/lib/src/error.rs b/lib/src/error.rs index 7dd20fe..4d0624f 100644 --- a/lib/src/error.rs +++ b/lib/src/error.rs @@ -63,23 +63,39 @@ pub enum ErrorKind { "failed to read end of central directory of zip archive{archive_path}", archive_path = SpacePath(archive_path), )] - ReadZipEocd { source: rawzip::Error, archive_path: Option }, + ReadZipEocd { + #[source] + source: rawzip::Error, + archive_path: Option, + }, #[error( "failed to read central directory file header of \ zip archive{archive_path}", archive_path = SpacePath(archive_path), )] - ReadZipCdfh { source: rawzip::Error, archive_path: Option }, + ReadZipCdfh { + #[source] + source: rawzip::Error, + archive_path: Option, + }, #[error( "failed to read local file header of zip archive{archive_path}", archive_path = SpacePath(archive_path), )] - ReadZipLocal { source: rawzip::Error, archive_path: Option }, + ReadZipLocal { + #[source] + source: rawzip::Error, + archive_path: Option, + }, #[error( "failed to write zip archive{archive_path}", archive_path = SpacePath(archive_path), )] - WriteZip { source: rawzip::Error, archive_path: Option }, + WriteZip { + #[source] + source: rawzip::Error, + archive_path: Option, + }, #[error( "zip archive{archive_path}'s end of central directory record \ expects {expected} entries, but found {actual} entries", @@ -134,31 +150,60 @@ pub enum ErrorKind { #[error("failed to create temporary file")] CreateTempFile(#[source] std::io::Error), #[error("failed to open file{path}", path = SpacePath(path))] - OpenFile { source: std::io::Error, path: Option }, + OpenFile { + #[source] + source: std::io::Error, + path: Option, + }, #[error("failed to read directory{path}", path = SpacePath(path))] - ReadDir { source: std::io::Error, path: Option }, + ReadDir { + #[source] + source: std::io::Error, + path: Option, + }, #[error("failed to read from file{path}", path = SpacePath(path))] - ReadFile { source: std::io::Error, path: Option }, + ReadFile { + #[source] + source: std::io::Error, + path: Option, + }, #[error("failed to seek in file{path}", path = SpacePath(path))] - SeekFile { source: std::io::Error, path: Option }, + SeekFile { + #[source] + source: std::io::Error, + path: Option, + }, #[error("failed to write to file{path}", path = SpacePath(path))] - WriteFile { source: std::io::Error, path: Option }, + WriteFile { + #[source] + source: std::io::Error, + path: Option, + }, #[error("failed to read stream")] ReadStream(#[source] Box), #[error("stream exceeded length limit ({limit})")] StreamLimit { limit: u64 }, #[error("failed to read hubris archive {path}")] - ReadHubrisArchive { source: hubtools::Error, path: Utf8PathBuf }, + ReadHubrisArchive { + #[source] + source: hubtools::Error, + path: Utf8PathBuf, + }, #[error("failed to read caboose from {path}")] ReadCaboose { + #[source] source: tufaceous_artifact::ReadCabooseError, path: Utf8PathBuf, }, #[error("failed to generate fake hubris archive")] GenerateFakeHubrisArchive(#[source] hubtools::Error), #[error("failed to read CoRIM manifest {path}")] - ReadCorim { source: rats_corim::Error, path: Utf8PathBuf }, + ReadCorim { + #[source] + source: rats_corim::Error, + path: Utf8PathBuf, + }, #[error("failed to generate fake measurement corpus")] GenerateFakeMeasurementCorpus(#[source] rats_corim::Error), #[error("failed to serialize fake measurement corpus")] @@ -180,7 +225,12 @@ pub enum ErrorKind { #[error("targets base URL unset")] TargetsBaseUrlUnset, #[error("failed to join {url} onto {base}")] - UrlJoin { source: url::ParseError, url: &'static str, base: String }, + UrlJoin { + #[source] + source: url::ParseError, + url: &'static str, + base: String, + }, #[error("no trust roots provided to load repository")] NoTrustRoots, #[error( @@ -195,7 +245,11 @@ pub enum ErrorKind { #[error("target {target_name} not found")] TargetNotFound { target_name: String }, #[error("failed to parse target {target}")] - ParseTargetJson { source: serde_json::Error, target: String }, + ParseTargetJson { + #[source] + source: serde_json::Error, + target: String, + }, #[error(transparent)] ArtifactVersion(#[from] tufaceous_artifact::ArtifactVersionError), @@ -238,9 +292,17 @@ pub enum ErrorKind { )] ConvertKnownTagsToMap(#[source] serde_json::Error), #[error("failed to read composite artifact {target}")] - ReadCompositeArtifact { source: std::io::Error, target: String }, + ReadCompositeArtifact { + #[source] + source: std::io::Error, + target: String, + }, #[error("failed to read oxide.json from {path}")] - ReadZoneOxideJson { source: std::io::Error, path: Utf8PathBuf }, + ReadZoneOxideJson { + #[source] + source: std::io::Error, + path: Utf8PathBuf, + }, #[error("importing v1 repository to an editor is not supported")] ImportV1Repo, } diff --git a/lib/src/zip_transport.rs b/lib/src/zip_transport.rs index b0779e9..1faa609 100644 --- a/lib/src/zip_transport.rs +++ b/lib/src/zip_transport.rs @@ -463,7 +463,12 @@ pub enum ZipTransportError { #[error(transparent)] Join(#[from] tokio::task::JoinError), #[error("failed to join {url} onto {base}")] - UrlJoin { source: url::ParseError, url: String, base: String }, + UrlJoin { + #[source] + source: url::ParseError, + url: String, + base: String, + }, #[error(transparent)] Zip(#[from] rawzip::Error), From a889f72edba0cec5b252f75592cf6adf413f5068 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 15 Jun 2026 21:29:32 -0700 Subject: [PATCH 117/137] fixup! simplify the zip writer, moving it to edit::sign --- lib/src/edit/sign.rs | 55 ++++++++++++++++++++++---------------------- lib/src/error.rs | 6 +++++ 2 files changed, 33 insertions(+), 28 deletions(-) diff --git a/lib/src/edit/sign.rs b/lib/src/edit/sign.rs index f89e293..9188312 100644 --- a/lib/src/edit/sign.rs +++ b/lib/src/edit/sign.rs @@ -299,21 +299,14 @@ impl SignedRepository<'_> { } tx.send(ZipWriterMessage::FinishArchive).await.ok(); task.await?.map_err(|error| { - let source = match error { - ZipWriterError::Zip(error) => error, - err @ (ZipWriterError::ExpectedFile - | ZipWriterError::ExpectedBytes) => { - std::io::Error::new(std::io::ErrorKind::InvalidInput, err) - .into() + match error { + ZipWriterError::Zip(source) => ErrorKind::WriteZip { + source, + archive_path: archive_path.map(Utf8Path::to_owned), + }, + ZipWriterError::StateMachine(reason) => { + ErrorKind::WriteZipStateMachine { reason } } - err @ ZipWriterError::UnexpectedEof => { - std::io::Error::new(std::io::ErrorKind::UnexpectedEof, err) - .into() - } - }; - ErrorKind::WriteZip { - source, - archive_path: archive_path.map(Utf8Path::to_owned), } .into() }) @@ -365,16 +358,15 @@ enum ZipWriterMessage { FinishFile, } -#[derive(Debug, thiserror::Error)] enum ZipWriterError { - #[error(transparent)] - Zip(#[from] rawzip::Error), - #[error("state machine violation: expected file start or archive finish")] - ExpectedFile, - #[error("state machine violation: expected file bytes or file finish")] - ExpectedBytes, - #[error("state machine violation: unexpected end of message stream")] - UnexpectedEof, + Zip(rawzip::Error), + StateMachine(&'static str), +} + +impl From for ZipWriterError { + fn from(error: rawzip::Error) -> Self { + ZipWriterError::Zip(error) + } } impl From for ZipWriterError { @@ -399,8 +391,9 @@ fn blocking_write_task( ) -> Result { let mut archive = ZipArchiveWriter::new(writer); loop { - let message = - rx.blocking_recv().ok_or(ZipWriterError::UnexpectedEof)?; + let message = rx.blocking_recv().ok_or( + ZipWriterError::StateMachine("unexpected end of message stream"), + )?; let (name, compression) = match message { ZipWriterMessage::FinishArchive => { return Ok(archive.finish()?); @@ -410,7 +403,9 @@ fn blocking_write_task( } ZipWriterMessage::WriteFileBytes(_) | ZipWriterMessage::FinishFile => { - return Err(ZipWriterError::ExpectedFile); + return Err(ZipWriterError::StateMachine( + "expected StartFile or FinishArchive", + )); } }; @@ -431,7 +426,9 @@ fn blocking_write_task( loop { let message = - rx.blocking_recv().ok_or(ZipWriterError::UnexpectedEof)?; + rx.blocking_recv().ok_or(ZipWriterError::StateMachine( + "unexpected end of message stream", + ))?; match message { ZipWriterMessage::WriteFileBytes(bytes) => { writer.write_all(&bytes)?; @@ -444,7 +441,9 @@ fn blocking_write_task( } ZipWriterMessage::FinishArchive | ZipWriterMessage::StartFile { .. } => { - return Err(ZipWriterError::ExpectedBytes); + return Err(ZipWriterError::StateMachine( + "expected WriteFileBytes or FinishFile", + )); } } } diff --git a/lib/src/error.rs b/lib/src/error.rs index 4d0624f..e35daa1 100644 --- a/lib/src/error.rs +++ b/lib/src/error.rs @@ -96,6 +96,11 @@ pub enum ErrorKind { source: rawzip::Error, archive_path: Option, }, + #[error( + "zip writer state machine violation: {reason} \ + (please report this logic error in Tufaceous)" + )] + WriteZipStateMachine { reason: &'static str }, #[error( "zip archive{archive_path}'s end of central directory record \ expects {expected} entries, but found {actual} entries", @@ -338,6 +343,7 @@ impl ErrorKind { ErrorKind::Join(_) | ErrorKind::ToughKey(_) | ErrorKind::WriteZip { .. } + | ErrorKind::WriteZipStateMachine { .. } | ErrorKind::CreateTempDir(_) | ErrorKind::CreateTempFile(_) | ErrorKind::OpenFile { .. } From 9f301a01037d1e06179ee4b2977aa4330cc2aa27 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 15 Jun 2026 21:45:42 -0700 Subject: [PATCH 118/137] reject repos with bad target metadata --- lib/src/edit/source.rs | 2 +- lib/src/error.rs | 12 ++++---- lib/src/repo.rs | 70 ++++++++++-------------------------------- lib/src/repo/check.rs | 61 ------------------------------------ lib/src/repo/v1.rs | 6 ++-- 5 files changed, 26 insertions(+), 125 deletions(-) diff --git a/lib/src/edit/source.rs b/lib/src/edit/source.rs index b5f1e9c..1e1a330 100644 --- a/lib/src/edit/source.rs +++ b/lib/src/edit/source.rs @@ -347,7 +347,7 @@ impl<'a> RepositorySource<'a> { let sha256 = match self.sha256.as_slice().try_into() { Ok(sha256) => ArtifactHash(sha256), Err(_) => { - return Err(ErrorKind::InvalidImportedHashLength { + return Err(ErrorKind::InvalidHashLength { target_name: self.target_name, } .into()); diff --git a/lib/src/error.rs b/lib/src/error.rs index e35daa1..7608f9f 100644 --- a/lib/src/error.rs +++ b/lib/src/error.rs @@ -255,6 +255,8 @@ pub enum ErrorKind { source: serde_json::Error, target: String, }, + #[error("unsafe target name {target_name}")] + UnsafeTargetName { target_name: String }, #[error(transparent)] ArtifactVersion(#[from] tufaceous_artifact::ArtifactVersionError), @@ -277,11 +279,8 @@ pub enum ErrorKind { second_target_name: String, tags: BTreeMap, }, - #[error( - "target {target_name} from imported repository has invalid SHA-256 \ - hash length; remove the target before calling finish" - )] - InvalidImportedHashLength { target_name: String }, + #[error("target {target_name} has invalid SHA-256 hash length")] + InvalidHashLength { target_name: String }, #[error("failed to serialize artifacts document")] SerializeArtifacts(#[source] serde_json::Error), #[error("failed to serialize Installinator document")] @@ -335,6 +334,7 @@ impl ErrorKind { | ErrorKind::ArtifactNotFound { .. } | ErrorKind::TargetNotFound { .. } | ErrorKind::ParseTargetJson { .. } + | ErrorKind::UnsafeTargetName { .. } | ErrorKind::ArtifactVersion(_) | ErrorKind::ConvertKnownTagsToMap(_) | ErrorKind::ReadCompositeArtifact { .. } @@ -365,7 +365,7 @@ impl ErrorKind { | ErrorKind::ConvertMetadataToMap(_) | ErrorKind::TargetNameCollision { .. } | ErrorKind::DisallowedTagCollision { .. } - | ErrorKind::InvalidImportedHashLength { .. } + | ErrorKind::InvalidHashLength { .. } | ErrorKind::SerializeArtifacts(_) | ErrorKind::SerializeInstallinator(_) | ErrorKind::NoSigningRoot diff --git a/lib/src/repo.rs b/lib/src/repo.rs index 89ed65d..64154a0 100644 --- a/lib/src/repo.rs +++ b/lib/src/repo.rs @@ -19,7 +19,6 @@ use rawzip::FileReader; use semver::Version; use serde::de::DeserializeOwned; use slog::Logger; -use slog::warn; use tokio::sync::Semaphore; use tokio::sync::TryAcquireError; use tokio::task::JoinSet; @@ -140,16 +139,15 @@ impl Repository { let (artifacts, artifact_data) = artifacts .into_iter() - .filter_map(|ArtifactSchema { target_name, version, tags }| { - let (hash, length) = - target_meta_skip(&repo, log, &target_name)?; + .map(|ArtifactSchema { target_name, version, tags }| { + let (hash, length) = target_meta(&repo, &target_name)?; let artifact = Artifact { version, tags, hash, length }; - Some(( + Ok::<_, Error>(( artifact.clone(), (artifact, ArtifactData::Target { target_name }), )) }) - .collect(); + .collect::>()?; Ok(Repository { log: log.clone(), inner: repo, @@ -457,54 +455,20 @@ async fn read_target_json( }) } -fn target_meta_inner( - target: &Target, -) -> Result<(ArtifactHash, u64), InvalidTargetError> { - Ok(( - ArtifactHash(target.hashes.sha256.as_ref().try_into().map_err( - |_| InvalidTargetError::ChecksumLength { - sha256: target.hashes.sha256.to_vec(), - }, - )?), - target.length, - )) -} - fn target_meta( repo: &tough::Repository, target_name: &str, -) -> Result<(ArtifactHash, u64), InvalidTargetError> { - let name = TargetName::new(target_name) - .map_err(|_| InvalidTargetError::NameRejected)?; - let Some(target) = repo.targets().signed.targets.get(&name) else { - return Err(InvalidTargetError::NotFound); - }; - target_meta_inner(target) -} - -fn target_meta_skip( - repo: &tough::Repository, - log: &Logger, - target_name: &str, -) -> Option<(ArtifactHash, u64)> { - target_meta(repo, target_name) - .inspect_err(|error| { - warn!( - log, - "skipping artifact"; - "target_name" => &target_name, - "error" => crate::util::error_chain(&error), - ); - }) - .ok() -} - -#[derive(Debug, thiserror::Error)] -enum InvalidTargetError { - #[error("target name rejected by tough")] - NameRejected, - #[error("target not found")] - NotFound, - #[error("incorrect sha256 length for {:?}", hex::encode(.sha256))] - ChecksumLength { sha256: Vec }, +) -> Result<(ArtifactHash, u64), Error> { + let name = TargetName::new(target_name).map_err(|_| { + ErrorKind::UnsafeTargetName { target_name: target_name.to_owned() } + })?; + let target = repo.targets().signed.targets.get(&name).ok_or_else(|| { + ErrorKind::TargetNotFound { target_name: target_name.to_owned() } + })?; + let hash = ArtifactHash(target.hashes.sha256.as_ref().try_into().map_err( + |_| ErrorKind::InvalidHashLength { + target_name: target_name.to_owned(), + }, + )?); + Ok((hash, target.length)) } diff --git a/lib/src/repo/check.rs b/lib/src/repo/check.rs index 1adecf7..9d68fc6 100644 --- a/lib/src/repo/check.rs +++ b/lib/src/repo/check.rs @@ -17,10 +17,6 @@ use tufaceous_artifact::OsVariant; use tufaceous_artifact::artifact_set::GetError; use crate::Repository; -use crate::repo::ArtifactData; -use crate::repo::InvalidTargetError; -use crate::repo::target_meta; -use crate::repo::target_meta_inner; impl Repository { /// Check the repository for consistency and other problems. @@ -35,29 +31,7 @@ impl Repository { pub async fn check_problems(&self) -> Vec { let mut problems = Vec::new(); - for (target_name, target) in self.targets() { - if target_name.raw() != target_name.resolved() { - problems.push(CheckProblem::BadTargetName { - target_name: target_name.raw().to_owned(), - }); - } - if let Err(error) = target_meta_inner(target) { - problems.push(CheckProblem::from_invalid_target( - error, - target_name.raw().to_owned(), - )); - } - } - for (artifact, data) in &self.artifact_data { - if let ArtifactData::Target { target_name } = data - && let Err(error) = target_meta(&self.inner, target_name) - { - problems.push(CheckProblem::from_invalid_target( - error, - target_name.to_owned(), - )); - } // Check that no artifact contains unknown tags. if artifact.known_tags().is_none() { problems.push(CheckProblem::UnknownTags { @@ -176,24 +150,6 @@ pub enum CheckProblem { #[error("no artifact matching {0}")] MissingArtifact(KnownArtifactTags), - /// An artifact's target name is not in the repository. - #[error( - "artifact with target name {target_name} listed but not present \ - in repository" - )] - MissingTarget { target_name: String }, - - /// A target name in the repository is not well-formed. - #[error("target name {target_name} is not well-formed")] - BadTargetName { target_name: String }, - - /// A SHA-256 checksum in the TUF repository metadata has an invalid length. - #[error( - "target {target_name} has SHA-256 checksum {} with invalid length", - hex::encode(.sha256) - )] - TargetHashLengthMismatch { target_name: String, sha256: Vec }, - /// Multiple artifacts for these tags were not expected. #[error("multiple artifacts found matching {0}")] MultipleArtifacts(KnownArtifactTags), @@ -225,20 +181,3 @@ pub enum CheckProblem { )] UnknownTags { target_name: String, tags: BTreeMap }, } - -impl CheckProblem { - fn from_invalid_target( - source: InvalidTargetError, - target_name: String, - ) -> Self { - match source { - InvalidTargetError::NameRejected => { - Self::BadTargetName { target_name } - } - InvalidTargetError::NotFound => Self::MissingTarget { target_name }, - InvalidTargetError::ChecksumLength { sha256 } => { - Self::TargetHashLengthMismatch { target_name, sha256 } - } - } - } -} diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs index 2fe295f..83bc98d 100644 --- a/lib/src/repo/v1.rs +++ b/lib/src/repo/v1.rs @@ -59,7 +59,7 @@ use crate::repo::ArtifactData; use crate::repo::read_target; use crate::repo::read_target_json; use crate::repo::read_target_vec; -use crate::repo::target_meta_skip; +use crate::repo::target_meta; use crate::util::ArtifactExt; pub(super) struct PartialRepository { @@ -97,9 +97,7 @@ pub(crate) async fn from_loaded( }; let mut installinator_document = None; for V1Artifact { version, kind, target } in v1_artifacts { - let Some((hash, length)) = target_meta_skip(repo, log, &target) else { - continue; - }; + let (hash, length) = target_meta(repo, &target)?; let kind = match kind { V1ArtifactKind::Known(kind) => kind, V1ArtifactKind::Unknown(kind) => { From cff1fc447c84efb9426063a9838afac858c19413 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 15 Jun 2026 21:49:12 -0700 Subject: [PATCH 119/137] verify subcommand should print repo hash --- bin/src/verify.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/src/verify.rs b/bin/src/verify.rs index 155df85..312d55a 100644 --- a/bin/src/verify.rs +++ b/bin/src/verify.rs @@ -10,6 +10,7 @@ use anyhow::ensure; use camino::Utf8PathBuf; use clap::Parser; use tufaceous::CheckProblem; +use tufaceous_artifact::ArtifactHash; use crate::load::LoadOptions; @@ -36,9 +37,13 @@ impl Args { .load_options .loader() .await? + .compute_archive_sha256(true) .v1_compatibility(true) .load_zip_path(self.repo.clone(), &crate::LOG) .await?; + let sha256 = ArtifactHash( + *repo.archive_sha256().expect("repo hash should be calculated"), + ); repo.verify_targets(self.threads).await?; @@ -49,7 +54,7 @@ impl Args { WriteProblems(&problems) ); - eprintln!("{}: OK", self.repo); + eprintln!("{}: OK, SHA256 = {sha256}", self.repo); Ok(()) } } From a0f34b8ba91ca675b26423d380c80f3cc3339d6b Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 15 Jun 2026 21:52:07 -0700 Subject: [PATCH 120/137] fix added/changed flags in readme --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f9f097b..8e67d4c 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,8 @@ As a general rule it is not possible to use this tooling to create a repository Most of the time you're working with repositories generated by the [Omicron release tooling](https://github.com/oxidecomputer/omicron/blob/main/docs/releng.adoc) (`cargo xtask releng` in Omicron). There is also a binary target that is a thin CLI on top of the library, which you can build with `cargo build --release --bin tufaceous`. +In all commands that read from a repository, use either `-r`/`--trust-roots` to provide a list of trust roots to verify signatures against, or `--blindly-trust` to use the `metadata/1.root.json` root in the repository as the trust root. If you want to load a repository with an expired signature, use `--allow-expired`. For convenience you can combine `--allow-expired` and `--blindly-trust` with `-f`/`--force-load`. + ### Verify a repository `tufaceous verify` will read a repository, print its SHA256 checksum, and verify the metadata and checksums of all the metadata in the repository. It will also run a series of compatibility checks on the repository and warn you of any detected issues. @@ -37,7 +39,7 @@ To unpack an entire repository, use `unzip` or ZIP tooling provided by your OS. `tufaceous build` has two required flags: - `--output file.zip`, which provides the path to write to -- `--version 1.0.0`, which provides the system version +- `--system-version 1.0.0`, which provides the system version The arguments are the set of artifacts to add: From 3f740da4f55ddc1e06d5b0bb97b26762363686e8 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 15 Jun 2026 21:56:12 -0700 Subject: [PATCH 121/137] make Repository::check_problems fallible --- bin/src/verify.rs | 2 +- lib/src/repo/check.rs | 34 +++++++++++++++++++--------------- lib/tests/check.rs | 4 ++-- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/bin/src/verify.rs b/bin/src/verify.rs index 312d55a..d3fdfb7 100644 --- a/bin/src/verify.rs +++ b/bin/src/verify.rs @@ -47,7 +47,7 @@ impl Args { repo.verify_targets(self.threads).await?; - let problems = repo.check_problems().await; + let problems = repo.check_problems().await?; ensure!( problems.is_empty(), "found compatibility problems:\n{}", diff --git a/lib/src/repo/check.rs b/lib/src/repo/check.rs index 9d68fc6..a674223 100644 --- a/lib/src/repo/check.rs +++ b/lib/src/repo/check.rs @@ -17,6 +17,7 @@ use tufaceous_artifact::OsVariant; use tufaceous_artifact::artifact_set::GetError; use crate::Repository; +use crate::error::Error; impl Repository { /// Check the repository for consistency and other problems. @@ -28,7 +29,7 @@ impl Repository { /// another version of Tufaceous, such as when the control plane decides to /// accept a repository. #[allow(clippy::too_many_lines)] - pub async fn check_problems(&self) -> Vec { + pub async fn check_problems(&self) -> Result, Error> { let mut problems = Vec::new(); for (artifact, data) in &self.artifact_data { @@ -108,23 +109,26 @@ impl Repository { if let Ok(artifact) = self.artifacts().get_only(&KnownArtifactTags::InstallinatorDocument) - && let Ok(stream) = self.read_artifact(artifact).await - && let Ok(bytes) = stream.map_ok(Vec::from).try_concat().await - && let Ok(doc) = serde_json::from_slice::( - &bytes, - ) - .map_err(|source| { - problems.push(CheckProblem::DeserializeInstallinator(source)); - }) - && doc.system_version.as_str() != self.system_version().to_string() { - problems.push(CheckProblem::InstallinatorVersion { - doc_version: doc.system_version.clone(), - system_version: self.system_version().clone(), - }); + let stream = self.read_artifact(artifact).await?; + let bytes = stream.map_ok(Vec::from).try_concat().await?; + match serde_json::from_slice::(&bytes) { + Ok(doc) => { + if doc.system_version.as_str() + != self.system_version().to_string() + { + problems.push(CheckProblem::InstallinatorVersion { + doc_version: doc.system_version.clone(), + system_version: self.system_version().clone(), + }); + } + } + Err(source) => problems + .push(CheckProblem::DeserializeInstallinator(source)), + } } - problems + Ok(problems) } } diff --git a/lib/tests/check.rs b/lib/tests/check.rs index 91694a4..1967a69 100644 --- a/lib/tests/check.rs +++ b/lib/tests/check.rs @@ -17,7 +17,7 @@ const V1: Version = Version::new(1, 0, 0); async fn fake_checks_out() -> Result<(), Error> { let log = slog::Logger::root(slog::Discard, slog::o!()); let repo = Repository::fake(V1, &log).await?; - let problems = repo.check_problems().await; + let problems = repo.check_problems().await?; assert!(problems.is_empty(), "repo has unexpected problems: {problems:?}"); Ok(()) } @@ -37,7 +37,7 @@ async fn missing_installinator() -> Result<(), Error> { .trust_root(signed.root()) .load_zip_buffer(zip, &log) .await?; - let problems = repo.check_problems().await; + let problems = repo.check_problems().await?; assert!( matches!( problems.as_slice(), From 509557c5155275a5cf73c74cfae2a48896b485b5 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 15 Jun 2026 22:05:50 -0700 Subject: [PATCH 122/137] list problem if metadata cannot convert to mapping --- lib/src/repo/check.rs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/lib/src/repo/check.rs b/lib/src/repo/check.rs index a674223..b373249 100644 --- a/lib/src/repo/check.rs +++ b/lib/src/repo/check.rs @@ -42,10 +42,13 @@ impl Repository { } } - match self - .structured_metadata() - .and_then(|structured_metadata| structured_metadata.to_map().ok()) - { + match self.structured_metadata().and_then(|m| match m.to_map() { + Ok(m) => Some(m), + Err(source) => { + problems.push(CheckProblem::ConvertMetadataMapping(source)); + None + } + }) { Some(metadata) => { if self.metadata() != &metadata { let diff = self.metadata().diff(&metadata); @@ -174,6 +177,14 @@ pub enum CheckProblem { #[error("couldn't parse metadata {0:?}")] UnknownMetadata(BTreeMap), + /// The repository metadata cannot be converted into a string mapping, which + /// is a bug. + #[error( + "couldn't convert metadata to string mapping; \ + this is a bug in Tufaceous" + )] + ConvertMetadataMapping(#[source] serde_json::Error), + /// The repository metadata contains an unknown key. #[error("unknown metadata key {0:?}")] UnknownMetadataKey(String), From 1dedde5ed871b013a9cf74fe47cbdf9f4d8eafc0 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 15 Jun 2026 22:06:05 -0700 Subject: [PATCH 123/137] make unix.z/cpio.z optional as intended --- lib/src/edit/os_images.rs | 5 +++-- lib/src/edit/source.rs | 13 +++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/src/edit/os_images.rs b/lib/src/edit/os_images.rs index e0a60df..f21188c 100644 --- a/lib/src/edit/os_images.rs +++ b/lib/src/edit/os_images.rs @@ -85,8 +85,9 @@ impl Input> { } }; for path in ["unix.z", "cpio.z"] { - let source = FileSource::open(dir.join(path)).await?; - extra_targets.insert(path.into(), source.into()); + if let Some(source) = FileSource::try_open(dir.join(path)).await? { + extra_targets.insert(path.into(), source.into()); + } } Ok(Self::OsImages { diff --git a/lib/src/edit/source.rs b/lib/src/edit/source.rs index 1e1a330..01979c7 100644 --- a/lib/src/edit/source.rs +++ b/lib/src/edit/source.rs @@ -204,6 +204,19 @@ impl FileSource { Ok(Self::from_file(file.into_std().await, path)) } + pub(crate) async fn try_open( + path: Utf8PathBuf, + ) -> Result, Error> { + let file = match File::open(&path).await { + Ok(file) => file, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + return Ok(None); + } + Err(err) => try_path!(Err(err), OpenFile, path), + }; + Ok(Some(Self::from_file(file.into_std().await, path))) + } + pub(crate) fn from_file(file: std::fs::File, path: Utf8PathBuf) -> Self { Self { inner: Arc::new(FileSourceInner { From 89c4f3927379ee2eb5f5353a050095229a25b98b Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 15 Jun 2026 22:17:26 -0700 Subject: [PATCH 124/137] comments for v1 --- lib/src/repo/v1.rs | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs index 83bc98d..30029a2 100644 --- a/lib/src/repo/v1.rs +++ b/lib/src/repo/v1.rs @@ -79,6 +79,10 @@ impl PartialRepository { } } +/// Attempt to load a `tough::Repository` as if it is a Tufaceous v1 repository, +/// converting artifacts into the v2 memory representation. +/// +/// Returns `None` if the v1 `artifacts.json` wasn't found. #[expect(clippy::too_many_lines)] pub(crate) async fn from_loaded( repo: &tough::Repository, @@ -112,6 +116,8 @@ pub(crate) async fn from_loaded( } }; let tags = match kind { + // These arms represent a single artifact, and return their tags + // from the match arm. V1KnownArtifactKind::GimletSp | V1KnownArtifactKind::PscSp | V1KnownArtifactKind::SwitchSp => { @@ -171,6 +177,15 @@ pub(crate) async fn from_loaded( } tags } + + V1KnownArtifactKind::MeasurementCorpus => { + KnownArtifactTags::MeasurementCorpus + } + + // These arms represent composite artifacts, and all diverge + // with `continue`. They use methods on `CompositeArtifact` to + // unpack and add multiple artifacts to `partial` instead of the + // single-artifact logic at the end of this match statement. V1KnownArtifactKind::GimletRot | V1KnownArtifactKind::PscRot | V1KnownArtifactKind::SwitchRot => { @@ -198,14 +213,6 @@ pub(crate) async fn from_loaded( continue; } - V1KnownArtifactKind::InstallinatorDocument => { - // Ignore this Installinator document, because it is written for - // the v1 artifacts. We need to generate a new one for the v2 - // artifacts once all of the potential artifacts are extracted. - installinator_document = Some((version, target)); - continue; - } - V1KnownArtifactKind::ControlPlane => { CompositeArtifact::unpack(repo, target) .await? @@ -214,8 +221,13 @@ pub(crate) async fn from_loaded( continue; } - V1KnownArtifactKind::MeasurementCorpus => { - KnownArtifactTags::MeasurementCorpus + // Do not directly use any Installinator document, because it is + // written for the v1 artifacts. We need to generate a new one + // for the v2 artifacts once all of the potential artifacts are + // extracted. + V1KnownArtifactKind::InstallinatorDocument => { + installinator_document = Some((version, target)); + continue; } }; @@ -226,6 +238,8 @@ pub(crate) async fn from_loaded( ); } + // If we found an Installinator document, generate a new one with the v2 + // artifacts. if let Some((version, target)) = installinator_document { generate_installinator_document(&mut partial, version, target).await?; } From e81ad2a71868cdf6c13de6fd6e6cdc8ede8a321c Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 15 Jun 2026 22:21:44 -0700 Subject: [PATCH 125/137] document GuessInput/GuessResult --- lib/src/edit/guess.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/src/edit/guess.rs b/lib/src/edit/guess.rs index 0d4ba5f..b2a5c11 100644 --- a/lib/src/edit/guess.rs +++ b/lib/src/edit/guess.rs @@ -18,11 +18,21 @@ use crate::error::Error; use crate::error::ErrorKind; use crate::error::try_path; +/// The input to guess methods: the first chunk of bytes from the file, and the +/// `FileSource` for converting into an [`Input`] if the guess method knows what +/// it is. pub(crate) struct GuessInput { pub(crate) file_start: Bytes, pub(crate) source: FileSource, } +/// A guess method can: +/// +/// - say it knows what an input is, diverging the control flow: +/// `Ok(ControlFlow::Break(input))` +/// - say it doesn't know what an input is, returning the input and continuing +/// the control flow: `Ok(ControlFlow::Continue(guess_input))` +/// - return an error: `Err(error)` pub(crate) type GuessResult = Result>, GuessInput>, Error>; From 0975ac6ea61b8a2ce28977a5a9fb805138c09ff4 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 15 Jun 2026 22:27:36 -0700 Subject: [PATCH 126/137] consistently use `tuf_repo: tough::Repository` --- lib/src/repo.rs | 50 +++++++++++++++++++++++++++------------------- lib/src/repo/v1.rs | 38 +++++++++++++++++------------------ 2 files changed, 49 insertions(+), 39 deletions(-) diff --git a/lib/src/repo.rs b/lib/src/repo.rs index 64154a0..c124744 100644 --- a/lib/src/repo.rs +++ b/lib/src/repo.rs @@ -45,7 +45,7 @@ pub type TargetStream = #[derive(Debug, Clone)] pub struct Repository { log: Logger, - inner: tough::Repository, + tuf_repo: tough::Repository, system_version: Version, trust_root: Vec, artifacts: ArtifactSet, @@ -107,20 +107,20 @@ impl Repository { } pub(crate) async fn from_loaded( - repo: tough::Repository, + tuf_repo: tough::Repository, log: &Logger, trust_root: Vec, v1_compatibility: bool, ) -> Result { let Some(ArtifactSetSchema { system_version, artifacts, metadata }) = - read_target_json(&repo, ArtifactSetSchema::TARGET_NAME).await? + read_target_json(&tuf_repo, ArtifactSetSchema::TARGET_NAME).await? else { if v1_compatibility - && let Some(partial) = v1::from_loaded(&repo, log).await? + && let Some(partial) = v1::from_loaded(&tuf_repo, log).await? { return Ok(Repository { log: log.clone(), - inner: repo, + tuf_repo, trust_root, system_version: partial.system_version, artifacts: partial.artifacts, @@ -140,7 +140,7 @@ impl Repository { let (artifacts, artifact_data) = artifacts .into_iter() .map(|ArtifactSchema { target_name, version, tags }| { - let (hash, length) = target_meta(&repo, &target_name)?; + let (hash, length) = target_meta(&tuf_repo, &target_name)?; let artifact = Artifact { version, tags, hash, length }; Ok::<_, Error>(( artifact.clone(), @@ -150,7 +150,7 @@ impl Repository { .collect::>()?; Ok(Repository { log: log.clone(), - inner: repo, + tuf_repo, trust_root, system_version, artifacts, @@ -186,7 +186,7 @@ impl Repository { } pub fn targets(&self) -> &HashMap { - &self.inner.targets().signed.targets + &self.tuf_repo.targets().signed.targets } pub fn artifacts(&self) -> &ArtifactSet { @@ -265,7 +265,7 @@ impl Repository { &self, target_name: &str, ) -> Result { - if let Some(stream) = read_target(&self.inner, target_name).await? { + if let Some(stream) = read_target(&self.tuf_repo, target_name).await? { Ok(Box::pin(stream)) } else { Err(ErrorKind::TargetNotFound { @@ -396,6 +396,11 @@ impl ArtifactData { } } +/// An [`Artifact`] and the [`Repository`] it belongs to, for convenience to +/// code that works at the artifact level but needs to read the artifact data +/// from the repository. +/// +/// Created by [`Repository::get_handle`] or [`Repository::handles`]. #[derive(Debug, Clone)] pub struct ArtifactHandle { artifact: Artifact, @@ -403,51 +408,55 @@ pub struct ArtifactHandle { } impl ArtifactHandle { + /// Return a reference to this handle's [`Artifact`]. pub fn artifact(&self) -> &Artifact { &self.artifact } + /// Convert this handle into its owned [`Artifact`], dropping its + /// association with its [`Repository`]. pub fn into_artifact(self) -> Artifact { self.artifact } + /// Read this artifact from its repository. pub async fn stream(&self) -> Result { self.repo.read_artifact(&self.artifact).await } } async fn read_target( - repo: &tough::Repository, + tuf_repo: &tough::Repository, target: &str, ) -> Result> + 'static>, Error> { let target_name = target.parse()?; // Ensure the target is in the top-level targets.json role and not a // delegated target; we don't permit the use of delegated targets in // Tufaceous currently. - if !repo.targets().signed.targets.contains_key(&target_name) { + if !tuf_repo.targets().signed.targets.contains_key(&target_name) { return Ok(None); } - Ok(repo + Ok(tuf_repo .read_target(&target_name) .await? .map(TryStreamExt::err_into::)) } async fn read_target_vec( - repo: &tough::Repository, + tuf_repo: &tough::Repository, target: &str, ) -> Result>, Error> { - let Some(stream) = read_target(repo, target).await? else { + let Some(stream) = read_target(tuf_repo, target).await? else { return Ok(None); }; stream.map_ok(Vec::from).try_concat().await.map(Some) } async fn read_target_json( - repo: &tough::Repository, + tuf_repo: &tough::Repository, target: &str, ) -> Result, Error> { - let Some(vec) = read_target_vec(repo, target).await? else { + let Some(vec) = read_target_vec(tuf_repo, target).await? else { return Ok(None); }; serde_json::from_slice(&vec).map_err(|source| { @@ -456,15 +465,16 @@ async fn read_target_json( } fn target_meta( - repo: &tough::Repository, + tuf_repo: &tough::Repository, target_name: &str, ) -> Result<(ArtifactHash, u64), Error> { let name = TargetName::new(target_name).map_err(|_| { ErrorKind::UnsafeTargetName { target_name: target_name.to_owned() } })?; - let target = repo.targets().signed.targets.get(&name).ok_or_else(|| { - ErrorKind::TargetNotFound { target_name: target_name.to_owned() } - })?; + let target = + tuf_repo.targets().signed.targets.get(&name).ok_or_else(|| { + ErrorKind::TargetNotFound { target_name: target_name.to_owned() } + })?; let hash = ArtifactHash(target.hashes.sha256.as_ref().try_into().map_err( |_| ErrorKind::InvalidHashLength { target_name: target_name.to_owned(), diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs index 30029a2..a079b3a 100644 --- a/lib/src/repo/v1.rs +++ b/lib/src/repo/v1.rs @@ -85,11 +85,11 @@ impl PartialRepository { /// Returns `None` if the v1 `artifacts.json` wasn't found. #[expect(clippy::too_many_lines)] pub(crate) async fn from_loaded( - repo: &tough::Repository, + tuf_repo: &tough::Repository, log: &Logger, ) -> Result, Error> { let Some(V1ArtifactSetSchema { system_version, artifacts: v1_artifacts }) = - read_target_json(repo, V1ArtifactSetSchema::TARGET_NAME).await? + read_target_json(tuf_repo, V1ArtifactSetSchema::TARGET_NAME).await? else { return Ok(None); }; @@ -101,7 +101,7 @@ pub(crate) async fn from_loaded( }; let mut installinator_document = None; for V1Artifact { version, kind, target } in v1_artifacts { - let (hash, length) = target_meta(repo, &target)?; + let (hash, length) = target_meta(tuf_repo, &target)?; let kind = match kind { V1ArtifactKind::Known(kind) => kind, V1ArtifactKind::Unknown(kind) => { @@ -121,7 +121,7 @@ pub(crate) async fn from_loaded( V1KnownArtifactKind::GimletSp | V1KnownArtifactKind::PscSp | V1KnownArtifactKind::SwitchSp => { - let image = read_target_vec(repo, &target).await?; + let image = read_target_vec(tuf_repo, &target).await?; let Some(image) = image else { continue }; let mut is_lab_image = false; let tags = caboose_tags(image, &target, |caboose| { @@ -150,7 +150,7 @@ pub(crate) async fn from_loaded( V1KnownArtifactKind::GimletRotBootloader | V1KnownArtifactKind::PscRotBootloader | V1KnownArtifactKind::SwitchRotBootloader => { - let image = read_target_vec(repo, &target).await?; + let image = read_target_vec(tuf_repo, &target).await?; let Some(image) = image else { continue }; let tags = caboose_tags( image, @@ -189,7 +189,7 @@ pub(crate) async fn from_loaded( V1KnownArtifactKind::GimletRot | V1KnownArtifactKind::PscRot | V1KnownArtifactKind::SwitchRot => { - CompositeArtifact::unpack(repo, target) + CompositeArtifact::unpack(tuf_repo, target) .await? .read_rot(log, &mut partial, version) .await?; @@ -197,24 +197,24 @@ pub(crate) async fn from_loaded( } V1KnownArtifactKind::Host => { - CompositeArtifact::unpack(repo, target).await?.read_os_image( - &mut partial, - OsVariant::Host, - &version, - )?; + CompositeArtifact::unpack(tuf_repo, target) + .await? + .read_os_image(&mut partial, OsVariant::Host, &version)?; continue; } V1KnownArtifactKind::Trampoline => { - CompositeArtifact::unpack(repo, target).await?.read_os_image( - &mut partial, - OsVariant::Recovery, - &version, - )?; + CompositeArtifact::unpack(tuf_repo, target) + .await? + .read_os_image( + &mut partial, + OsVariant::Recovery, + &version, + )?; continue; } V1KnownArtifactKind::ControlPlane => { - CompositeArtifact::unpack(repo, target) + CompositeArtifact::unpack(tuf_repo, target) .await? .read_control_plane(&mut partial) .await?; @@ -387,11 +387,11 @@ struct CompositeArtifact { impl CompositeArtifact { async fn unpack( - repo: &tough::Repository, + tuf_repo: &tough::Repository, target_name: String, ) -> Result { let stream = - read_target(repo, &target_name).await?.ok_or_else(|| { + read_target(tuf_repo, &target_name).await?.ok_or_else(|| { ErrorKind::TargetNotFound { target_name: target_name.clone() } })?; pin_mut!(stream); From f503e6af6ef4d09b2b44aaac36ebb1c4940c6c1d Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 15 Jun 2026 22:31:54 -0700 Subject: [PATCH 127/137] avoid overrelying on RotSlot::fmt --- lib/src/repo/v1.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs index a079b3a..4c8a2e7 100644 --- a/lib/src/repo/v1.rs +++ b/lib/src/repo/v1.rs @@ -466,7 +466,10 @@ impl CompositeArtifact { version: ArtifactVersion, ) -> Result<(), Error> { for slot in [RotSlot::A, RotSlot::B] { - let path = Utf8PathBuf::from(format!("archive-{slot}.zip")); + let path = Utf8PathBuf::from(match slot { + RotSlot::A => "archive-a.zip", + RotSlot::B => "archive-b.zip", + }); let Some(UnpackedArtifact { file, hash, length }) = self.entries.remove(&path) else { From 5bd5bd08bb0383c44dab6721d90b44b86ca095fb Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 25 Jun 2026 06:29:59 +0000 Subject: [PATCH 128/137] ensure TargetSource::stream is Send --- lib/src/edit/source.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/edit/source.rs b/lib/src/edit/source.rs index 01979c7..4b98f58 100644 --- a/lib/src/edit/source.rs +++ b/lib/src/edit/source.rs @@ -78,7 +78,7 @@ impl TargetSource<'_> { pub(crate) fn stream( &self, - ) -> Pin> + '_>> { + ) -> Pin> + Send + '_>> { match self { TargetSource::Bytes(source) => { Box::pin(source.stream().err_into::()) From 52d37f2565764fa6a6d0cd6b3d451e25d5359779 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 25 Jun 2026 17:03:47 +0000 Subject: [PATCH 129/137] correctly use the friendly CA name in the target name --- lib/src/edit/input.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/src/edit/input.rs b/lib/src/edit/input.rs index 39cb30c..e982f1d 100644 --- a/lib/src/edit/input.rs +++ b/lib/src/edit/input.rs @@ -144,7 +144,8 @@ impl Input { rkth = tags .rot_rkth .as_ref() - .map_or("unsigned", RotKeyTableHash::as_str), + .and_then(RotKeyTableHash::friendly_ca_name) + .unwrap_or("unsigned"), slot = tags.rot_slot ); vec![Output::new(target_name, version, &tags.into(), source)?] @@ -156,7 +157,8 @@ impl Input { rkth = tags .rot_rkth .as_ref() - .map_or("unsigned", RotKeyTableHash::as_str), + .and_then(RotKeyTableHash::friendly_ca_name) + .unwrap_or("unsigned"), ); vec![Output::new(target_name, version, &tags.into(), source)?] } From 53b20abf9c97772f0ab25d08a00d36ad49ebd79a Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 30 Jun 2026 19:19:00 -0700 Subject: [PATCH 130/137] properly handle JoinError in the blocking stream pattern --- lib/src/edit/source.rs | 16 ++++------ lib/src/lib.rs | 1 + lib/src/mpsc_stream.rs | 64 ++++++++++++++++++++++++++++++++++++++++ lib/src/repo.rs | 6 ++-- lib/src/repo/v1.rs | 47 ++++++++--------------------- lib/src/zip_transport.rs | 53 +++++++++------------------------ 6 files changed, 99 insertions(+), 88 deletions(-) create mode 100644 lib/src/mpsc_stream.rs diff --git a/lib/src/edit/source.rs b/lib/src/edit/source.rs index 4b98f58..0858b6c 100644 --- a/lib/src/edit/source.rs +++ b/lib/src/edit/source.rs @@ -39,7 +39,6 @@ use serde::Serialize; use sha2::Digest; use sha2::Sha256; use tokio::fs::File; -use tokio::sync::mpsc; use tufaceous_artifact::ArtifactHash; use crate::Repository; @@ -309,9 +308,8 @@ impl FileSource { } pub(crate) fn stream(&self) -> impl Stream> { - let (tx, mut rx) = mpsc::channel::>(1); let inner = self.inner.clone(); - tokio::task::spawn_blocking(move || { + crate::mpsc_stream::mpsc_stream(None, move |tx| { let mut buf = BytesMut::with_capacity(8192); let mut offset = 0; loop { @@ -328,22 +326,18 @@ impl FileSource { source, path: Some(inner.path.clone()), }; - tx.blocking_send(Err(err.into())).ok(); - return; + return tx.blocking_send(Err(err.into())); } } let bytes = buf.split().freeze(); if bytes.is_empty() { - return; + return Ok(()); } offset += usize64!(bytes.len()); - if tx.blocking_send(Ok(bytes)).is_err() { - return; - } + tx.blocking_send(Ok(bytes))?; } - }); - futures_util::stream::poll_fn(move |cx| rx.poll_recv(cx)) + }) } } diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 92c0ac8..e457ad9 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -21,6 +21,7 @@ macro_rules! usize64 { pub mod edit; pub mod error; mod loader; +mod mpsc_stream; mod repo; mod schema; mod util; diff --git a/lib/src/mpsc_stream.rs b/lib/src/mpsc_stream.rs new file mode 100644 index 0000000..c5bdbe8 --- /dev/null +++ b/lib/src/mpsc_stream.rs @@ -0,0 +1,64 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use futures_util::FutureExt; +use futures_util::Stream; +use futures_util::StreamExt; +use futures_util::stream; +use slog::Logger; +use slog::error; +use tokio::sync::mpsc::Sender; +use tokio::sync::mpsc::error::SendError; +use tokio::task::JoinError; + +pub(crate) fn mpsc_stream( + log: Option, + f: F, +) -> impl Stream> + 'static +where + T: Send + 'static, + E: std::error::Error + From + Send + 'static, + F: FnOnce(Sender>) -> Result<(), SendError>> + + Send + + 'static, +{ + let (tx, mut rx) = tokio::sync::mpsc::channel(1); + let task = tokio::task::spawn_blocking(move || f(tx)); + stream::poll_fn(move |cx| rx.poll_recv(cx)).chain( + async move { + stream::iter(match task.await { + Ok(Ok(())) => None, + + // It is presumed a `SendError` is returned from an attempt to + // send a value across the channel created by `mpsc_stream`. + // This would indicate that either the receiver was explicitly + // closed (which this function does not do), or that the + // receiver was dropped (which would not be possible if we are + // in this match statement). Nonetheless, we will log such an + // oddity, and if the value that failed to send itself contained + // an error, we will return it as this stream's final value. + Ok(Err(SendError(Ok(_)))) => { + log.inspect(|log| { + error!(log, "stream reader unexpectedly hung up"); + }); + None + } + Ok(Err(SendError(Err(error)))) => { + log.inspect(|log| { + error!( + log, + "stream reader unexpectedly hung up \ + before receiving error"; + "err" => &crate::util::error_chain(&error), + ); + }); + Some(Err(error)) + } + + Err(join_error) => Some(Err(E::from(join_error))), + }) + } + .flatten_stream(), + ) +} diff --git a/lib/src/repo.rs b/lib/src/repo.rs index c124744..9ad6675 100644 --- a/lib/src/repo.rs +++ b/lib/src/repo.rs @@ -250,9 +250,9 @@ impl Repository { length: artifact.length, }; Ok(Box::pin(unpacked.stream( - self.log.clone(), - original_target_name.clone(), - inner_path.clone(), + &self.log, + original_target_name, + inner_path, ))) } } diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs index 4c8a2e7..c950a44 100644 --- a/lib/src/repo/v1.rs +++ b/lib/src/repo/v1.rs @@ -30,8 +30,8 @@ use serde::Deserialize; use sha2::Digest; use sha2::Sha256; use slog::Logger; -use slog::error; use slog::info; +use slog::o; use slog::warn; use tokio::sync::mpsc; use tufaceous_artifact::Artifact; @@ -286,12 +286,16 @@ impl UnpackedArtifact { pub(crate) fn stream( self, - log: Logger, - original_target_name: String, - inner_path: Utf8PathBuf, + log: &Logger, + original_target_name: &str, + inner_path: &Utf8Path, ) -> impl Stream> + 'static { - let (tx, mut rx) = mpsc::channel::>(1); - let task = tokio::task::spawn_blocking(move || { + let log = log.new(o!( + "stream" => format!("{}::stream", std::any::type_name::()), + "original_target_name" => original_target_name.to_owned(), + "inner_path" => inner_path.to_string(), + )); + crate::mpsc_stream::mpsc_stream(Some(log), move |tx| { type SendError = mpsc::error::SendError>; let mut buf = BytesMut::with_capacity(8192); @@ -308,8 +312,7 @@ impl UnpackedArtifact { } Err(source) => { let err = ErrorKind::ReadFile { source, path: None }; - tx.blocking_send(Err(err.into()))?; - return Ok::<_, SendError>(()); + return tx.blocking_send(Err(err.into())); } } @@ -335,33 +338,7 @@ impl UnpackedArtifact { tx.blocking_send(Err( ErrorKind::ReadFile { source, path: None }.into() )) - }); - - tokio::task::spawn(async move { - if let Ok(Err(send_error)) = task.await { - match send_error.0 { - Ok(_) => { - error!( - log, - "unpacked file reader hung up mid-stream"; - "original_target_name" => original_target_name, - "inner_path" => inner_path.as_str(), - ); - } - Err(err) => { - error!( - log, - "unpacked file reader hung up mid-stream \ - before receiving error"; - "original_target_name" => original_target_name, - "inner_path" => inner_path.as_str(), - "err" => &crate::util::error_chain(&err), - ); - } - } - } - }); - futures_util::stream::poll_fn(move |cx| rx.poll_recv(cx)) + }) } } diff --git a/lib/src/zip_transport.rs b/lib/src/zip_transport.rs index 1faa609..efd00d3 100644 --- a/lib/src/zip_transport.rs +++ b/lib/src/zip_transport.rs @@ -66,6 +66,7 @@ use bytes::BytesMut; use camino::Utf8PathBuf; use flate2::read::DeflateDecoder; use futures_util::Stream; +use futures_util::TryStreamExt; use rawzip::CompressionMethod; use rawzip::FileReader; use rawzip::ReaderAt; @@ -73,9 +74,8 @@ use rawzip::ZipArchive; use rawzip::ZipArchiveEntryWayfinder; use rawzip::ZipFileHeaderRecord; use slog::Logger; -use slog::error; +use slog::o; use slog::warn; -use tokio::sync::mpsc; use tough::Transport; use tough::TransportError; use tough::TransportErrorKind; @@ -274,12 +274,11 @@ impl ZipTransport { ) -> impl Stream> { // This could be a lot nicer but the ZIP entry reader can't cross in/out // of `spawn_blocking` because the type has an associated lifetime. - let (tx, mut rx) = mpsc::channel::>(1); - let cloned_url = url.clone(); - let task = tokio::task::spawn_blocking(move || { - type SendError = - mpsc::error::SendError>; - + let log = self.log.new(o!( + "stream" => format!("{}::stream", std::any::type_name::()), + "url" => url.to_string(), + )); + crate::mpsc_stream::mpsc_stream(Some(log), move |tx| { let mut reader = match self .inner .archive @@ -296,8 +295,7 @@ impl ZipTransport { }) { Ok(reader) => reader, Err(error) => { - tx.blocking_send(Err(error.into_tough_error(url)))?; - return Ok::<_, SendError>(()); + return tx.blocking_send(Err(error)); } }; @@ -308,43 +306,20 @@ impl ZipTransport { } buf.resize(buf.capacity(), 0); match reader.read(&mut buf) { - Ok(0) => return Ok::<_, SendError>(()), + Ok(0) => return Ok(()), Ok(n) => { buf.truncate(n); tx.blocking_send(Ok(buf.split().freeze()))?; } Err(error) => { - tx.blocking_send(Err(ZipTransportError::from(error) - .into_tough_error(url)))?; - return Ok::<_, SendError>(()); + return tx.blocking_send(Err(ZipTransportError::from( + error, + ))); } } } - }); - - tokio::task::spawn(async move { - if let Ok(Err(send_error)) = task.await { - match send_error.0 { - Ok(_) => { - error!( - self.log, - "zip file reader hung up mid-stream"; - "url" => &cloned_url.to_string(), - ); - } - Err(err) => { - error!( - self.log, - "zip file reader hung up mid-stream \ - before receiving error"; - "url" => &cloned_url.to_string(), - "err" => &crate::util::error_chain(&err), - ); - } - } - } - }); - futures_util::stream::poll_fn(move |cx| rx.poll_recv(cx)) + }) + .map_err(move |e: ZipTransportError| e.into_tough_error(url.clone())) } } From e37c3996f9c78ff05148ec0d3f35a0536c56ceac Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 30 Jun 2026 21:40:29 -0700 Subject: [PATCH 131/137] max parallelism in RepositoryEditor::finish --- lib/src/edit/editor.rs | 44 ++++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs index 6134b6c..8c70923 100644 --- a/lib/src/edit/editor.rs +++ b/lib/src/edit/editor.rs @@ -4,12 +4,13 @@ use std::collections::BTreeMap; use std::collections::HashMap; +use std::num::NonZero; +use std::sync::Arc; use camino::Utf8Path; use camino::Utf8PathBuf; -use futures_util::FutureExt; -use futures_util::TryFutureExt; use semver::Version; +use tokio::sync::Semaphore; use tokio::task::JoinSet; use tufaceous_artifact::ArtifactHash; use tufaceous_artifact::ArtifactVersion; @@ -45,6 +46,7 @@ pub struct RepositoryEditor<'a> { targets: HashMap>>, artifacts: BTreeMap, metadata: BTreeMap, + max_threads: NonZero, } impl<'a> RepositoryEditor<'a> { @@ -57,6 +59,8 @@ impl<'a> RepositoryEditor<'a> { targets: HashMap::new(), artifacts: BTreeMap::new(), metadata: BTreeMap::new(), + max_threads: std::thread::available_parallelism() + .unwrap_or(NonZero::::MIN), }) } @@ -371,6 +375,18 @@ impl<'a> RepositoryEditor<'a> { Ok(self) } + /// Set the maximum number of concurrent threads used for hashing targets + /// when `finish` is called. + /// + /// Defaults to [`std::thread::available_parallelism`] (or `1` if that + /// fails). + pub fn set_max_threads( + self, + max_threads: impl Into>, + ) -> Self { + Self { max_threads: max_threads.into(), ..self } + } + /// Finalize the artifacts and targets, returning an [`UnsignedRepository`]. pub async fn finish(self) -> Result, Error> { let mut artifacts = self.artifacts; @@ -402,23 +418,31 @@ impl<'a> RepositoryEditor<'a> { // we spawn their calculation tasks on a JoinSet. Sources from borrowed // repositories can't be moved into a task, but we already know their // hash. + let semaphore = Arc::new(Semaphore::new(self.max_threads.get())); let mut all_targets = Vec::new(); let mut tasks = JoinSet::new(); for (target_name, sources) in self.targets { for source in sources { let target_name = target_name.clone(); + let semaphore = semaphore.clone(); match source { TargetSource::Bytes(source) => { - let future = source - .into_target() - .map(|target| Ok((target_name, target))); - tasks.spawn(future); + tasks.spawn(async move { + let Ok(_permit) = semaphore.acquire().await else { + unreachable!("we never close the semaphore"); + }; + let target = source.into_target().await; + Ok::<_, Error>((target_name, target)) + }); } TargetSource::File(source) => { - let future = source - .into_target() - .map_ok(|target| (target_name, target)); - tasks.spawn(future); + tasks.spawn(async move { + let Ok(_permit) = semaphore.acquire().await else { + unreachable!("we never close the semaphore"); + }; + let target = source.into_target().await?; + Ok((target_name, target)) + }); } TargetSource::Repository(source) => { all_targets.push((target_name, source.into_target()?)); From 67e9ea291c173882acb94932362f33cce3f5020c Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Wed, 1 Jul 2026 15:35:46 -0700 Subject: [PATCH 132/137] wow --- artifact/src/artifact.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/artifact/src/artifact.rs b/artifact/src/artifact.rs index 262aa5a..0de3073 100644 --- a/artifact/src/artifact.rs +++ b/artifact/src/artifact.rs @@ -115,7 +115,7 @@ impl FromStr for ArtifactHash { type Err = ParseHexError; fn from_str(s: &str) -> std::result::Result { - s.parse() + HexArray::<32>::from_str(s).map(|a| Self(a.0)) } } @@ -123,6 +123,16 @@ impl FromStr for ArtifactHash { mod tests { use super::*; + #[test] + fn fromstr_works() { + assert_eq!( + "0000000000000000000000000000000000000000000000000000000000000000" + .parse::() + .unwrap(), + ArtifactHash([0; 32]) + ); + } + #[test] fn display_respects_padding() { let h = ArtifactHash([0; 32]); From 470478c27b3ba7ae114399b10e33e1fdb96b5045 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Wed, 1 Jul 2026 16:56:13 -0700 Subject: [PATCH 133/137] return io errors while guessing artifacts --- lib/src/edit/hubris_archive.rs | 13 +++++++++++-- lib/src/edit/os_images.rs | 13 ++++++++++++- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/lib/src/edit/hubris_archive.rs b/lib/src/edit/hubris_archive.rs index 0d80abf..178ef6e 100644 --- a/lib/src/edit/hubris_archive.rs +++ b/lib/src/edit/hubris_archive.rs @@ -111,10 +111,19 @@ impl Input> { if !input.file_start.starts_with(b"PK\x03\x04") { return Ok(ControlFlow::Continue(input)); } - let Ok(archive) = input.source.read_hubris_archive().await else { - return Ok(ControlFlow::Continue(input)); + let archive = match input.source.read_hubris_archive().await { + Ok(archive) => archive, + Err(err) => { + if matches!(*err, ErrorKind::ReadHubrisArchive { .. }) { + // seemingly not a valid Hubris archive, keep guessing + return Ok(ControlFlow::Continue(input)); + } + // otherwise, probably an IO error + return Err(err); + } }; let Ok(caboose) = archive.read_caboose() else { + // seemingly not a valid Hubris archive, keep guessing return Ok(ControlFlow::Continue(input)); }; // HACK: We are reading the `image-name` file in the archive, which diff --git a/lib/src/edit/os_images.rs b/lib/src/edit/os_images.rs index f21188c..6d65070 100644 --- a/lib/src/edit/os_images.rs +++ b/lib/src/edit/os_images.rs @@ -109,7 +109,18 @@ impl Input> { // Read the header block from the image and guess whether it's a // recovery image based on the image name. let mut buf = [0; 4096]; - file.read_exact(&mut buf).await.ok()?; + if let Err(source) = file.read_exact(&mut buf).await { + if source.kind() == std::io::ErrorKind::UnexpectedEof { + // The read was fine, the file is just too small to be a real + // OS image. + return None; + } + return Some(Err(ErrorKind::ReadFile { + source, + path: Some(path.to_owned()), + } + .into())); + } if !buf.starts_with(&OXIDE_BOOT_MAGIC) { return None; } From 85ba134d7a80fd7cf48aae0cb15bd2245c2fc087 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Wed, 1 Jul 2026 16:59:02 -0700 Subject: [PATCH 134/137] check max_root_size when blindly trusting 1.root.json --- lib/src/error.rs | 3 +++ lib/src/loader.rs | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/lib/src/error.rs b/lib/src/error.rs index 7608f9f..6843a72 100644 --- a/lib/src/error.rs +++ b/lib/src/error.rs @@ -238,6 +238,8 @@ pub enum ErrorKind { }, #[error("no trust roots provided to load repository")] NoTrustRoots, + #[error("maximum root size {max_size} exceeded")] + MaxRootSizeExceeded { max_size: u64 }, #[error( "artifact ({tags} version={version} hash={hash} length={length}) \ not found", @@ -331,6 +333,7 @@ impl ErrorKind { | ErrorKind::TargetsBaseUrlUnset | ErrorKind::UrlJoin { .. } | ErrorKind::NoTrustRoots + | ErrorKind::MaxRootSizeExceeded { .. } | ErrorKind::ArtifactNotFound { .. } | ErrorKind::TargetNotFound { .. } | ErrorKind::ParseTargetJson { .. } diff --git a/lib/src/loader.rs b/lib/src/loader.rs index 408bc9e..c18f989 100644 --- a/lib/src/loader.rs +++ b/lib/src/loader.rs @@ -319,6 +319,12 @@ impl RepositoryLoader { stream.try_next().await.map_err(ErrorKind::Fetch)? { root.extend_from_slice(&bytes); + if usize64!(root.len()) > self.limits.max_root_size { + return Err(ErrorKind::MaxRootSizeExceeded { + max_size: self.limits.max_root_size, + } + .into()); + } } vec![root] } From 50e55db4a331be3179e72f2f018b2837a3163431 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Wed, 1 Jul 2026 17:23:36 -0700 Subject: [PATCH 135/137] fixup! return io errors while guessing artifacts --- lib/src/edit/os_images.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/edit/os_images.rs b/lib/src/edit/os_images.rs index 6d65070..bf7d710 100644 --- a/lib/src/edit/os_images.rs +++ b/lib/src/edit/os_images.rs @@ -117,7 +117,7 @@ impl Input> { } return Some(Err(ErrorKind::ReadFile { source, - path: Some(path.to_owned()), + path: Some(phase_2_path), } .into())); } From d692cc702f531fd1ca9c621d37b47cefdf85f88a Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 2 Jul 2026 09:37:30 -0700 Subject: [PATCH 136/137] further guessing about guess errors --- lib/src/edit/measurement_corpus.rs | 21 ++++++++++++++------- lib/src/edit/os_images.rs | 14 +++++++++++++- lib/src/edit/zone_image.rs | 4 ++++ 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/lib/src/edit/measurement_corpus.rs b/lib/src/edit/measurement_corpus.rs index 6f55be1..59ff511 100644 --- a/lib/src/edit/measurement_corpus.rs +++ b/lib/src/edit/measurement_corpus.rs @@ -52,13 +52,20 @@ impl Input> { let corim = match ciborium::from_reader::(&mut &*input.file_start) { Ok(corim) => Some(corim), - Err(ciborium::de::Error::Io(err)) - if err.kind() == std::io::ErrorKind::UnexpectedEof => - { - // This was plausibly a CoRIM manifest until we hit the end - // of the buffer, indicating a very high likelihood that if - // we read the entire thing it'd still be a CoRIM manifest. - None + Err(ciborium::de::Error::Io(source)) => { + if source.kind() == std::io::ErrorKind::UnexpectedEof { + // This was plausibly a CoRIM manifest until we hit the + // end of the buffer, indicating a very high likelihood + // that if we read the entire thing it'd still be a + // CoRIM manifest. + None + } else { + return Err(ErrorKind::ReadFile { + source, + path: Some(input.source.path().to_owned()), + } + .into()); + } } Err(_) => { return Ok(ControlFlow::Continue(input)); diff --git a/lib/src/edit/os_images.rs b/lib/src/edit/os_images.rs index bf7d710..84ff5ac 100644 --- a/lib/src/edit/os_images.rs +++ b/lib/src/edit/os_images.rs @@ -105,7 +105,19 @@ impl Input> { version: &ArtifactVersion, ) -> Option> { let phase_2_path = path.join(PHASE_2_PATH); - let mut file = File::open(&phase_2_path).await.ok()?; + let mut file = match File::open(&phase_2_path).await { + Ok(file) => file, + Err(source) => { + if source.kind() == std::io::ErrorKind::NotFound { + return None; + } + return Some(Err(ErrorKind::OpenFile { + source, + path: Some(phase_2_path), + } + .into())); + } + }; // Read the header block from the image and guess whether it's a // recovery image based on the image name. let mut buf = [0; 4096]; diff --git a/lib/src/edit/zone_image.rs b/lib/src/edit/zone_image.rs index f0ba316..0b34e60 100644 --- a/lib/src/edit/zone_image.rs +++ b/lib/src/edit/zone_image.rs @@ -54,6 +54,10 @@ impl Input> { // small, so it should be contained entirely within the first 4K of the // compressed tarball. let mut archive = tar::Archive::new(GzDecoder::new(&*input.file_start)); + // The tar crate returns std::io::Error so it's difficult to + // differentiate between "I couldn't read the file" and "the file isn't + // a tar archive", but at this point in guessing we're unlikely to have + // OS-level IO issues. let Ok(layer_info) = Metadata::read_from_tar(&mut archive) .and_then(Metadata::into_layer_info) else { From 2c4c6422f76f0f5a81834607d2aab8312b91c84d Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 2 Jul 2026 10:00:46 -0700 Subject: [PATCH 137/137] fix os image guess error --- lib/src/edit/os_images.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/src/edit/os_images.rs b/lib/src/edit/os_images.rs index 84ff5ac..4c3a01c 100644 --- a/lib/src/edit/os_images.rs +++ b/lib/src/edit/os_images.rs @@ -108,7 +108,11 @@ impl Input> { let mut file = match File::open(&phase_2_path).await { Ok(file) => file, Err(source) => { - if source.kind() == std::io::ErrorKind::NotFound { + if matches!( + source.kind(), + std::io::ErrorKind::NotFound + | std::io::ErrorKind::NotADirectory + ) { return None; } return Some(Err(ErrorKind::OpenFile {