diff --git a/Cargo.lock b/Cargo.lock index 9885ad0..ce349f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -38,6 +38,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -101,18 +107,6 @@ name = "anyhow" version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b964d184e89d9b6b67dd2715bc8e74cf3107fb2b529990c90cf517326150bf4" -dependencies = [ - "backtrace", -] - -[[package]] -name = "arbitrary" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" -dependencies = [ - "derive_arbitrary", -] [[package]] name = "arc-swap" @@ -129,35 +123,6 @@ dependencies = [ "nodrop", ] -[[package]] -name = "assert_cmd" -version = "2.0.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1835b7f27878de8525dc71410b5a31cdcc5f230aed5ba5df968e09c201b23d" -dependencies = [ - "anstyle", - "bstr", - "doc-comment", - "libc", - "predicates", - "predicates-core", - "predicates-tree", - "wait-timeout", -] - -[[package]] -name = "async-compression" -version = "0.4.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93c1f86859c1af3d514fa19e8323147ff10ea98684e6c7b307912509f50e67b2" -dependencies = [ - "compression-codecs", - "compression-core", - "futures-core", - "pin-project-lite", - "tokio", -] - [[package]] name = "async-recursion" version = "1.1.1" @@ -166,48 +131,31 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", -] - -[[package]] -name = "async-stream" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" -dependencies = [ - "async-stream-impl", - "futures-core", - "pin-project-lite", + "syn 2.0.96", ] [[package]] -name = "async-stream-impl" -version = "0.3.6" +name = "async-trait" +version = "0.1.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +checksum = "644dd749086bf3771a2fbc5f256fdb982d53f011c7d5d560304eafeecebce79d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.96", ] [[package]] -name = "async-trait" -version = "0.1.89" +name = "atomicwrites" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "3ef1bb8d1b645fe38d51dfc331d720fb5fc2c94b440c76cc79c80ff265ca33e3" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", + "rustix 0.38.37", + "tempfile", + "windows-sys 0.52.0", ] -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - [[package]] name = "autocfg" version = "1.4.0" @@ -216,9 +164,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-lc-rs" -version = "1.16.2" +version = "1.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" +checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -227,9 +175,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.39.1" +version = "0.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a25cf98105baa966497416dbd42565ce3a8cf8dbfd59803ec9ad46f3126399" +checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7" dependencies = [ "cc", "cmake", @@ -322,23 +270,22 @@ dependencies = [ ] [[package]] -name = "bstr" -version = "1.10.0" +name = "block-buffer" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" dependencies = [ - "memchr", - "regex-automata", - "serde", + "hybrid-array", ] [[package]] -name = "buf-list" -version = "1.0.3" +name = "bstr" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f56bd1685d994a3e2a3ed802eb1ecee8cb500b0ad4df48cb4d5d1a2f04749c3a" +checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c" dependencies = [ - "bytes", + "memchr", + "serde", ] [[package]] @@ -347,6 +294,18 @@ version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +[[package]] +name = "byte-wrapper" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b041d2328ab2041b4e9d8f21e3b0b5b8bb61e9a23bb9171387771c314ea340" +dependencies = [ + "hex", + "schemars 0.8.22", + "serde_core", + "serde_json", +] + [[package]] name = "byteorder" version = "1.5.0" @@ -382,12 +341,9 @@ dependencies = [ [[package]] name = "camino" -version = "1.2.2" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e629a66d692cb9ff1a1c664e41771b3dcaf961985a9774c0eb0bd1b51cf60a48" -dependencies = [ - "serde_core", -] +checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3" [[package]] name = "camino-tempfile" @@ -419,10 +375,11 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.44" +version = "0.4.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" dependencies = [ + "android-tzdata", "iana-time-zone", "js-sys", "num-traits", @@ -431,6 +388,33 @@ dependencies = [ "windows-link", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "clap" version = "4.5.31" @@ -463,7 +447,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.96", ] [[package]] @@ -487,23 +471,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" -[[package]] -name = "compression-codecs" -version = "0.4.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "680dc087785c5230f8e8843e2e57ac7c1c90488b6a91b88caa265410568f441b" -dependencies = [ - "compression-core", - "flate2", - "memchr", -] - -[[package]] -name = "compression-core" -version = "0.4.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" - [[package]] name = "console" version = "0.15.10" @@ -524,14 +491,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] -name = "core-foundation" -version = "0.9.4" +name = "const-oid" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", -] +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" [[package]] name = "core-foundation-sys" @@ -548,6 +511,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc" version = "3.2.1" @@ -593,6 +565,12 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.1.6" @@ -603,6 +581,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "daft" version = "0.1.1" @@ -621,26 +608,43 @@ checksum = "a851e4efac7afda4a5e1a11fea1d65ef528dbdcff08177d7087e4bb6225e60a9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.96", ] [[package]] -name = "datatest-stable" -version = "0.2.9" +name = "darling" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a560b3fd20463b56397bd457aa71243ccfdcffe696050b66e3b1e0ec0457e7f1" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" dependencies = [ - "camino", - "fancy-regex", - "libtest-mimic", - "walkdir", + "darling_core", + "darling_macro", ] [[package]] -name = "debug-ignore" -version = "1.0.5" +name = "darling_core" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.96", +] + +[[package]] +name = "darling_macro" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffe7ed1d93f4553003e20b629abe9085e1e81b1429520f897f8f8860bc6dfc21" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.96", +] [[package]] name = "der" @@ -648,7 +652,7 @@ version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" dependencies = [ - "const-oid", + "const-oid 0.9.6", "der_derive", "flagset", "pem-rfc7468", @@ -663,7 +667,7 @@ checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.96", ] [[package]] @@ -673,118 +677,49 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ "powerfmt", + "serde_core", ] -[[package]] -name = "derive_arbitrary" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "difflib" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" - [[package]] name = "digest" version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.6", ] [[package]] -name = "displaydoc" -version = "0.2.5" +name = "digest" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", ] [[package]] -name = "doc-comment" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" - -[[package]] -name = "dropshot" -version = "0.17.0" +name = "digest-io" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "409eb76c7ea1623d270393248ff55ec436841fcd724c2e1c9de294291edd35f5" +checksum = "2de63d600bc7fab91180bc17385f29b342468dc8ef2af09dceba450a293de3da" dependencies = [ - "async-compression", - "async-stream", - "async-trait", - "base64 0.22.1", - "bytes", - "camino", - "chrono", - "debug-ignore", - "dropshot_endpoint", - "form_urlencoded", - "futures", - "hostname 0.4.2", - "http", - "http-body-util", - "hyper", - "hyper-util", - "indexmap 2.14.0", - "multer", - "openapiv3", - "paste", - "percent-encoding", - "rustls 0.22.4", - "rustls-pemfile", - "schemars", - "scopeguard", - "semver", - "serde", - "serde_json", - "serde_path_to_error", - "serde_urlencoded", - "sha1", - "slog", - "slog-async", - "slog-bunyan", - "slog-json", - "slog-term", - "thiserror 2.0.18", - "tokio", - "tokio-rustls", - "tokio-util", - "toml 1.1.2+spec-1.1.0", - "uuid", - "version_check", - "waitgroup", + "digest 0.11.3", ] [[package]] -name = "dropshot_endpoint" -version = "0.17.0" +name = "displaydoc" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "906c3adfd4472030607130ed763e9af1b85f7e18832dd22998379d42ff81c28d" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ - "heck", "proc-macro2", "quote", - "semver", - "serde", - "serde_tokenstream", - "syn 2.0.117", + "syn 2.0.96", ] [[package]] @@ -799,27 +734,12 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "feeef44e73baff3a26d371801df019877a9866a8c493d315ab00177843314f35" -[[package]] -name = "either" -version = "1.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7914353092ddf589ad78f25c5c1c21b7f80b0ff8621e7c814c3485b5306da9d" - [[package]] name = "encode_unicode" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" -[[package]] -name = "encoding_rs" -version = "0.8.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" -dependencies = [ - "cfg-if", -] - [[package]] name = "env_logger" version = "0.10.2" @@ -837,15 +757,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" -[[package]] -name = "erased-serde" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c138974f9d5e7fe373eb04df7cae98833802ae4b11c24ac7039a21d5af4b26c" -dependencies = [ - "serde", -] - [[package]] name = "errno" version = "0.3.13" @@ -856,12 +767,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "escape8259" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5692dd7b5a1978a5aeb0ce83b7655c58ca8efdcb79d21036ea249da95afec2c6" - [[package]] name = "expectorate" version = "1.1.0" @@ -873,17 +778,6 @@ dependencies = [ "similar", ] -[[package]] -name = "fancy-regex" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2" -dependencies = [ - "bit-set", - "regex-automata", - "regex-syntax", -] - [[package]] name = "fastrand" version = "2.1.1" @@ -924,27 +818,12 @@ dependencies = [ "miniz_oxide", ] -[[package]] -name = "float-cmp" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" -dependencies = [ - "num-traits", -] - [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foldhash" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" - [[package]] name = "form_urlencoded" version = "1.2.2" @@ -954,15 +833,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fs-err" -version = "2.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88a41f105fe1d5b6b34b2055e3dc59bb79b46b48b2040b9e6c7b4b5de097aa41" -dependencies = [ - "autocfg", -] - [[package]] name = "fs_extra" version = "1.3.0" @@ -977,9 +847,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.32" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -992,9 +862,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.32" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -1002,15 +872,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.32" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -1019,38 +889,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.32" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-macro" -version = "0.3.32" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.96", ] [[package]] name = "futures-sink" -version = "0.3.32" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", @@ -1060,6 +930,7 @@ dependencies = [ "futures-task", "memchr", "pin-project-lite", + "pin-utils", "slab", ] @@ -1092,23 +963,10 @@ checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", "libc", - "r-efi 5.3.0", + "r-efi", "wasi 0.14.2+wasi-0.2.4", ] -[[package]] -name = "getrandom" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" -dependencies = [ - "cfg-if", - "libc", - "r-efi 6.0.0", - "wasip2", - "wasip3", -] - [[package]] name = "gimli" version = "0.31.1" @@ -1129,22 +987,14 @@ dependencies = [ ] [[package]] -name = "h2" -version = "0.4.6" +name = "half" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http", - "indexmap 2.14.0", - "slab", - "tokio", - "tokio-util", - "tracing", + "cfg-if", + "crunchy", + "zerocopy 0.8.27", ] [[package]] @@ -1164,18 +1014,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" -dependencies = [ - "foldhash", -] - -[[package]] -name = "hashbrown" -version = "0.17.0" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" [[package]] name = "heck" @@ -1200,82 +1041,13 @@ name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -dependencies = [ - "serde", -] - -[[package]] -name = "hostname" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" -dependencies = [ - "libc", - "match_cfg", - "winapi", -] - -[[package]] -name = "hostname" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "617aaa3557aef3810a6369d0a99fac8a080891b68bd9f9812a1eeda0c0730cbd" -dependencies = [ - "cfg-if", - "libc", - "windows-link", -] - -[[package]] -name = "http" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" -dependencies = [ - "bytes", - "itoa", -] - -[[package]] -name = "http-body" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" -dependencies = [ - "bytes", - "http", -] - -[[package]] -name = "http-body-util" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" -dependencies = [ - "bytes", - "futures-core", - "http", - "http-body", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" - -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hubtools" -version = "0.4.6" -source = "git+https://github.com/oxidecomputer/hubtools.git?branch=main#3b72a550f2b5ec7ad489b8822b26239b7176bfdd" +version = "0.4.7" +source = "git+https://github.com/oxidecomputer/hubtools.git?rev=2b1ef9b38d75563ea800baa3b17327eec17b1b7a#2b1ef9b38d75563ea800baa3b17327eec17b1b7a" dependencies = [ + "digest 0.10.7", "hex", "lpc55_areas", "lpc55_sign", @@ -1285,64 +1057,19 @@ dependencies = [ "thiserror 1.0.69", "tlvc", "tlvc-text", - "toml 0.7.8", + "toml", "x509-cert", "zerocopy 0.6.6", - "zip 0.6.6", -] - -[[package]] -name = "humantime" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" - -[[package]] -name = "hyper" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" -dependencies = [ - "atomic-waker", - "bytes", - "futures-channel", - "futures-core", - "h2", - "http", - "http-body", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "smallvec 1.14.0", - "tokio", - "want", + "zip", ] [[package]] -name = "hyper-util" -version = "0.1.20" +name = "hybrid-array" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" dependencies = [ - "base64 0.22.1", - "bytes", - "futures-channel", - "futures-util", - "http", - "http-body", - "hyper", - "ipnet", - "libc", - "percent-encoding", - "pin-project-lite", - "socket2", - "system-configuration", - "tokio", - "tower-layer", - "tower-service", - "tracing", - "windows-registry 0.6.1", + "typenum", ] [[package]] @@ -1483,14 +1210,14 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.96", ] [[package]] -name = "id-arena" -version = "2.3.0" +name = "ident_case" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" @@ -1513,12 +1240,6 @@ dependencies = [ "icu_properties", ] -[[package]] -name = "indent_write" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cfe9645a18782869361d9c8732246be7b410ad4e919d3609ebabdac00ba12c3" - [[package]] name = "indexmap" version = "1.9.3" @@ -1527,25 +1248,30 @@ checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown 0.12.3", + "serde", ] [[package]] name = "indexmap" -version = "2.14.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" dependencies = [ "equivalent", - "hashbrown 0.17.0", + "hashbrown 0.15.1", "serde", - "serde_core", ] [[package]] -name = "ipnet" -version = "2.10.1" +name = "io-uring" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" +checksum = "fdd7bddefd0a8833b88a4b68f90dae22c7450d11b354198baee3874fd811b344" +dependencies = [ + "bitflags", + "cfg-if", + "libc", +] [[package]] name = "is-terminal" @@ -1564,15 +1290,6 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" -[[package]] -name = "itertools" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.11" @@ -1590,13 +1307,10 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.97" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1840c94c045fbcf8ba2812c95db44499f7c64910a912551aaaa541decebcacf" +checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" dependencies = [ - "cfg-if", - "futures-util", - "once_cell", "wasm-bindgen", ] @@ -1609,17 +1323,11 @@ dependencies = [ "spin", ] -[[package]] -name = "leb128fmt" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" - [[package]] name = "libc" -version = "0.2.186" +version = "0.2.170" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "875b3680cb2f8f71bdcf9a30f38d48282f5d3c95cbf9b3fa57269bb5d5c06828" [[package]] name = "libm" @@ -1638,18 +1346,6 @@ dependencies = [ "redox_syscall", ] -[[package]] -name = "libtest-mimic" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc0bda45ed5b3a2904262c1bb91e526127aa70e7ef3758aba2ef93cf896b9b58" -dependencies = [ - "clap", - "escape8259", - "termcolor", - "threadpool", -] - [[package]] name = "linux-raw-sys" version = "0.4.14" @@ -1668,16 +1364,6 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" -[[package]] -name = "lock_api" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" -dependencies = [ - "autocfg", - "scopeguard", -] - [[package]] name = "log" version = "0.4.22" @@ -1701,7 +1387,7 @@ version = "0.3.4" source = "git+https://github.com/oxidecomputer/lpc55_support#17d04af60b3a4fd82c77b1a33ca5370943cd25d9" dependencies = [ "byteorder", - "const-oid", + "const-oid 0.9.6", "crc-any", "der", "env_logger", @@ -1714,18 +1400,12 @@ dependencies = [ "rsa", "serde", "serde-hex", - "sha2", + "sha2 0.10.8", "thiserror 1.0.69", "x509-cert", "zerocopy 0.6.6", ] -[[package]] -name = "match_cfg" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" - [[package]] name = "maybe-uninit" version = "2.0.0" @@ -1738,12 +1418,6 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - [[package]] name = "miniz_oxide" version = "0.8.0" @@ -1755,30 +1429,14 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" dependencies = [ + "hermit-abi 0.3.9", "libc", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.61.2", -] - -[[package]] -name = "multer" -version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" -dependencies = [ - "bytes", - "encoding_rs", - "futures-util", - "http", - "httparse", - "memchr", - "mime", - "spin", - "version_check", + "windows-sys 0.52.0", ] [[package]] @@ -1796,19 +1454,12 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" -[[package]] -name = "normalize-line-endings" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" - [[package]] name = "num-bigint-dig" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" dependencies = [ - "byteorder", "lazy_static", "libm", "num-integer", @@ -1822,9 +1473,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" @@ -1856,25 +1507,6 @@ dependencies = [ "libm", ] -[[package]] -name = "num_cpus" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" -dependencies = [ - "hermit-abi 0.3.9", - "libc", -] - -[[package]] -name = "num_threads" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" -dependencies = [ - "libc", -] - [[package]] name = "object" version = "0.30.4" @@ -1913,17 +1545,6 @@ version = "1.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" -[[package]] -name = "openapiv3" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8d427828b22ae1fff2833a03d8486c2c881367f1c336349f307f321e7f4d05" -dependencies = [ - "indexmap 2.14.0", - "serde", - "serde_json", -] - [[package]] name = "packed_struct" version = "0.10.1" @@ -1946,35 +1567,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "parking_lot" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec 1.14.0", - "windows-targets", -] - -[[package]] -name = "parse-size" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "487f2ccd1e17ce8c1bfab3a65c89525af41cfad4c8659021a1e9a2aacd73b89b" - [[package]] name = "paste" version = "1.0.15" @@ -2029,7 +1621,7 @@ checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.96", ] [[package]] @@ -2038,6 +1630,12 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkcs1" version = "0.7.5" @@ -2080,46 +1678,6 @@ dependencies = [ "zerocopy 0.7.35", ] -[[package]] -name = "predicates" -version = "3.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5d19ee57562043d37e82899fade9a22ebab7be9cef5026b07fda9cdd4293573" -dependencies = [ - "anstyle", - "difflib", - "float-cmp", - "normalize-line-endings", - "predicates-core", - "regex", -] - -[[package]] -name = "predicates-core" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae8177bee8e75d6846599c6b9ff679ed51e882816914eec639944d7c9aa11931" - -[[package]] -name = "predicates-tree" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41b740d195ed3166cd147c8047ec98db0e22ec019eb8eeb76d343b795304fb13" -dependencies = [ - "predicates-core", - "termtree", -] - -[[package]] -name = "prettyplease" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" -dependencies = [ - "proc-macro2", - "syn 2.0.117", -] - [[package]] name = "proc-macro2" version = "1.0.93" @@ -2170,12 +1728,6 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" -[[package]] -name = "r-efi" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" - [[package]] name = "radium" version = "0.7.0" @@ -2221,6 +1773,27 @@ dependencies = [ "rand_core", ] +[[package]] +name = "rats-corim" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/rats-corim.git?rev=f0d5d5168d3d31487a56df32c676b0c6240bcc6b#f0d5d5168d3d31487a56df32c676b0c6240bcc6b" +dependencies = [ + "ciborium", + "ciborium-io", + "clap", + "hex", + "serde", + "serde_with", + "strum", + "thiserror 2.0.12", +] + +[[package]] +name = "rawzip" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d9575f44c8cf85bc843ad666dcdf20d05a7753772bef56eb2a5140282b32150" + [[package]] name = "redox_syscall" version = "0.5.7" @@ -2230,6 +1803,26 @@ dependencies = [ "bitflags", ] +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] + [[package]] name = "regex" version = "1.11.1" @@ -2260,48 +1853,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] -name = "reqwest" -version = "0.12.12" +name = "ring" +version = "0.17.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43e734407157c3c2034e0258f5e4473ddb361b1e85f95a66690d67264d7cd1da" -dependencies = [ - "base64 0.22.1", - "bytes", - "futures-core", - "futures-util", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-util", - "ipnet", - "js-sys", - "log", - "mime", - "once_cell", - "percent-encoding", - "pin-project-lite", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tokio-util", - "tower", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-streams", - "web-sys", - "windows-registry 0.2.0", -] - -[[package]] -name = "ring" -version = "0.17.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ac5d832aa16abd7d1def883a8545280c20a60f523a370aa3a9617c2b8550ee" +checksum = "70ac5d832aa16abd7d1def883a8545280c20a60f523a370aa3a9617c2b8550ee" dependencies = [ "cc", "cfg-if", @@ -2325,12 +1880,12 @@ dependencies = [ [[package]] name = "rsa" -version = "0.9.6" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d0e5124fcb30e76a7e79bfee683a2746db83784b86289f6251b54b7950a0dfc" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ - "const-oid", - "digest", + "const-oid 0.9.6", + "digest 0.10.7", "num-bigint-dig", "num-integer", "num-traits", @@ -2338,7 +1893,7 @@ dependencies = [ "pkcs8", "rand_core", "serde", - "sha2", + "sha2 0.10.8", "signature", "spki", "subtle", @@ -2377,20 +1932,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "rustls" -version = "0.22.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" -dependencies = [ - "log", - "ring", - "rustls-pki-types", - "rustls-webpki 0.102.8", - "subtle", - "zeroize", -] - [[package]] name = "rustls" version = "0.23.40" @@ -2401,20 +1942,11 @@ dependencies = [ "log", "once_cell", "rustls-pki-types", - "rustls-webpki 0.103.13", + "rustls-webpki", "subtle", "zeroize", ] -[[package]] -name = "rustls-pemfile" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" -dependencies = [ - "rustls-pki-types", -] - [[package]] name = "rustls-pki-types" version = "1.14.1" @@ -2424,17 +1956,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "rustls-webpki" -version = "0.102.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" -dependencies = [ - "ring", - "rustls-pki-types", - "untrusted 0.9.0", -] - [[package]] name = "rustls-webpki" version = "0.103.13" @@ -2449,9 +1970,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.17" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "rusty-fork" @@ -2488,10 +2009,32 @@ checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615" dependencies = [ "dyn-clone", "schemars_derive", - "semver", "serde", "serde_json", - "uuid", +] + +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9558e172d4e8533736ba97870c4b2cd63f84b382a3d6eb063da41b91cce17289" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", ] [[package]] @@ -2503,23 +2046,16 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.117", + "syn 2.0.96", ] -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - [[package]] name = "semver" -version = "1.0.28" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +checksum = "f79dfe2d285b0488816f30e700a7438c5a73d816b5b7d3ac72fbc48b0d185e03" dependencies = [ "serde", - "serde_core", ] [[package]] @@ -2560,7 +2096,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.96", ] [[package]] @@ -2571,40 +2107,19 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", -] - -[[package]] -name = "serde_human_bytes" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/serde_human_bytes?branch=main#0a09794501b6208120528c3b457d5f3a8cb17424" -dependencies = [ - "hex", - "serde", + "syn 2.0.96", ] [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.139" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "44f86c3acccc9c65b153fe1b85a3be07fe5515274ec9f0653b4a0875731c72a6" dependencies = [ "itoa", "memchr", + "ryu", "serde", - "serde_core", - "zmij", -] - -[[package]] -name = "serde_path_to_error" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" -dependencies = [ - "itoa", - "serde", - "serde_core", ] [[package]] @@ -2626,58 +2141,57 @@ dependencies = [ ] [[package]] -name = "serde_spanned" -version = "1.1.1" +name = "serde_with" +version = "3.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +checksum = "c522100790450cf78eeac1507263d0a350d4d5b30df0c8e1fe051a10c22b376e" dependencies = [ - "serde_core", + "base64 0.22.1", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.7.1", + "schemars 0.9.0", + "schemars 1.1.0", + "serde", + "serde_derive", + "serde_json", + "serde_with_macros", + "time", ] [[package]] -name = "serde_tokenstream" -version = "0.2.3" +name = "serde_with_macros" +version = "3.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7c49585c52c01f13c5c2ebb333f14f6885d76daa768d8a037d28017ec538c69" +checksum = "327ada00f7d64abaac1e55a6911e90cf665aa051b9a561c7006c157f4633135e" dependencies = [ + "darling", "proc-macro2", "quote", - "serde", - "syn 2.0.117", -] - -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", + "syn 2.0.96", ] [[package]] -name = "sha1" -version = "0.10.6" +name = "sha2" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.14", + "digest 0.10.7", ] [[package]] name = "sha2" -version = "0.10.8" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -2686,22 +2200,13 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" -[[package]] -name = "signal-hook-registry" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" -dependencies = [ - "libc", -] - [[package]] name = "signature" version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "digest", + "digest 0.10.7", "rand_core", ] @@ -2722,15 +2227,9 @@ dependencies = [ [[package]] name = "slog" -version = "2.8.2" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b3b8565691b22d2bdfc066426ed48f837fc0c5f2c8cad8d9718f7f99d6995c1" -dependencies = [ - "anyhow", - "erased-serde", - "rustversion", - "serde_core", -] +checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" [[package]] name = "slog-async" @@ -2744,18 +2243,6 @@ dependencies = [ "thread_local", ] -[[package]] -name = "slog-bunyan" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcaaf6e68789d3f0411f1e72bc443214ef252a1038b6e344836e50442541f190" -dependencies = [ - "hostname 0.3.1", - "slog", - "slog-json", - "time", -] - [[package]] name = "slog-envlogger" version = "2.2.0" @@ -2771,18 +2258,6 @@ dependencies = [ "slog-term", ] -[[package]] -name = "slog-json" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e1e53f61af1e3c8b852eef0a9dee29008f55d6dd63794f3f12cef786cf0f219" -dependencies = [ - "serde", - "serde_json", - "slog", - "time", -] - [[package]] name = "slog-scope" version = "4.4.0" @@ -2854,17 +2329,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.117", -] - -[[package]] -name = "socket2" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" -dependencies = [ - "libc", - "windows-sys 0.61.2", + "syn 2.0.96", ] [[package]] @@ -2904,7 +2369,7 @@ dependencies = [ "proc-macro2", "quote", "structmeta-derive", - "syn 2.0.117", + "syn 2.0.96", ] [[package]] @@ -2915,7 +2380,7 @@ checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.96", ] [[package]] @@ -2937,7 +2402,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.117", + "syn 2.0.96", ] [[package]] @@ -2959,24 +2424,15 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.117" +version = "2.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] -[[package]] -name = "sync_wrapper" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" -dependencies = [ - "futures-core", -] - [[package]] name = "synstructure" version = "0.13.1" @@ -2985,28 +2441,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", -] - -[[package]] -name = "system-configuration" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" -dependencies = [ - "bitflags", - "core-foundation", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" -dependencies = [ - "core-foundation-sys", - "libc", + "syn 2.0.96", ] [[package]] @@ -3051,7 +2486,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8c27177b12a6399ffc08b98f76f7c9a1f4fe9fc967c784c5a071fa8d93cf7e1" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3073,12 +2508,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "termtree" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" - [[package]] name = "test-strategy" version = "0.4.0" @@ -3088,7 +2517,7 @@ dependencies = [ "proc-macro2", "quote", "structmeta", - "syn 2.0.117", + "syn 2.0.96", ] [[package]] @@ -3102,11 +2531,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.18" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" dependencies = [ - "thiserror-impl 2.0.18", + "thiserror-impl 2.0.12", ] [[package]] @@ -3117,18 +2546,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.96", ] [[package]] name = "thiserror-impl" -version = "2.0.18" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.96", ] [[package]] @@ -3141,15 +2570,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "threadpool" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" -dependencies = [ - "num_cpus", -] - [[package]] name = "time" version = "0.3.47" @@ -3158,9 +2578,7 @@ checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ "deranged", "itoa", - "libc", "num-conv", - "num_threads", "powerfmt", "serde_core", "time-core", @@ -3226,7 +2644,7 @@ checksum = "8d9ef545650e79f30233c0003bcc2504d7efac6dad25fca40744de773fe2049c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.96", ] [[package]] @@ -3252,48 +2670,58 @@ dependencies = [ [[package]] name = "tokio" -version = "1.52.1" +version = "1.47.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" +checksum = "36cde6a64bcbb101731e7db34c087674206357a5316e6f695f5fef730bd711de" dependencies = [ + "backtrace", "bytes", + "io-uring", "libc", "mio", - "parking_lot", "pin-project-lite", - "signal-hook-registry", - "socket2", + "slab", "tokio-macros", - "windows-sys 0.61.2", ] [[package]] name = "tokio-macros" -version = "2.7.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.96", ] [[package]] -name = "tokio-rustls" -version = "0.25.0" +name = "tokio-stream" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" dependencies = [ - "rustls 0.22.4", - "rustls-pki-types", + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-test" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6d24790a10a7af737693a3e8f1d03faef7e6ca0cc99aae5066f533766de545" +dependencies = [ + "futures-core", "tokio", + "tokio-stream", ] [[package]] name = "tokio-util" -version = "0.7.13" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" +checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" dependencies = [ "bytes", "futures-core", @@ -3309,36 +2737,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" dependencies = [ "serde", - "serde_spanned 0.6.8", - "toml_datetime 0.6.8", - "toml_edit 0.19.15", -] - -[[package]] -name = "toml" -version = "0.8.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148" -dependencies = [ - "serde", - "serde_spanned 0.6.8", - "toml_datetime 0.6.8", - "toml_edit 0.22.24", -] - -[[package]] -name = "toml" -version = "1.1.2+spec-1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" -dependencies = [ - "indexmap 2.14.0", - "serde_core", - "serde_spanned 1.1.1", - "toml_datetime 1.1.1+spec-1.1.0", - "toml_parser", - "toml_writer", - "winnow 1.0.2", + "serde_spanned", + "toml_datetime", + "toml_edit", ] [[package]] @@ -3350,56 +2751,19 @@ dependencies = [ "serde", ] -[[package]] -name = "toml_datetime" -version = "1.1.1+spec-1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" -dependencies = [ - "serde_core", -] - [[package]] name = "toml_edit" version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.7.1", "serde", - "serde_spanned 0.6.8", - "toml_datetime 0.6.8", - "winnow 0.5.40", + "serde_spanned", + "toml_datetime", + "winnow", ] -[[package]] -name = "toml_edit" -version = "0.22.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" -dependencies = [ - "indexmap 2.14.0", - "serde", - "serde_spanned 0.6.8", - "toml_datetime 0.6.8", - "winnow 0.7.3", -] - -[[package]] -name = "toml_parser" -version = "1.1.2+spec-1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" -dependencies = [ - "winnow 1.0.2", -] - -[[package]] -name = "toml_writer" -version = "1.1.1+spec-1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" - [[package]] name = "tough" version = "0.22.0" @@ -3420,8 +2784,7 @@ dependencies = [ "olpc-cjson", "pem", "percent-encoding", - "reqwest", - "rustls 0.23.40", + "rustls", "serde", "serde_json", "serde_plain", @@ -3436,153 +2799,85 @@ dependencies = [ ] [[package]] -name = "tower" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +name = "tufaceous" +version = "0.2.0" dependencies = [ - "futures-core", + "atomicwrites", + "aws-lc-rs", + "bytes", + "camino", + "camino-tempfile", + "chrono", + "ciborium", + "daft", + "digest-io", + "flate2", "futures-util", - "pin-project-lite", - "sync_wrapper", - "tokio", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - -[[package]] -name = "tower-service" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" - -[[package]] -name = "tracing" -version = "0.1.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" -dependencies = [ - "pin-project-lite", - "tracing-core", -] - -[[package]] -name = "tracing-core" -version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" -dependencies = [ - "once_cell", -] - -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - -[[package]] -name = "tufaceous" -version = "0.1.0" -dependencies = [ - "anyhow", - "assert_cmd", - "camino", - "chrono", - "clap", - "console", - "datatest-stable", - "dropshot", - "fs-err", - "humantime", - "predicates", + "hex", + "hubtools", + "memchr", + "rats-corim", + "rawzip", "semver", + "serde", + "serde_json", + "sha2 0.11.0", "slog", - "slog-async", - "slog-envlogger", - "slog-term", - "tempfile", + "tar", + "thiserror 2.0.12", "tokio", + "tokio-test", + "tough", "tufaceous-artifact", - "tufaceous-lib", + "tufaceous-brand-metadata", + "url", ] [[package]] name = "tufaceous-artifact" -version = "0.1.0" +version = "0.2.0" dependencies = [ + "byte-wrapper", "daft", "expectorate", - "hex", + "hubtools", "proptest", "regex", - "schemars", - "semver", + "schemars 0.8.22", "serde", - "serde_human_bytes", "serde_json", - "strum", "test-strategy", - "thiserror 2.0.18", + "thiserror 2.0.12", ] [[package]] -name = "tufaceous-brand-metadata" -version = "0.1.0" +name = "tufaceous-bin" +version = "0.2.0" dependencies = [ + "anyhow", + "camino", + "chrono", + "clap", + "futures-util", "semver", - "serde", - "serde_json", - "tar", + "slog", + "slog-async", + "slog-envlogger", + "slog-term", + "tokio", + "tough", + "tufaceous", "tufaceous-artifact", ] [[package]] -name = "tufaceous-lib" -version = "0.1.0" +name = "tufaceous-brand-metadata" +version = "0.2.0" dependencies = [ - "anyhow", - "async-trait", - "aws-lc-rs", - "base64 0.22.1", - "buf-list", - "bytes", - "camino", - "camino-tempfile", - "chrono", - "daft", - "debug-ignore", - "dropshot", - "flate2", - "fs-err", - "futures", - "hex", - "hubtools", - "indent_write", - "itertools", - "parse-size", - "rand", - "semver", "serde", "serde_json", - "serde_path_to_error", - "sha2", - "slog", "tar", - "thiserror 2.0.18", - "tokio", - "toml 0.8.20", - "tough", "tufaceous-artifact", - "tufaceous-brand-metadata", - "url", - "zip 4.2.0", ] [[package]] @@ -3593,9 +2888,9 @@ checksum = "82205ffd44a9697e34fc145491aa47310f9871540bb7909eaa9365e0a9a46607" [[package]] name = "typenum" -version = "1.17.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "unarray" @@ -3630,12 +2925,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" -[[package]] -name = "unicode-xid" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" - [[package]] name = "untrusted" version = "0.7.1" @@ -3650,9 +2939,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.8" +version = "2.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" dependencies = [ "form_urlencoded", "idna", @@ -3678,18 +2967,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" -[[package]] -name = "uuid" -version = "1.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" -dependencies = [ - "getrandom 0.4.2", - "js-sys", - "serde_core", - "wasm-bindgen", -] - [[package]] name = "version_check" version = "0.9.5" @@ -3705,15 +2982,6 @@ dependencies = [ "libc", ] -[[package]] -name = "waitgroup" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1f50000a783467e6c0200f9d10642f4bc424e39efc1b770203e88b488f79292" -dependencies = [ - "atomic-waker", -] - [[package]] name = "walkdir" version = "2.5.0" @@ -3724,15 +2992,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -3748,54 +3007,37 @@ dependencies = [ "wit-bindgen-rt", ] -[[package]] -name = "wasip2" -version = "1.0.3+wasi-0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" -dependencies = [ - "wit-bindgen 0.57.1", -] - -[[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" -dependencies = [ - "wit-bindgen 0.51.0", -] - [[package]] name = "wasm-bindgen" -version = "0.2.120" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df52b6d9b87e0c74c9edfa1eb2d9bf85e5d63515474513aa50fa181b3c4f5db1" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" dependencies = [ "cfg-if", "once_cell", - "rustversion", "wasm-bindgen-macro", - "wasm-bindgen-shared", ] [[package]] -name = "wasm-bindgen-futures" -version = "0.4.45" +name = "wasm-bindgen-backend" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" dependencies = [ - "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.96", + "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.120" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b1041f495fb322e64aca85f5756b2172e35cd459376e67f2a6c9dffcedb103" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3803,98 +3045,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.120" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dcd0ff20416988a18ac686d4d4d0f6aae9ebf08a389ff5d29012b05af2a1b41" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ - "bumpalo", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.96", + "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.120" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49757b3c82ebf16c57d69365a142940b384176c24df52a087fb748e2085359ea" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "wasm-encoder" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" -dependencies = [ - "leb128fmt", - "wasmparser", -] - -[[package]] -name = "wasm-metadata" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" -dependencies = [ - "anyhow", - "indexmap 2.14.0", - "wasm-encoder", - "wasmparser", -] - -[[package]] -name = "wasm-streams" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e072d4e72f700fb3443d8fe94a39315df013eef1104903cdb0a2abd322bbecd" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - -[[package]] -name = "wasmparser" -version = "0.244.0" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" -dependencies = [ - "bitflags", - "hashbrown 0.15.5", - "indexmap 2.14.0", - "semver", -] - -[[package]] -name = "web-sys" -version = "0.3.72" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" [[package]] name = "winapi-util" @@ -3905,12 +3071,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - [[package]] name = "windows-core" version = "0.52.0" @@ -3922,68 +3082,9 @@ dependencies = [ [[package]] name = "windows-link" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" - -[[package]] -name = "windows-registry" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" -dependencies = [ - "windows-result 0.2.0", - "windows-strings 0.1.0", - "windows-targets", -] - -[[package]] -name = "windows-registry" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" -dependencies = [ - "windows-link", - "windows-result 0.4.1", - "windows-strings 0.5.1", -] - -[[package]] -name = "windows-result" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-result" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-strings" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" -dependencies = [ - "windows-result 0.2.0", - "windows-targets", -] - -[[package]] -name = "windows-strings" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" -dependencies = [ - "windows-link", -] +checksum = "6dccfd733ce2b1753b03b6d3c65edf020262ea35e20ccdf3e288043e6dd620e3" [[package]] name = "windows-sys" @@ -4003,15 +3104,6 @@ dependencies = [ "windows-targets", ] -[[package]] -name = "windows-sys" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" -dependencies = [ - "windows-link", -] - [[package]] name = "windows-targets" version = "0.52.6" @@ -4085,47 +3177,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "winnow" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7f4ea97f6f78012141bcdb6a216b2609f0979ada50b20ca5b52dde2eac2bb1" -dependencies = [ - "memchr", -] - -[[package]] -name = "winnow" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0" - -[[package]] -name = "wit-bindgen" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" -dependencies = [ - "wit-bindgen-rust-macro", -] - -[[package]] -name = "wit-bindgen" -version = "0.57.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" - -[[package]] -name = "wit-bindgen-core" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" -dependencies = [ - "anyhow", - "heck", - "wit-parser", -] - [[package]] name = "wit-bindgen-rt" version = "0.39.0" @@ -4135,74 +3186,6 @@ dependencies = [ "bitflags", ] -[[package]] -name = "wit-bindgen-rust" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" -dependencies = [ - "anyhow", - "heck", - "indexmap 2.14.0", - "prettyplease", - "syn 2.0.117", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" -dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn 2.0.117", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" -dependencies = [ - "anyhow", - "bitflags", - "indexmap 2.14.0", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder", - "wasm-metadata", - "wasmparser", - "wit-parser", -] - -[[package]] -name = "wit-parser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" -dependencies = [ - "anyhow", - "id-arena", - "indexmap 2.14.0", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", - "wasmparser", -] - [[package]] name = "write16" version = "1.0.0" @@ -4230,7 +3213,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" dependencies = [ - "const-oid", + "const-oid 0.9.6", "der", "spki", "tls_codec", @@ -4267,7 +3250,7 @@ checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.96", "synstructure", ] @@ -4291,6 +3274,15 @@ dependencies = [ "zerocopy-derive 0.7.35", ] +[[package]] +name = "zerocopy" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" +dependencies = [ + "zerocopy-derive 0.8.27", +] + [[package]] name = "zerocopy-derive" version = "0.6.6" @@ -4299,7 +3291,7 @@ checksum = "125139de3f6b9d625c39e2efdd73d41bdac468ccd556556440e322be0e1bbd91" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.96", ] [[package]] @@ -4310,7 +3302,18 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.96", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", ] [[package]] @@ -4330,7 +3333,7 @@ checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.96", "synstructure", ] @@ -4351,7 +3354,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.96", ] [[package]] @@ -4373,7 +3376,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.96", ] [[package]] @@ -4387,21 +3390,3 @@ dependencies = [ "crc32fast", "crossbeam-utils", ] - -[[package]] -name = "zip" -version = "4.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ab361742de920c5535880f89bbd611ee62002bf11341d16a5f057bb8ba6899" -dependencies = [ - "arbitrary", - "crc32fast", - "indexmap 2.14.0", - "memchr", -] - -[[package]] -name = "zmij" -version = "1.0.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/Cargo.toml b/Cargo.toml index fa55309..0ded5e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,65 +8,65 @@ members = [ ] [workspace.package] -version = "0.1.0" -edition = "2021" +version = "0.2.0" +edition = "2024" license = "MPL-2.0" publish = false [workspace.dependencies] anyhow = "1.0.96" assert_cmd = "2.0.16" -async-trait = "0.1.86" +atomicwrites = "0.4.4" aws-lc-rs = "1.16.2" -base64 = "0.22.1" -buf-list = "1.0.3" +byte-wrapper = { version = "0.1.0", default-features = false, features = ["alloc", "hex", "serde"] } bytes = "1.11.1" -camino = { version = "1.1.9", features = ["serde1"] } +camino = "1.1.9" camino-tempfile = "1.1.1" chrono = { version = "0.4.40", default-features = false, features = ["std"] } -clap = { version = "4.5.31", features = ["cargo", "derive", "env", "wrap_help"] } -console = { version = "0.15.10", default-features = false } +ciborium = "0.2.2" +clap = { version = "4.5.31", features = ["cargo", "derive", "wrap_help"] } daft = { version = "0.1.1", features = ["derive"] } -datatest-stable = "0.2.9" -debug-ignore = "1.0.5" -dropshot = "0.17.0" +digest-io = "0.1.0" expectorate = "1.1.0" flate2 = "1.0.35" -fs-err = "2.11.0" -futures = "0.3.31" +futures-util = "0.3.31" hex = "0.4.3" -hubtools = { git = "https://github.com/oxidecomputer/hubtools.git", branch = "main" } -humantime = "2.1.0" -indent_write = "2.2.0" -itertools = "0.13.0" -parse-size = "1.1.0" -predicates = "3.1.3" +hubtools = { git = "https://github.com/oxidecomputer/hubtools.git", rev = "2b1ef9b38d75563ea800baa3b17327eec17b1b7a" } +memchr = "2.7.4" proptest = "1.5.0" -rand = "0.8.6" +rats-corim = { git = "https://github.com/oxidecomputer/rats-corim.git", rev = "f0d5d5168d3d31487a56df32c676b0c6240bcc6b" } +rawzip = "0.4.4" regex = "1.11.1" -schemars = { version = "0.8.21", features = ["semver"] } +schemars = "0.8.21" semver = { version = "1.0.25", features = ["serde"] } serde = { version = "1.0.218", features = ["derive"] } -serde_human_bytes = { git = "https://github.com/oxidecomputer/serde_human_bytes", branch = "main" } serde_json = "1.0.139" -serde_path_to_error = "0.1.16" -sha2 = "0.10.8" +sha2 = "0.11.0" slog = "2.7.0" slog-async = "2.8.0" slog-envlogger = "2.2.0" -slog-term = "2.9.1" -strum = { version = "0.26.3", features = ["derive"] } +slog-term = "2.9.2" tar = "0.4.45" -tempfile = "3.13.0" test-strategy = "0.4.0" thiserror = "2.0.12" -tokio = "1.43.0" -toml = "0.8.20" -tough = { version = "0.22.0", features = [ "http" ] } -tufaceous-artifact = { path = "artifact", default-features = false } -tufaceous-brand-metadata = { path = "brand-metadata" } -tufaceous-lib = { path = "lib" } +tokio = "1.47.0" +tokio-test = "0.4.5" +tough = "0.22.0" +tufaceous.path = "lib" +tufaceous-artifact.path = "artifact" +tufaceous-brand-metadata.path = "brand-metadata" url = "2.5.4" -zip = { version = "4.2.0", default-features = false } [workspace.lints.clippy] +pedantic = { level = "warn", priority = -1 } +missing-errors-doc = "allow" +must-use-candidate = "allow" + +[profile.dev.package.flate2] +opt-level = 3 +[profile.dev.package.miniz_oxide] +opt-level = 3 +[profile.dev.package.rawzip] +opt-level = 3 +[profile.dev.package.sha2] +opt-level = 3 diff --git a/README.md b/README.md new file mode 100644 index 0000000..8e67d4c --- /dev/null +++ b/README.md @@ -0,0 +1,68 @@ +# Tufaceous + +Tufaceous is the library and tooling for Oxide update repositories. It is a layer of Oxide-specific tooling on top of [tough](https://github.com/awslabs/tough), a Rust library that implements [TUF](https://theupdateframework.io/). + +This README is intended to provide a high-level overview of the library and instructions for adding to it. More context can be found in [RFD 621](https://rfd.shared.oxide.computer/rfd/0621). + +## Repository format + +We selected ZIP archives as the primary format for our repositories, despite ZIP's many shortcomings. Because Oxide customers are receiving update repositories and uploading them to their systems, we wanted them to be readable for most users so that they could see what is inside. + +The ZIP archive has a top-level `repo` directory. Inside this directory is a `metadata` directory, containing the signed TUF metadata, and a `targets` directory, containing all the targets described by that metadata. + +## Artifacts + +"Artifacts" are our jargon for a file that the Oxide control plane needs in order to perform updates. (All artifacts are "targets", TUF jargon for a file described by the repository metadata, but not all targets are artifacts.) + +Artifacts in the repository are described by the `artifacts-v2.json`. Each artifact has a set of tags that is used by the control plane to decide what to do with the artifact. When creating and using artifacts, these tags are strongly-typed (see `KnownArtifactTags` in `artifact/src/tags.rs`), but they are encoded as an arbitrary key-value mapping. This allows us to potentially release a new type of artifact and use it in the same release, once Nexus has been updated. + +As a general rule it is not possible to use this tooling to create a repository with arbitrary blobs and artifacts in it. The library is very opinionated about how artifacts are named to keep things usable for the CLI and for users of these repositories. + +## Tooling + +Most of the time you're working with repositories generated by the [Omicron release tooling](https://github.com/oxidecomputer/omicron/blob/main/docs/releng.adoc) (`cargo xtask releng` in Omicron). There is also a binary target that is a thin CLI on top of the library, which you can build with `cargo build --release --bin tufaceous`. + +In all commands that read from a repository, use either `-r`/`--trust-roots` to provide a list of trust roots to verify signatures against, or `--blindly-trust` to use the `metadata/1.root.json` root in the repository as the trust root. If you want to load a repository with an expired signature, use `--allow-expired`. For convenience you can combine `--allow-expired` and `--blindly-trust` with `-f`/`--force-load`. + +### Verify a repository + +`tufaceous verify` will read a repository, print its SHA256 checksum, and verify the metadata and checksums of all the metadata in the repository. It will also run a series of compatibility checks on the repository and warn you of any detected issues. + +### Unpack a repository + +Scripts can use `tufaceous list-targets` to get a list of targets from the repository, and `tufaceous show-target` to write a target to stdout. + +To unpack an entire repository, use `unzip` or ZIP tooling provided by your OS. + +### Build a repository + +`tufaceous build` has two required flags: + +- `--output file.zip`, which provides the path to write to +- `--system-version 1.0.0`, which provides the system version + +The arguments are the set of artifacts to add: + +- A measurement corpus is a CoRIM file +- An OS image is a directory containing `cosmo.rom`, `gimlet.rom`, and `zfs.img`, as produced by `helios-build image` +- An RoT, RoT bootloader, or SP image is a Hubris archive (ZIP format) +- A zone image is a tarball that starts with the `oxide.json` Omicron brand metadata + +When building a repository, an Installinator document is automatically created based on the artifacts in the repository. You can avoid this with `--no-installinator-document`. + +### Add/remove artifacts from a repository + +Using `tufaceous edit` you can remove targets from a repository using `--remove-targets` and add new artifacts using `--add-artifacts`. + +The Installinator document is automatically regenerated based on the artifacts in the final repository. You can avoid this with `--no-installinator-document`, which will keep an existing Installinator document if there is one. + +## Adding new artifacts + +In order to add a new type of artifact, you will need to make a series of changes to the library: + +- Modify `KnownArtifactTags` to describe your new artifact +- Add methods for adding an actual artifact and a fake artifact to `RepositoryEditor` +- Add a fake artifact in `Input::fake` (`lib/src/edit/fake.rs`) +- Add code for guessing this type of artifact to `Input::guess` (`lib/src/edit/guess.rs`) + +As a general rule, Tufaceous artifacts are self-describing. The "guess" code is used by the CLI and generates tags from the artifact to avoid needing an unwieldy set of command line options to describe tags. If your artifact is not self-describing, you can staple a header of some kind to the beginning of the artifact that control plane code knows to strip before using the artifact. diff --git a/artifact/Cargo.toml b/artifact/Cargo.toml index c0e8f99..2fc8c80 100644 --- a/artifact/Cargo.toml +++ b/artifact/Cargo.toml @@ -4,20 +4,21 @@ version.workspace = true edition.workspace = true license.workspace = true publish.workspace = true +rust-version = "1.94" [features] +hubtools = ["dep:hubtools"] proptest = ["dep:proptest", "dep:test-strategy"] -schemars = ["dep:schemars"] +schemars = ["dep:schemars", "byte-wrapper/schemars08"] [dependencies] +byte-wrapper.workspace = true daft.workspace = true -hex.workspace = true +hubtools = { workspace = true, optional = true } proptest = { workspace = true, optional = true } schemars = { workspace = true, optional = true } -semver.workspace = true serde.workspace = true -serde_human_bytes.workspace = true -strum.workspace = true +serde_json.workspace = true test-strategy = { workspace = true, optional = true } thiserror.workspace = true diff --git a/artifact/src/artifact.rs b/artifact/src/artifact.rs index 5f66f14..0de3073 100644 --- a/artifact/src/artifact.rs +++ b/artifact/src/artifact.rs @@ -2,58 +2,72 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use std::{fmt, str::FromStr}; +use std::collections::BTreeMap; +use std::fmt::Debug; +use std::fmt::Display; +use std::str::FromStr; +use byte_wrapper::HexArray; +use byte_wrapper::ParseHexError; use daft::Diffable; -use hex::FromHexError; -use semver::Version; -use serde::{Deserialize, Serialize}; - -use crate::{ArtifactKind, ArtifactVersion}; - -/// Description of the `artifacts.json` target found in rack update -/// repositories. -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ArtifactsDocument { - pub system_version: Version, - pub artifacts: Vec, -} +use serde::Deserialize; +use serde::Serialize; + +use crate::ArtifactVersion; +use crate::DisplayTags; +use crate::KnownArtifactTags; + +/// An artifact in a repository. +/// +/// Artifacts are files present in a repository (in TUF parlance, "targets") +/// that the control plane must be aware of and that are used during +/// control-plane-driven update of the system. +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, +)] +pub struct Artifact { + /// The version of the artifact. + pub version: ArtifactVersion, -impl ArtifactsDocument { - /// The name of the artifacts document: `artifacts.json`. - pub const FILE_NAME: &'static str = "artifacts.json"; + /// The artifact's tags. + /// + /// Tags describe how an artifact is to be used by the control plane. + /// In this form, they are an arbitrary mapping of string keys to string + /// values. Using [`Artifact::known_tags`], they can be (fallibly) converted + /// into a strongly-typed description of the artifact. + /// + /// When recording artifacts for later use, the control plane must always + /// record these tags as-is, even if `Artifact::known_tags` returns `None`. + pub tags: BTreeMap, - /// Creates an artifacts document with the provided system version and an - /// empty list of artifacts. - pub fn empty(system_version: Version) -> Self { - Self { system_version, artifacts: Vec::new() } - } + /// The SHA256 checksum of the artifact. + pub hash: ArtifactHash, + + /// The length of the artifact in bytes. + pub length: u64, } -/// Describes an artifact available in the repository. -#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] -pub struct Artifact { - /// Used to differentiate between different series of artifacts of the same - /// kind. This is used by the control plane to select the correct artifact. +impl Artifact { + /// Resolves [`Artifact::tags`] into [`KnownArtifactTags`]. /// - /// For SP and ROT images ([`GimletSp`](crate::KnownArtifactKind::GimletSp), - /// [`GimletRot`](crate::KnownArtifactKind::GimletRot), - /// [`PscSp`](crate::KnownArtifactKind::PscSp), - /// [`PscRot`](crate::KnownArtifactKind::PscRot), - /// [`SwitchSp`](crate::KnownArtifactKind::SwitchSp), - /// [`SwitchRot`](crate::KnownArtifactKind::SwitchRot)), `name` is the value - /// of the board (`BORD`) tag in the image caboose. + /// Returns `None` if the tags do not resolve to any known artifact. This + /// is expected if the repository was generated by a future version of + /// Tufaceous that knows about some new artifact. /// - /// In the future when - /// [`ControlPlane`](crate::KnownArtifactKind::ControlPlane) is split up - /// into separate zones, `name` will be the zone name. - pub name: String, - pub version: ArtifactVersion, - pub kind: ArtifactKind, - pub target: String, + /// If you instead want an error when the tags do not resolve to any known + /// artifact, use [`KnownArtifactTags::from_tags`]. + pub fn known_tags(&self) -> Option { + KnownArtifactTags::from_tags(self.tags.clone()).ok() + } + + /// Returns an adapter for displaying [`Artifact::tags`] as a human-readable + /// string. + pub fn display_tags(&self) -> DisplayTags<'_> { + DisplayTags::from(&self.tags) + } } -/// The hash of an artifact. +/// The SHA256 checksum of an artifact. #[derive( Copy, Clone, @@ -71,89 +85,54 @@ pub struct Artifact { #[cfg_attr(feature = "proptest", derive(test_strategy::Arbitrary))] #[serde(transparent)] pub struct ArtifactHash( - #[serde(with = "serde_human_bytes::hex_array")] + #[serde(with = "HexArray::<32>")] #[cfg_attr( feature = "schemars", - schemars(schema_with = "hex_schema::<32>") + schemars(schema_with = "HexArray::<32>::json_schema") )] pub [u8; 32], ); impl AsRef<[u8]> for ArtifactHash { fn as_ref(&self) -> &[u8] { - &self.0 + self.0.as_slice() } } -impl fmt::Debug for ArtifactHash { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_tuple("ArtifactHash").field(&hex::encode(self.0)).finish() +impl Debug for ArtifactHash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_tuple("ArtifactHash").field(&HexArray(self.0)).finish() } } -impl fmt::Display for ArtifactHash { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - hex::encode(self.0).fmt(f) +impl Display for ArtifactHash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + Display::fmt(&HexArray(self.0), f) } } impl FromStr for ArtifactHash { - type Err = FromHexError; + type Err = ParseHexError; fn from_str(s: &str) -> std::result::Result { - let mut out = [0u8; 32]; - hex::decode_to_slice(s, &mut out)?; - Ok(Self(out)) + HexArray::<32>::from_str(s).map(|a| Self(a.0)) } } -/// A hash-based identifier for an artifact or deployment unit: the kind and -/// hash. -#[derive( - Debug, - Diffable, - Clone, - PartialEq, - Eq, - Hash, - Ord, - PartialOrd, - Deserialize, - Serialize, -)] -#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] -pub struct ArtifactHashId { - /// The kind of artifact this is. - pub kind: ArtifactKind, - - /// The hash of the artifact. - pub hash: ArtifactHash, -} - -impl fmt::Display for ArtifactHashId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}/{}", self.kind, self.hash) - } -} - -/// Produce an OpenAPI schema describing a hex array of a specific length (e.g., -/// a hash digest). -#[cfg(feature = "schemars")] -fn hex_schema( - gen: &mut schemars::SchemaGenerator, -) -> schemars::schema::Schema { - use schemars::JsonSchema; - - let mut schema: schemars::schema::SchemaObject = - ::json_schema(gen).into(); - schema.format = Some(format!("hex string ({N} bytes)")); - schema.into() -} - #[cfg(test)] mod tests { use super::*; + #[test] + fn fromstr_works() { + assert_eq!( + "0000000000000000000000000000000000000000000000000000000000000000" + .parse::() + .unwrap(), + ArtifactHash([0; 32]) + ); + } + #[test] fn display_respects_padding() { let h = ArtifactHash([0; 32]); diff --git a/artifact/src/hubris.rs b/artifact/src/hubris.rs new file mode 100644 index 0000000..db59d26 --- /dev/null +++ b/artifact/src/hubris.rs @@ -0,0 +1,148 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +#![cfg(feature = "hubtools")] + +use hubtools::Caboose; +use hubtools::CabooseError; + +use crate::KnownArtifactTags; +use crate::RotBootloaderTags; +use crate::RotKeyTableHash; +use crate::RotSlot; +use crate::RotTags; +use crate::SpTags; + +impl RotTags { + /// Attempts to read the values of `RotTags` from a Hubris caboose. + /// + /// `slot` is not part of the caboose and must be provided. + /// + /// # Errors + /// + /// Returns an error if the caboose is not valid, if `BORD` is not present, + /// or if `BORD` or `SIGN` are not valid UTF-8 strings. + pub fn from_caboose( + caboose: &Caboose, + slot: RotSlot, + ) -> Result { + Ok(Self { + rot_board: read_board(caboose)?, + rot_rkth: read_sign(caboose)?, + rot_slot: slot, + }) + } +} + +impl RotBootloaderTags { + /// Attempts to read the values of `RotBootloaderTags` from a Hubris + /// caboose. + /// + /// # Errors + /// + /// Returns an error if the caboose is not valid, if `BORD` is not present, + /// or if `BORD` or `SIGN` are not valid UTF-8 strings. + pub fn from_caboose(caboose: &Caboose) -> Result { + Ok(Self { + rot_board: read_board(caboose)?, + rot_rkth: read_sign(caboose)?, + }) + } +} + +impl SpTags { + /// Attempts to read the values of `SpTags` from a Hubris caboose. + /// + /// # Errors + /// + /// Returns an error if the caboose is not valid, or if `BORD` is not + /// present or is not not a valid UTF-8 string. + pub fn from_caboose(caboose: &Caboose) -> Result { + Ok(Self { sp_board: read_board(caboose)? }) + } +} + +impl KnownArtifactTags { + /// Attempts to read the values of [`KnownArtifactTags::Rot`] from + /// a Hubris caboose. + /// + /// `slot` is not part of the caboose and must be provided. + /// + /// # Errors + /// + /// Returns an error if the caboose is not valid, if `BORD` is not present, + /// or if `BORD` or `SIGN` are not valid UTF-8 strings. + pub fn from_rot_caboose( + caboose: &Caboose, + slot: RotSlot, + ) -> Result { + RotTags::from_caboose(caboose, slot).map(KnownArtifactTags::Rot) + } + + /// Attempts to read the values of [`KnownArtifactTags::RotBootloader`] from + /// a Hubris caboose. + /// + /// # Errors + /// + /// Returns an error if the caboose is not valid, if `BORD` is not present, + /// or if `BORD` or `SIGN` are not valid UTF-8 strings. + pub fn from_rot_bootloader_caboose( + caboose: &Caboose, + ) -> Result { + RotBootloaderTags::from_caboose(caboose) + .map(KnownArtifactTags::RotBootloader) + } + + /// Attempts to read the values of [`KnownArtifactTags::Sp`] from a Hubris + /// caboose. + /// + /// # Errors + /// + /// Returns an error if the caboose is not valid, or if `BORD` is not + /// present or is not not a valid UTF-8 string. + pub fn from_sp_caboose( + caboose: &Caboose, + ) -> Result { + SpTags::from_caboose(caboose).map(KnownArtifactTags::Sp) + } +} + +fn utf8<'a>( + s: &'a [u8], + tag: &'static str, +) -> Result<&'a str, ReadCabooseError> { + std::str::from_utf8(s) + .map_err(|source| ReadCabooseError::Utf8 { tag, source }) +} + +fn read_board(caboose: &Caboose) -> Result { + Ok(utf8(caboose.board()?, "BORD")?.to_owned()) +} + +fn read_sign( + caboose: &Caboose, +) -> Result, ReadCabooseError> { + match caboose.sign() { + Ok(sign) => Ok(Some(RotKeyTableHash::new(utf8(sign, "SIGN")?))), + Err(CabooseError::MissingTag { .. }) => Ok(None), + Err(error) => Err(error.into()), + } +} + +/// An error that occurred while reading a Hubris archive. +#[derive(Debug, thiserror::Error)] +pub enum ReadCabooseError { + /// The caboose was not valid, or the required tag was missing. + #[error(transparent)] + Caboose(#[from] hubtools::CabooseError), + /// The tag value was not valid UTF-8. + #[error("{tag} is not valid UTF-8")] + Utf8 { + /// The tag that was not valid UTF-8. + tag: &'static str, + /// The underlying error. + #[source] + source: std::str::Utf8Error, + }, +} diff --git a/artifact/src/installinator.rs b/artifact/src/installinator.rs index 0c35cfd..e65e9c3 100644 --- a/artifact/src/installinator.rs +++ b/artifact/src/installinator.rs @@ -2,66 +2,108 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use semver::Version; -use serde::{Deserialize, Serialize}; +use std::collections::BTreeSet; + +use serde::Deserialize; +use serde::Serialize; use crate::ArtifactHash; +use crate::ArtifactVersion; -/// Artifact-specific information used by installinator. +/// Artifact-specific information used by Installinator. +/// +/// There are (normally) no backwards compatibility constraints for this +/// document, because the version of Tufaceous used to generate it (via the +/// control plane releng tooling) is the same as the version of Tufaceous used +/// to read it (via Installinator), and this is enforced by the use of workspace +/// dependencies in Omicron. +/// +/// Both Wicketd and Nexus treat this document as an opaque blob, since we'd +/// like previous versions of those services to be able to start the "sled +/// recovery" workflow in which the recovery phase 1 image is flashed to a +/// sled, which retrieves the recovery phase 2 image via the SP, which starts +/// Installinator and retrieves the artifacts required to boot normally via the +/// bootstrap network. +/// +/// # v1 compatibility notes +/// +/// In the first release after Tufaceous v2 is integrated in Wicket and +/// Installinator, this will be the situation mupdate is in: +/// +/// - Installinator is using Tufaceous v2 to read the Installinator document. +/// - Wicket on the currently-running system is using Tufaceous v1, and does +/// not extract the composite control plane artifact; thus the individual zone +/// artifacts are not available via the bootstrap network. +/// - The Installinator document was written by Tufaceous v1, because Wicket +/// can only understand v1 repos. (Tufaceous v2 will not generate v1 repos, so +/// releng will use both versions until we can stop generating v1 repos.) +/// +/// This violates the constraint that the same version of Tufaceous is used to +/// generate and read the Installinator document. So, Installinator needs its +/// own backwards compatibility code, which some temporary code in this module +/// supports: /// -/// This document contains information used by installinator to learn about -/// which artifacts to fetch. Unlike -/// [`ArtifactsDocument`](crate::ArtifactsDocument): +/// - `name` is accepted as an alias for [`InstallinatorArtifact::file_name`]. +/// We renamed this field in Tufaceous v2 for clarity. +/// - `ControlPlane` is an accepted variant for [`InstallinatorArtifactKind`]. +/// This indicates to Installinator that an older version of Wicket was used +/// to read the repository, and that it will need to fetch the composite +/// artifact and unpack the control plane zones. /// -/// * This document is treated as an opaque blob by Wicketd and Nexus, since -/// we'd like previous versions of those services to be able to process newer -/// versions of this document. -/// * There are no backwards compatibility constraints for this document. The -/// version of installinator that processes this document is the same as the -/// version of tufaceous that creates it. +/// These temporary compatibility shims can be removed (along with +/// Installinator's v1 compatibility code) in the control plane release after +/// Tufaceous v2 was integrated into Wicket and Installinator, assuming that we +/// continue to disallow skipping major versions during upgrades. #[derive(Debug, Clone, Deserialize, Serialize)] pub struct InstallinatorDocument { - pub system_version: Version, - pub artifacts: Vec, + /// The system version of the repository this document is associated with. + pub system_version: ArtifactVersion, + /// The list of Installinator artifacts. + pub artifacts: BTreeSet, } impl InstallinatorDocument { - /// Creates an installinator document with the provided system version and + /// Creates an Installinator document with the provided system version and /// an empty list of artifacts. - pub fn empty(system_version: Version) -> Self { - Self { system_version, artifacts: Vec::new() } - } - - pub fn file_name(&self) -> String { - format!("installinator_document-{}.json", self.system_version) + pub fn empty(system_version: ArtifactVersion) -> Self { + Self { system_version, artifacts: BTreeSet::new() } } } -/// Describes an artifact available to installinator. -/// -/// The fields here match [`Artifact`](crate::Artifact). -#[derive(Debug, Clone, Deserialize, Serialize)] +/// Describes an artifact available to Installinator. +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize, +)] pub struct InstallinatorArtifact { - pub name: String, - /// The kind of artifact. - /// - /// This is an [`InstallinatorArtifactKind`] rather than an - /// [`ArtifactKind`](crate::ArtifactKind) because there aren't any backwards - /// compatibility constraints with `InstallinatorArtifact`. + /// The artifact kind. + #[serde(flatten)] pub kind: InstallinatorArtifactKind, + /// The SHA256 hash of the artifact. pub hash: ArtifactHash, + /// A file name without directory separators; not necessarily the target + /// name. + // (temporary alias; see "v1 compatibility notes" above) + #[serde(alias = "name")] + pub file_name: String, } -/// The artifact kind for an installinator artifact. -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(rename_all = "snake_case")] +/// The artifact kind for an Installinator artifact. +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize, +)] +#[serde(tag = "kind", rename_all = "snake_case")] pub enum InstallinatorArtifactKind { /// The host phase 2 artifact. - /// - /// This is extracted from the composite host artifact. HostPhase2, - /// The composite control plane artifact. - ControlPlane, - /// Measurement Corpus + /// A measurement corpus. MeasurementCorpus, + /// A control plane zone artifact. + Zone { + /// The zone name, as self-identified in the tarball's `oxide.json` + /// file. This may differ from the file name. + zone_name: String, + }, + /// A tarball of control plane zones. See "v1 compatibility notes" on + /// [`InstallinatorDocument`]. + ControlPlane, } diff --git a/artifact/src/kind.rs b/artifact/src/kind.rs deleted file mode 100644 index d9f9d28..0000000 --- a/artifact/src/kind.rs +++ /dev/null @@ -1,361 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use std::{borrow::Cow, convert::Infallible, fmt, str::FromStr}; - -use daft::Diffable; -use serde::{Deserialize, Serialize}; -use strum::{Display, EnumIter, EnumString, IntoEnumIterator}; -use thiserror::Error; - -/// The kind of artifact we are dealing with. -/// -/// To ensure older versions of Nexus can work with update repositories that -/// describe artifact kinds it is not yet aware of, this is a newtype wrapper -/// around a string. The set of known artifact kinds is described in -/// [`KnownArtifactKind`], and this type has conversions to and from it. -#[derive( - Debug, - Diffable, - Clone, - PartialEq, - Eq, - Hash, - Ord, - PartialOrd, - Deserialize, - Serialize, -)] -#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] -#[serde(transparent)] -pub struct ArtifactKind(Cow<'static, str>); - -impl ArtifactKind { - /// Creates a new `ArtifactKind` from a string. - pub fn new(kind: String) -> Self { - Self(kind.into()) - } - - /// Creates a new `ArtifactKind` from a static string. - pub const fn from_static(kind: &'static str) -> Self { - Self(Cow::Borrowed(kind)) - } - - /// Creates a new `ArtifactKind` from a known kind. - pub fn from_known(kind: KnownArtifactKind) -> Self { - Self::new(kind.to_string()) - } - - /// Returns the kind as a string. - pub fn as_str(&self) -> &str { - &self.0 - } - - /// Converts self to a `KnownArtifactKind`, if it is known. - pub fn to_known(&self) -> Option { - self.0.parse().ok() - } -} - -/// These artifact kinds are not stored anywhere, but are derived from stored -/// kinds and used as internal identifiers. -impl ArtifactKind { - /// Gimlet root of trust bootloader slot image identifier. - /// - /// Derived from [`KnownArtifactKind::GimletRotBootloader`]. - pub const GIMLET_ROT_STAGE0: Self = - Self::from_static("gimlet_rot_bootloader"); - - /// Gimlet root of trust A slot image identifier. - /// - /// Derived from [`KnownArtifactKind::GimletRot`]. - pub const GIMLET_ROT_IMAGE_A: Self = - Self::from_static("gimlet_rot_image_a"); - - /// Gimlet root of trust B slot image identifier. - /// - /// Derived from [`KnownArtifactKind::GimletRot`]. - pub const GIMLET_ROT_IMAGE_B: Self = - Self::from_static("gimlet_rot_image_b"); - - /// PSC root of trust stage0 image identifier. - /// - /// Derived from [`KnownArtifactKind::PscRotBootloader`]. - pub const PSC_ROT_STAGE0: Self = Self::from_static("psc_rot_bootloader"); - - /// PSC root of trust A slot image identifier. - /// - /// Derived from [`KnownArtifactKind::PscRot`]. - pub const PSC_ROT_IMAGE_A: Self = Self::from_static("psc_rot_image_a"); - - /// PSC root of trust B slot image identifier. - /// - /// Derived from [`KnownArtifactKind::PscRot`]. - pub const PSC_ROT_IMAGE_B: Self = Self::from_static("psc_rot_image_b"); - - /// Switch root of trust A slot image identifier. - /// - /// Derived from [`KnownArtifactKind::SwitchRotBootloader`]. - pub const SWITCH_ROT_STAGE0: Self = - Self::from_static("switch_rot_bootloader"); - - /// Switch root of trust A slot image identifier. - /// - /// Derived from [`KnownArtifactKind::SwitchRot`]. - pub const SWITCH_ROT_IMAGE_A: Self = - Self::from_static("switch_rot_image_a"); - - /// Switch root of trust B slot image identifier. - /// - /// Derived from [`KnownArtifactKind::SwitchRot`]. - pub const SWITCH_ROT_IMAGE_B: Self = - Self::from_static("switch_rot_image_b"); - - /// Gimlet Host phase 1 identifier. - /// - /// Derived from [`KnownArtifactKind::Host`]. - pub const GIMLET_HOST_PHASE_1: Self = - Self::from_static("gimlet_host_phase_1"); - - /// Cosmo Host phase 1 identifier. - /// - /// Derived from [`KnownArtifactKind::Host`]. - pub const COSMO_HOST_PHASE_1: Self = - Self::from_static("cosmo_host_phase_1"); - - /// Host phase 2 identifier. - /// - /// Derived from [`KnownArtifactKind::Host`]. - pub const HOST_PHASE_2: Self = Self::from_static("host_phase_2"); - - /// Gimlet Trampoline phase 1 identifier. - /// - /// Derived from [`KnownArtifactKind::Trampoline`]. - pub const GIMLET_TRAMPOLINE_PHASE_1: Self = - Self::from_static("gimlet_trampoline_phase_1"); - - /// Cosmo Trampoline phase 1 identifier. - /// - /// Derived from [`KnownArtifactKind::Trampoline`]. - pub const COSMO_TRAMPOLINE_PHASE_1: Self = - Self::from_static("cosmo_trampoline_phase_1"); - - /// Trampoline phase 2 identifier. - /// - /// Derived from [`KnownArtifactKind::Trampoline`]. - pub const TRAMPOLINE_PHASE_2: Self = - Self::from_static("trampoline_phase_2"); - - /// Measurement Corpus identifier. - /// - /// Derived from [`KnownArtifactKind::MeasurementCorpus`]. - pub const MEASUREMENT_CORPUS: Self = - Self::from_static("measurement_corpus"); -} - -impl From for ArtifactKind { - fn from(kind: KnownArtifactKind) -> Self { - Self::from_known(kind) - } -} - -impl fmt::Display for ArtifactKind { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.fmt(f) - } -} - -impl FromStr for ArtifactKind { - type Err = Infallible; - - fn from_str(s: &str) -> Result { - Ok(Self::new(s.to_owned())) - } -} - -/// Kinds of update artifacts, as used by Nexus to determine what updates are available and by -/// sled-agent to determine how to apply an update when asked. -#[derive( - Clone, - Copy, - Debug, - PartialEq, - Eq, - Hash, - Ord, - PartialOrd, - Display, - EnumString, - Deserialize, - Serialize, - EnumIter, -)] -#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] -#[strum(serialize_all = "snake_case")] -#[serde(rename_all = "snake_case")] -pub enum KnownArtifactKind { - // Sled Artifacts - GimletSp, - GimletRot, - GimletRotBootloader, - Host, - Trampoline, - /// Installinator document identifier. - /// - /// While the installinator document is a metadata file similar to - /// [`ArtifactsDocument`](crate::ArtifactsDocument), Wicketd and Nexus treat - /// it as an opaque single-unit artifact to avoid backwards compatibility - /// issues. - InstallinatorDocument, - /// Composite artifact of all control plane zones - ControlPlane, - /// Individual control plane zone - Zone, - /// MeasurementCorpus - MeasurementCorpus, - - // PSC Artifacts - PscSp, - PscRot, - PscRotBootloader, - - // Switch Artifacts - SwitchSp, - SwitchRot, - SwitchRotBootloader, -} - -impl KnownArtifactKind { - /// For an RoT variant, returns A and B deployment unit kinds. - pub fn rot_a_and_b_kinds( - self, - ) -> Result<(ArtifactKind, ArtifactKind), NotRotVariantError> { - match self { - KnownArtifactKind::GimletRot => Ok(( - ArtifactKind::GIMLET_ROT_IMAGE_A, - ArtifactKind::GIMLET_ROT_IMAGE_B, - )), - KnownArtifactKind::PscRot => Ok(( - ArtifactKind::PSC_ROT_IMAGE_A, - ArtifactKind::PSC_ROT_IMAGE_B, - )), - KnownArtifactKind::SwitchRot => Ok(( - ArtifactKind::SWITCH_ROT_IMAGE_A, - ArtifactKind::SWITCH_ROT_IMAGE_B, - )), - KnownArtifactKind::GimletSp - | KnownArtifactKind::GimletRotBootloader - | KnownArtifactKind::Host - | KnownArtifactKind::Trampoline - | KnownArtifactKind::InstallinatorDocument - | KnownArtifactKind::MeasurementCorpus - | KnownArtifactKind::ControlPlane - | KnownArtifactKind::Zone - | KnownArtifactKind::PscSp - | KnownArtifactKind::PscRotBootloader - | KnownArtifactKind::SwitchSp - | KnownArtifactKind::SwitchRotBootloader => { - Err(NotRotVariantError(self)) - } - } - } - - /// Returns an iterator over all the variants in this struct. - /// - /// This is provided as a helper so dependent packages don't have to pull in - /// strum explicitly. - pub fn iter() -> KnownArtifactKindIter { - ::iter() - } - - /// For fake artifacts we generate for tests, what `SIGN` value do we insert - /// in the Hubris caboose for this artifact kind? - pub fn fake_artifact_hubris_sign(&self) -> Option { - match self { - // Only RoT and RoT bootloader artifacts are signed. We want to use - // a distinct sign value for kind of system, just like real systems - // have. - KnownArtifactKind::GimletRot - | KnownArtifactKind::GimletRotBootloader => { - Some("sign-gimlet".to_string()) - } - KnownArtifactKind::SwitchRot - | KnownArtifactKind::SwitchRotBootloader => { - Some("sign-switch".to_string()) - } - KnownArtifactKind::PscRot | KnownArtifactKind::PscRotBootloader => { - Some("sign-psc".to_string()) - } - - KnownArtifactKind::GimletSp - | KnownArtifactKind::Host - | KnownArtifactKind::Trampoline - | KnownArtifactKind::InstallinatorDocument - | KnownArtifactKind::MeasurementCorpus - | KnownArtifactKind::ControlPlane - | KnownArtifactKind::Zone - | KnownArtifactKind::PscSp - | KnownArtifactKind::SwitchSp => None, - } - } -} - -#[derive(Debug, Error)] -#[error("expected an RoT variant, found {0:?}")] -pub struct NotRotVariantError(KnownArtifactKind); - -#[cfg(test)] -mod tests { - use super::{ArtifactKind, KnownArtifactKind}; - - #[test] - fn serde_artifact_kind() { - assert_eq!( - serde_json::from_str::("\"gimlet_sp\"") - .unwrap() - .to_known(), - Some(KnownArtifactKind::GimletSp) - ); - assert_eq!( - serde_json::from_str::("\"fhqwhgads\"") - .unwrap() - .to_known(), - None, - ); - assert!(serde_json::from_str::("null").is_err()); - - assert_eq!( - serde_json::to_string(&ArtifactKind::from_known( - KnownArtifactKind::GimletSp - )) - .unwrap(), - "\"gimlet_sp\"" - ); - assert_eq!( - serde_json::to_string(&ArtifactKind::new("fhqwhgads".to_string())) - .unwrap(), - "\"fhqwhgads\"" - ); - } - - #[test] - fn known_artifact_kind_roundtrip() { - for kind in KnownArtifactKind::iter() { - let as_string = kind.to_string(); - let kind2 = as_string.parse::().unwrap_or_else( - |error| panic!("error parsing kind {as_string}: {error}"), - ); - assert_eq!(kind, kind2); - } - } - - #[test] - fn display_respects_padding() { - let kind = ArtifactKind::from_static("foo"); - assert_eq!(format!("{kind:x>10}"), "xxxxxxxfoo"); - - let kind = KnownArtifactKind::Host; - assert_eq!(format!("{kind:x>10}"), "xxxxxxhost"); - } -} diff --git a/artifact/src/lib.rs b/artifact/src/lib.rs index 78e7351..b9081e5 100644 --- a/artifact/src/lib.rs +++ b/artifact/src/lib.rs @@ -2,12 +2,34 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +//! tufaceous-artifact defines the core types of tufaceous so that they can be +//! used without the full tufaceous library. + +#![warn(missing_docs)] + mod artifact; +mod hubris; mod installinator; -mod kind; +mod map; +mod metadata; +mod rkth; +mod set; +mod tags; mod version; pub use artifact::*; +#[cfg(feature = "hubtools")] +pub use hubris::*; pub use installinator::*; -pub use kind::*; +pub use metadata::*; +pub use rkth::*; +pub use set::ArtifactSet; +pub use tags::*; pub use version::*; + +/// Types related to [`ArtifactSet`]. +pub mod artifact_set { + pub use crate::set::GetError; + pub use crate::set::IntoIter; + pub use crate::set::Iter; +} diff --git a/artifact/src/map.rs b/artifact/src/map.rs new file mode 100644 index 0000000..2cb43db --- /dev/null +++ b/artifact/src/map.rs @@ -0,0 +1,72 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Utilities for mappings of string keys to string values +//! (`BTreeMap`). +//! +//! Tufaceous uses these string mappings for future-proofing artifact-level +//! metadata ("tags") and repository-level metadata, so that all of the +//! repository information can be losslessly entered into a database even if an +//! older version of Tufaceous is used to read a repository. + +// TODO: This (as of writing) is the only use of serde_json in the entire +// tufaceous-artifact crate, and it's not even for writing/parsing JSON. +// This is "unfortunate but reasonable": anything we expect to be using +// tufaceous-artifact is highly likely to use serde_json anyway. + +use std::collections::BTreeMap; + +use serde::Deserialize; +use serde::Serialize; +use serde::de::DeserializeOwned; + +/// Creates a struct `D` from a string mapping, round-tripping via +/// `serde_json::Value`. +/// +/// This could someday be implemented without round-tripping through +/// `serde_json::Value` but implementing a proper deserializer is a lot of +/// effort that, as of writing this comment, is out of scope. +pub(crate) fn from_map( + map: BTreeMap, +) -> Result { + D::deserialize(map.into_iter().collect::()) +} + +/// Creates a string mapping from a struct `S`, round-tripping via +/// `serde_json::Value`. +/// +/// This could someday be implemented without round-tripping through +/// `serde_json::Value` but implementing a proper serializer is a lot of effort +/// that, as of writing this comment, is out of scope. +pub(crate) fn to_map( + s: &S, +) -> Result, serde_json::Error> { + BTreeMap::deserialize(serde_json::to_value(s)?) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use serde::Deserialize; + + #[test] + fn from_optional_string() { + #[derive(Debug, PartialEq, Deserialize)] + struct Map { + optional: Option, + } + + let mut tags = BTreeMap::new(); + assert_eq!( + super::from_map::(tags.clone()).unwrap(), + Map { optional: None } + ); + tags.insert("optional".to_owned(), "value".to_owned()); + assert_eq!( + super::from_map::(tags).unwrap(), + Map { optional: Some("value".to_owned()) } + ); + } +} diff --git a/artifact/src/metadata.rs b/artifact/src/metadata.rs new file mode 100644 index 0000000..ef20b62 --- /dev/null +++ b/artifact/src/metadata.rs @@ -0,0 +1,77 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::BTreeMap; + +use serde::Deserialize; +use serde::Serialize; + +/// Structured repository-level metadata stored in `artifacts-v2.json`. +// +// NOTE: Similar to KnownArtifactTags, this struct must serialize to +// and deserialize from a mapping of string keys to string values. The +// `metadata_roundtrip` test covers this (crate::map::to_map returns an error if +// this does not hold). +// +// Additionally, it is a requirement that all fields are optional; it must +// be possible to deserialize this struct from older metadata, and the oldest +// metadata is no metadata at all. +// +// As such, a couple of recommendations for adding new fields: +// 1. Wrap all fields in [`Option`] and mark with +// `#[serde(skip_serializing_if = "Option::is_none")]`. +// 2. Mark nested structs with `#[serde(flatten)]`. +// +// If you add metadata, please add to the list in the +// `deserialize_previously_valid` test below. +#[derive(Debug, Default, Clone, PartialEq, Eq, Deserialize, Serialize)] +#[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] +pub struct Metadata { + // We don't actually have any metadata yet, but we wanted to set up the + // scaffolding to make it possible in the future. +} + +impl Metadata { + /// Deserialize structured metadata from a string mapping. + pub fn from_map( + map: BTreeMap, + ) -> Result { + crate::map::from_map(map) + } + + /// Serialize structured metadata into a string mapping. + pub fn to_map( + &self, + ) -> Result, serde_json::Error> { + crate::map::to_map(self) + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use test_strategy::proptest; + + use crate::Metadata; + + #[test] + fn deserialize_previously_valid() { + #[expect(clippy::single_element_loop)] + for previously_valid in [BTreeMap::new()] { + Metadata::from_map(previously_valid).unwrap(); + } + } + + #[test] + fn default_is_empty() { + assert!(Metadata::default().to_map().unwrap().is_empty()); + } + + #[proptest] + fn metadata_roundtrip(metadata: Metadata) { + let map = metadata.to_map().unwrap(); + assert_eq!(Metadata::from_map(map).unwrap(), metadata); + } +} diff --git a/artifact/src/rkth.rs b/artifact/src/rkth.rs new file mode 100644 index 0000000..5c86cd3 --- /dev/null +++ b/artifact/src/rkth.rs @@ -0,0 +1,131 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::HashMap; +use std::fmt::Debug; +use std::fmt::Display; +use std::sync::LazyLock; + +use byte_wrapper::HexArray; +use serde::Deserialize; +use serde::Serialize; + +const CA_LIST: [(&str, &str); 9] = [ + ( + "f62eb434dd27302521d12958a7c3c18c69f6e0239361654f0a09e5cc554e0fab", + "production-cosmo-rot", + ), + ( + "5796ee3433f840519c3bcde73e19ee82ccb6af3857eddaabb928b8d9726d93c0", + "production-gimlet-rot", + ), + ( + "31942f8d53dc908c5cb338bdcecb204785fa87834e8b18f706fc972a42886c8b", + "production-psc-rot", + ), + ( + "5c69a42ee1f1e6cd5f356d14f81d46f8dbee783bb28777334226c689f169c0eb", + "production-sidecar-rot", + ), + ( + "855a51518d13dd696a8102a5b2436bb2e8c39a6c0102909fce0a32adc6b76fb4", + "staging-cosmo-rot", + ), + ( + "11594bb5548a757e918e6fe056e2ad9e084297c9555417a025d8788eacf55daf", + "staging-gimlet-rot", + ), + ( + "f592d8f109b81881221eed5af6438abad9b5df8c220b9129c03763e7e10b22c7", + "staging-psc-rot", + ), + ( + "1432cc4cfe5688c51b55546fe37837c753cfbc89e8c3c6aabcf977fdf0c41e27", + "staging-sidecar-rot", + ), + ( + "84332ef8279df87fbb759dc3866cbc50cd246fbb5a64705a7e60ba86bf01c27d", + "test-bart", + ), +]; +static CA_MAP: LazyLock> = + LazyLock::new(|| HashMap::from(CA_LIST)); + +/// The RoT Key Table Hash, which identifies a CA that signed a Hubris image. +/// +/// Used in [`RotTags`] and [`RotBootloaderTags`]. +/// +/// This is usually a lowercase hexadecimal string, but this is not enforced by +/// the library. +/// +/// [`RotTags`]: crate::RotTags +/// [`RotBootloaderTags`]: crate::RotBootloaderTags +#[derive( + Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, +)] +#[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] +#[serde(transparent)] +pub struct RotKeyTableHash(pub String); + +impl RotKeyTableHash { + /// Create a `RotKeyTableHash` from a string. + pub fn new(s: impl Into) -> Self { + Self(s.into()) + } + + /// Returns a string slice of the underlying string. + pub fn as_str(&self) -> &str { + &self.0 + } + + /// Create a `RotKeyTableHash` from an array of 32 bytes. + pub fn from_bytes(hash: [u8; 32]) -> Self { + Self(HexArray(hash).to_string()) + } + + /// Returns a friendly name for the CA this RKTH represents, if one is + /// known. Returns `None` otherwise. + pub fn friendly_ca_name(&self) -> Option<&'static str> { + CA_MAP.get(self.as_str()).copied() + } +} + +impl AsRef for RotKeyTableHash { + fn as_ref(&self) -> &str { + self.as_str() + } +} + +impl Debug for RotKeyTableHash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + Debug::fmt(&self.0, f) + } +} + +impl Display for RotKeyTableHash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.friendly_ca_name().unwrap_or(self.as_str())) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + + use crate::rkth::CA_LIST; + use crate::rkth::CA_MAP; + + #[test] + fn ensure_consistency() { + // no duplicate keys + assert_eq!(CA_MAP.len(), CA_LIST.len()); + // no duplicate values + let values = CA_MAP.values().collect::>(); + assert_eq!(values.len(), CA_LIST.len()); + // CA_LIST is ordered by friendly name + for [(_, v1), (_, v2)] in CA_LIST.array_windows() { + assert!(v1 < v2, "{v1} incorrectly placed before {v2} in CA_LIST"); + } + } +} diff --git a/artifact/src/set.rs b/artifact/src/set.rs new file mode 100644 index 0000000..9d2da96 --- /dev/null +++ b/artifact/src/set.rs @@ -0,0 +1,276 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::BTreeMap; +use std::collections::BTreeSet; +use std::collections::btree_map::IntoValues; +use std::collections::btree_map::Values; +use std::collections::btree_set; +use std::iter::Chain; +use std::iter::Flatten; + +use serde::Deserialize; +use serde::Serialize; + +use crate::Artifact; +use crate::KnownArtifactTags; + +/// A set of [`Artifact`]s. +/// +/// This type is logically equivalent to `BTreeSet`, but has +/// the additional ability to look up specific artifacts based on their +/// [`KnownArtifactTags`]. (Because of this, artifacts are first sorted by the +/// `Ord` implementation of `KnownArtifactTags`.) +/// +/// Artifacts are consistently ordered within this struct, so two sets +/// containing the same artifacts are equal to each other. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ArtifactSet { + known: BTreeMap>, + unknown: BTreeSet, +} + +impl ArtifactSet { + /// Makes a new, empty `ArtifactSet`. + /// + /// Does not allocate anything on its own. + pub const fn new() -> Self { + Self { known: BTreeMap::new(), unknown: BTreeSet::new() } + } + + /// Adds an artifact to the set. + /// + /// Returns whether the artifact was newly inserted. That is: + /// + /// * If the set did not previously contain an equal artifact, `true` is + /// returned. + /// * If the set already contained an equal value, `false` is returned, and + /// the entry is not updated. + pub fn insert(&mut self, artifact: Artifact) -> bool { + match artifact.known_tags() { + Some(tags) => self.known.entry(tags).or_default().insert(artifact), + None => self.unknown.insert(artifact), + } + } + + /// Returns the number of artifacts in the set. + pub fn len(&self) -> usize { + self.known.values().map(BTreeSet::len).sum::() + + self.unknown.len() + } + + /// Returns `true` if the set contains no artifacts. + pub fn is_empty(&self) -> bool { + self.known.is_empty() && self.unknown.is_empty() + } + + /// Returns `true` if the set contains an artifact equal to `artifact`. + pub fn contains(&self, artifact: &Artifact) -> bool { + if let Some(tags) = artifact.known_tags() { + self.known.get(&tags).is_some_and(|set| set.contains(artifact)) + } else { + self.unknown.contains(artifact) + } + } + + /// Returns the single artifact matching `tags`. + /// + /// # Errors + /// + /// Returns an error if there is not exactly one artifact matching `tags`. + pub fn get_only( + &self, + tags: &KnownArtifactTags, + ) -> Result<&Artifact, GetError> { + let set = self.known.get(tags).ok_or(GetError::NotFound)?; + if set.len() == 1 + && let Some(artifact) = set.first() + { + Ok(artifact) + } else { + Err(GetError::TooMany) + } + } + + /// Returns the set of all artifacts matching `tags`. + /// + /// If you are not certain that you want all matching artifacts, prefer + /// [`ArtifactSet::get_only`] instead. The control plane must not randomly + /// choose from matching artifacts when it expects only one. + pub fn get_all(&self, tags: &KnownArtifactTags) -> &BTreeSet { + static EMPTY: BTreeSet = BTreeSet::new(); + self.known.get(tags).unwrap_or(&EMPTY) + } + + /// Returns an iterator of artifacts where the tags match `predicate`. + /// + /// # Example + /// + /// ``` + /// # use tufaceous_artifact::ArtifactSet; + /// # use tufaceous_artifact::KnownArtifactTags; + /// # let set = ArtifactSet::new(); + /// for zone in set + /// .filter_tags(|tags| matches!(tags, KnownArtifactTags::Zone { .. })) + /// { + /// // ... + /// } + /// ``` + pub fn filter_tags( + &self, + mut predicate: impl FnMut(&KnownArtifactTags) -> bool, + ) -> impl Iterator { + self.known + .iter() + .filter_map(move |(tags, artifacts)| { + predicate(tags).then_some(artifacts) + }) + .flatten() + } + + /// Returns an iterator over the artifacts in the set. + pub fn iter(&self) -> Iter<'_> { + Iter { inner: self.known.values().flatten().chain(&self.unknown) } + } + + /// Returns an iterator over the known artifacts in the set. + pub fn known( + &self, + ) -> impl Iterator< + Item = (&KnownArtifactTags, impl ExactSizeIterator), + > { + self.known.iter().map(|(tags, artifacts)| (tags, artifacts.iter())) + } +} + +impl Extend for ArtifactSet { + fn extend>(&mut self, iter: T) { + for artifact in iter { + self.insert(artifact); + } + } +} + +impl From<[Artifact; N]> for ArtifactSet { + fn from(arr: [Artifact; N]) -> Self { + if N == 0 { ArtifactSet::new() } else { arr.into_iter().collect() } + } +} + +impl FromIterator for ArtifactSet { + fn from_iter>(iter: T) -> Self { + let mut artifacts = Self::default(); + artifacts.extend(iter); + artifacts + } +} + +impl IntoIterator for ArtifactSet { + type Item = Artifact; + type IntoIter = IntoIter; + + fn into_iter(self) -> Self::IntoIter { + IntoIter { + inner: self.known.into_values().flatten().chain(self.unknown), + } + } +} + +impl<'a> IntoIterator for &'a ArtifactSet { + type Item = &'a Artifact; + type IntoIter = Iter<'a>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +impl<'de> Deserialize<'de> for ArtifactSet { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct Visitor; + + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = ArtifactSet; + + fn expecting( + &self, + f: &mut std::fmt::Formatter, + ) -> std::fmt::Result { + write!(f, "a sequence") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + std::iter::from_fn(|| seq.next_element().transpose()).collect() + } + } + + deserializer.deserialize_seq(Visitor) + } +} + +impl Serialize for ArtifactSet { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.collect_seq(self.iter()) + } +} + +/// An iterator that moves values out of an [`ArtifactSet`]. +/// +/// This struct is created by [`ArtifactSet::into_iter`] (provided by the +/// [`IntoIterator`] trait). +#[derive(Debug)] +pub struct IntoIter { + inner: Chain< + Flatten>>, + btree_set::IntoIter, + >, +} + +impl Iterator for IntoIter { + type Item = Artifact; + + fn next(&mut self) -> Option { + self.inner.next() + } +} + +/// An iterator that borrows values from an [`ArtifactSet`]. +/// +/// This struct is created by [`ArtifactSet::iter`]. +#[derive(Debug, Clone)] +pub struct Iter<'a> { + inner: Chain< + Flatten>>, + btree_set::Iter<'a, Artifact>, + >, +} + +impl<'a> Iterator for Iter<'a> { + type Item = &'a Artifact; + + fn next(&mut self) -> Option { + self.inner.next() + } +} + +/// Returned by [`ArtifactSet::get_only`] when there is not exactly one matching +/// artifact. +#[derive(Debug, thiserror::Error)] +pub enum GetError { + /// An artifact matching the requested tags was not found. + #[error("artifact not found")] + NotFound, + /// There were multiple artifacts matching the requested tags. + #[error("more than one artifact found")] + TooMany, +} diff --git a/artifact/src/tags.rs b/artifact/src/tags.rs new file mode 100644 index 0000000..84aaf86 --- /dev/null +++ b/artifact/src/tags.rs @@ -0,0 +1,419 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::borrow::Cow; +use std::collections::BTreeMap; +use std::fmt::Display; + +use serde::Deserialize; +use serde::Serialize; + +use crate::RotKeyTableHash; +use crate::installinator::InstallinatorArtifactKind; + +/// Sets of artifact tags known to the control plane. +// +// NOTE: This struct must serialize and deserialize from a mapping of +// string keys to string values. The `tags_roundtrip` test covers this +// (crate::map::to_map returns an error if this does not hold). +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, +)] +#[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] +#[serde(tag = "kind", rename_all = "snake_case")] +#[non_exhaustive] +pub enum KnownArtifactTags { + /// JSON document describing the artifacts Installinator is responsible for + /// writing during mupdate and sled recovery. + InstallinatorDocument, + + /// CoRIM manifest for remote attestation. + MeasurementCorpus, + + /// Phase 1 OS image, written to flash. Differs based on the target board. + OsPhase1(OsPhase1Tags), + + /// Phase 2 OS image, a ZFS pool with an Oxide-specific header written to + /// M.2 storage. Common across all target boards. + OsPhase2(OsPhase2Tags), + + /// Hubris archive for a Root of Trust image. + Rot(RotTags), + + /// Hubris archive for a Root of Trust bootloader. + RotBootloader(RotBootloaderTags), + + /// Hubris archive for a Service Processor image. + Sp(SpTags), + + /// Tarball of a Helios zone image. + Zone(ZoneTags), +} + +impl KnownArtifactTags { + /// Resolves known tags from a tag mapping. + /// + /// # Errors + /// + /// Returns an error if the `kind` tag is missing, not a known kind, or + /// required tags for that kind are not present. + pub fn from_tags( + tags: BTreeMap, + ) -> Result { + crate::map::from_map(tags) + } + + /// Converts these known tags to a tag mapping. + pub fn to_tags( + &self, + ) -> Result, serde_json::Error> { + crate::map::to_map(self) + } + + /// Converts these tags into an [`InstallinatorArtifactKind`] if this + /// artifact kind should be included in the Installinator document. + pub fn to_installinator(&self) -> Option { + match self { + KnownArtifactTags::MeasurementCorpus => { + Some(InstallinatorArtifactKind::MeasurementCorpus) + } + KnownArtifactTags::OsPhase2(OsPhase2Tags { + os_variant: OsVariant::Host, + }) => Some(InstallinatorArtifactKind::HostPhase2), + KnownArtifactTags::Zone(ZoneTags { zone_name }) => { + Some(InstallinatorArtifactKind::Zone { + zone_name: zone_name.clone(), + }) + } + + KnownArtifactTags::InstallinatorDocument + | KnownArtifactTags::OsPhase1(_) + | KnownArtifactTags::OsPhase2(OsPhase2Tags { + os_variant: OsVariant::Recovery, + }) + | KnownArtifactTags::Rot(_) + | KnownArtifactTags::RotBootloader(_) + | KnownArtifactTags::Sp(_) => None, + } + } + + /// Returns `true` if this tag set is allowed to appear in the repository + /// more than once. + /// + /// # ⚠️ Causality Hazard ⚠️ + /// + /// This definition is subject to change in future repositories. Do **not** + /// use this to enforce logic if you are possibly reading a repository made + /// by a different version of Tufaceous. + pub fn allow_multiple_artifacts(&self) -> bool { + match self { + KnownArtifactTags::MeasurementCorpus => true, + + KnownArtifactTags::InstallinatorDocument + | KnownArtifactTags::OsPhase1(_) + | KnownArtifactTags::OsPhase2(_) + | KnownArtifactTags::Rot(_) + | KnownArtifactTags::RotBootloader(_) + | KnownArtifactTags::Sp(_) + | KnownArtifactTags::Zone(_) => false, + } + } +} + +impl Display for KnownArtifactTags { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let tags = self.to_tags().map_err(|_| std::fmt::Error)?; + DisplayTags::from(tags).fmt(f) + } +} + +macro_rules! display_serialize { + ($ty:ty) => { + impl std::fmt::Display for $ty { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.serialize(f) + } + } + }; +} + +/// The inner value of [`KnownArtifactTags::OsPhase1`]. +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, +)] +#[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] +pub struct OsPhase1Tags { + /// OS board artifact tag (gimlet or cosmo). + pub os_board: OsBoard, + /// OS variant artifact tag (host or recovery). + pub os_variant: OsVariant, +} + +impl From for KnownArtifactTags { + fn from(tags: OsPhase1Tags) -> Self { + KnownArtifactTags::OsPhase1(tags) + } +} + +/// The inner value of [`KnownArtifactTags::OsPhase2`]. +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, +)] +#[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] +pub struct OsPhase2Tags { + /// OS variant artifact tag (host or recovery). + pub os_variant: OsVariant, +} + +impl From for KnownArtifactTags { + fn from(tags: OsPhase2Tags) -> Self { + KnownArtifactTags::OsPhase2(tags) + } +} + +/// OS variant artifact tag (host or recovery). +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Deserialize, + Serialize, +)] +#[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] +#[serde(rename_all = "snake_case")] +pub enum OsVariant { + /// The host OS. + Host, + /// The recovery OS (sometimes called the trampoline OS), which contains + /// Installinator and is used to install the host OS. + Recovery, +} +display_serialize!(OsVariant); + +/// OS board artifact tag (gimlet, cosmo, etc). +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, +)] +#[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] +#[serde(transparent)] +pub struct OsBoard(pub Cow<'static, str>); + +impl OsBoard { + /// First-generation SP3 compute sled. + pub const GIMLET: Self = Self(Cow::Borrowed("gimlet")); + /// Second-generation SP5 compute sled. + pub const COSMO: Self = Self(Cow::Borrowed("cosmo")); +} + +impl Display for OsBoard { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + Display::fmt(&self.0, f) + } +} + +/// The inner value of [`KnownArtifactTags::Rot`]. +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, +)] +#[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] +pub struct RotTags { + /// The `BORD` field in the caboose (such as `oxide-rot-1`). + pub rot_board: String, + /// The RoT Key Table Hash (RKTH). This is the `SIGN` field in the caboose. + /// + /// For unsigned images this will not be present; this will generally + /// never occur in release repos but can be useful on hardware that has + /// not fully made it through manufacturing yet. + #[serde(skip_serializing_if = "Option::is_none")] + pub rot_rkth: Option, + /// RoT images are compiled for two different locations in flash; this + /// identifies which slot this image belongs to. + pub rot_slot: RotSlot, +} + +impl From for KnownArtifactTags { + fn from(tags: RotTags) -> Self { + KnownArtifactTags::Rot(tags) + } +} + +/// RoT slot artifact tag (A or B). +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Deserialize, + Serialize, +)] +#[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] +#[serde(rename_all = "snake_case")] +pub enum RotSlot { + /// Slot A. + A, + /// Slot B. + B, +} +display_serialize!(RotSlot); + +/// The inner value of [`KnownArtifactTags::RotBootloader`]. +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, +)] +#[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] +pub struct RotBootloaderTags { + /// The `BORD` field in the caboose (such as `oxide-rot-1`). + pub rot_board: String, + /// The `SIGN` field in the caboose. This is the Root Key Table Hash + /// (RKTH). + /// + /// For unsigned images this will not be present; this will generally + /// never occur in release repos but can be useful on hardware that has + /// not fully made it through manufacturing yet. + #[serde(skip_serializing_if = "Option::is_none")] + pub rot_rkth: Option, +} + +impl From for KnownArtifactTags { + fn from(tags: RotBootloaderTags) -> Self { + KnownArtifactTags::RotBootloader(tags) + } +} + +/// The inner value of [`KnownArtifactTags::Sp`]. +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, +)] +#[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] +pub struct SpTags { + /// The `BORD` field in the caboose (such as `cosmo-b`). + pub sp_board: String, +} + +impl From for KnownArtifactTags { + fn from(tags: SpTags) -> Self { + KnownArtifactTags::Sp(tags) + } +} + +/// The inner value of [`KnownArtifactTags::Zone`]. +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize, +)] +#[cfg_attr(any(test, feature = "proptest"), derive(test_strategy::Arbitrary))] +pub struct ZoneTags { + /// The zone name, as self-identified in the tarball's `oxide.json` + /// file. This may differ from the filename on disk. + pub zone_name: String, +} + +impl From for KnownArtifactTags { + fn from(tags: ZoneTags) -> Self { + KnownArtifactTags::Zone(tags) + } +} + +/// An adapter that implements [`Display`] for a set of tags. +/// +/// This is intended for error and log messages and is not a portable format. +/// +/// # Example +/// +/// ``` +/// # use std::collections::BTreeMap; +/// # use tufaceous_artifact::DisplayTags; +/// let tags = BTreeMap::from([ +/// ("foo".to_string(), "yes".to_string()), +/// ("bar".to_string(), "definitely".to_string()), +/// ("kind".to_string(), "thing".to_string()), +/// ]); +/// assert_eq!( +/// DisplayTags::from(&tags).to_string(), +/// "kind=thing,bar=definitely,foo=yes" +/// ); +/// ``` +#[derive(Debug, Clone)] +pub struct DisplayTags<'a>(pub(crate) Cow<'a, BTreeMap>); + +impl Display for DisplayTags<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut comma = ""; + let kind = self.0.get_key_value("kind").into_iter(); + let remainder = self.0.iter().filter(|(k, _)| *k != "kind"); + for (key, value) in kind.chain(remainder) { + write!(f, "{comma}{key}={value}")?; + comma = ","; + } + Ok(()) + } +} + +impl From> for DisplayTags<'static> { + fn from(tags: BTreeMap) -> Self { + Self(Cow::Owned(tags)) + } +} + +impl<'a> From<&'a BTreeMap> for DisplayTags<'a> { + fn from(tags: &'a BTreeMap) -> Self { + Self(Cow::Borrowed(tags)) + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use test_strategy::proptest; + + use crate::KnownArtifactTags; + use crate::RotKeyTableHash; + use crate::RotSlot; + use crate::RotTags; + + /// [`KnownArtifactTags::to_tags`] is not allowed to fail (but is marked + /// fallible because it's Serde under the hood); its result must round trip + /// back to the same value when deserialized. + #[proptest] + fn tags_roundtrip(tags: KnownArtifactTags) { + let map = tags.to_tags().unwrap(); + assert_eq!(KnownArtifactTags::from_tags(map).unwrap(), tags); + } + + #[test] + fn rot_rkth() { + let mut tags = BTreeMap::from([ + ("kind".to_owned(), "rot".to_owned()), + ("rot_board".to_owned(), "oxide-rot-1".to_owned()), + // rot_rkth not included + ("rot_slot".to_owned(), "a".to_owned()), + ]); + assert_eq!( + KnownArtifactTags::from_tags(tags.clone()).unwrap(), + KnownArtifactTags::Rot(RotTags { + rot_board: "oxide-rot-1".to_owned(), + rot_rkth: None, + rot_slot: RotSlot::A + }) + ); + tags.insert("rot_rkth".to_owned(), "meow".to_owned()); + assert_eq!( + KnownArtifactTags::from_tags(tags).unwrap(), + KnownArtifactTags::Rot(RotTags { + rot_board: "oxide-rot-1".to_owned(), + rot_rkth: Some(RotKeyTableHash::new("meow")), + rot_slot: RotSlot::A + }) + ); + } +} diff --git a/artifact/src/version.rs b/artifact/src/version.rs index 5478c7c..4bb71d0 100644 --- a/artifact/src/version.rs +++ b/artifact/src/version.rs @@ -2,10 +2,13 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use std::{borrow::Cow, fmt, str::FromStr}; +use std::borrow::Cow; +use std::fmt; +use std::str::FromStr; use daft::Diffable; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; +use serde::Serialize; use thiserror::Error; /// An artifact version. @@ -65,6 +68,10 @@ impl ArtifactVersion { } /// Creates a new `ArtifactVersion` from a static string. + /// + /// # Errors + /// + /// Returns an error if the string is not a valid artifact version. pub fn new_static( version: &'static str, ) -> Result { @@ -75,8 +82,11 @@ impl ArtifactVersion { } } - /// Creates a new `ArtifactVersion` at compile time, panicking if it is - /// invalid. + /// Creates a new `ArtifactVersion` from a static string. + /// + /// # Panics + /// + /// Panics if the string is not a valid artifact version. pub const fn new_const(s: &'static str) -> Self { match validate_version(s) { Ok(()) => Self(Cow::Borrowed(s)), @@ -168,19 +178,28 @@ static PROPTEST_REGEX: &str = { #[derive(Debug, Clone, PartialEq, Eq, Error)] #[non_exhaustive] pub enum ArtifactVersionError { + /// The version string was empty. #[error("version is empty")] Empty, + /// The version string was too long. #[error( "version is too long ({len} bytes, max {})", ArtifactVersion::MAX_LEN )] - TooLong { len: usize }, + TooLong { + /// Length of the version string that was too long. + len: usize, + }, #[error( "version contains invalid byte `{}` (allowed: {})", b.escape_ascii(), ArtifactVersion::REGEX )] - InvalidByte { b: u8 }, + /// The version string contained an invalid byte. + InvalidByte { + /// The first invalid byte. + b: u8, + }, } impl ArtifactVersionError { diff --git a/bin/Cargo.toml b/bin/Cargo.toml index 69c6c03..583569e 100644 --- a/bin/Cargo.toml +++ b/bin/Cargo.toml @@ -1,38 +1,29 @@ [package] -name = "tufaceous" +name = "tufaceous-bin" version.workspace = true edition.workspace = true license.workspace = true publish.workspace = true -[[test]] -name = "manifest-tests" -harness = false +[[bin]] +name = "tufaceous" +path = "src/main.rs" [dependencies] -anyhow = { workspace = true, features = ["backtrace"] } +anyhow.workspace = true camino.workspace = true chrono.workspace = true clap.workspace = true -console.workspace = true -humantime.workspace = true +futures-util.workspace = true semver.workspace = true slog.workspace = true slog-async.workspace = true slog-envlogger.workspace = true slog-term.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +tough.workspace = true +tufaceous.workspace = true tufaceous-artifact.workspace = true -tufaceous-lib.workspace = true - -[dev-dependencies] -assert_cmd.workspace = true -datatest-stable.workspace = true -dropshot.workspace = true -fs-err.workspace = true -predicates.workspace = true -tempfile.workspace = true -tokio = { workspace = true, features = ["test-util"] } [lints] workspace = true diff --git a/bin/README.adoc b/bin/README.adoc deleted file mode 100644 index 86e4cfc..0000000 --- a/bin/README.adoc +++ /dev/null @@ -1,36 +0,0 @@ -# tufaceous - -Rack update repository generation tool. - -## TUF, keys and lifetime - -Rack update repositories use TUF. Consider reading https://theupdateframework.io/overview/[the TUF overview] and https://theupdateframework.io/metadata/[a high level summary of the metadata mandated by the specification]. - -The only keys currently supported by tufaceous are Ed25519 keys. Support for hardware-backed keys is planned. - -Each role has an expiration date. The default is one week, suitable for development testing. This can be modified with the `--expiry` option. - -## init - -Create a new repository in the current directory with `tufaceous init`. - -To change the target directory, use `-r/--repo`. This is accepted on all subcommands, and needs to come before the subcommand because Clap is picky. - -This will generate a new Ed25519 private key and display it on stderr if no keys are provided. - -Currently if keys are provided, they are allowed to sign all roles. For the time being if you need more advanced editing of the root role, use https://crates.io/crates/tuftool[tuftool]'s `root` subcommands. - -## add zones - -Usage: - ----- -tuftool [-r PATH/TO/REPO] add-zone [--name NAME] ZONE_TAR_GZ VERSION ----- - -Example: - ----- -$ tuftool add-zone out/nexus.tar.gz 0.0.0 -added zone nexus, version 0.0.0 ----- diff --git a/bin/invalid-manifests/duplicate-artifact-2.toml b/bin/invalid-manifests/duplicate-artifact-2.toml deleted file mode 100644 index d8f5a59..0000000 --- a/bin/invalid-manifests/duplicate-artifact-2.toml +++ /dev/null @@ -1,24 +0,0 @@ -# This manifest contains artifacts with the same kind and hash, but different -# artifact versions. - -system_version = "1.0.0" - -[[artifact.gimlet_sp]] -name = "fake-gimlet-sp" -version = "1.0.0" -source = { kind = "fake", size = "1MiB", data_version = "0.0.0" } - -[[artifact.switch_sp]] -name = "fake-switch-sp" -version = "1.0.0" -source = { kind = "fake", size = "1MiB", data_version = "0.0.0" } - -[[artifact.gimlet_sp]] -name = "fake-gimlet-sp" -version = "2.0.0" -source = { kind = "fake", size = "1MiB", data_version = "0.0.0" } - -[[artifact.switch_sp]] -name = "fake-switch-sp" -version = "2.0.0" -source = { kind = "fake", size = "1MiB", data_version = "0.0.0" } diff --git a/bin/invalid-manifests/duplicate-artifact.toml b/bin/invalid-manifests/duplicate-artifact.toml deleted file mode 100644 index 298ee87..0000000 --- a/bin/invalid-manifests/duplicate-artifact.toml +++ /dev/null @@ -1,14 +0,0 @@ -# This manifest contains two artifacts with the same name and version, which -# will hash to the same value. - -system_version = "1.0.0" - -[[artifact.gimlet_sp]] -name = "fake-gimlet-sp" -version = "1.0.0" -source = { kind = "fake", size = "1MiB" } - -[[artifact.gimlet_sp]] -name = "fake-gimlet-sp" -version = "1.0.0" -source = { kind = "fake", size = "1MiB" } diff --git a/bin/invalid-manifests/duplicate-zone-file-name.toml b/bin/invalid-manifests/duplicate-zone-file-name.toml deleted file mode 100644 index 0ccf489..0000000 --- a/bin/invalid-manifests/duplicate-zone-file-name.toml +++ /dev/null @@ -1,14 +0,0 @@ -# This manifest has two control plane zones that hash to the same contents. This -# is not allowed. - -system_version = "1.0.0" - -[[artifact.control_plane]] -name = "fake-control-plane" -version = "1.0.0" -[artifact.control_plane.source] -kind = "composite-control-plane" -zones = [ - { kind = "fake", artifact_name = "zone-1", file_name = "zone1.tar.gz", size = "1MiB" }, - { kind = "fake", artifact_name = "zone-1", file_name = "zone1-dup.tar.gz", size = "1MiB" }, -] diff --git a/bin/manifests/fake-non-semver.toml b/bin/manifests/fake-non-semver.toml deleted file mode 100644 index 1ef2508..0000000 --- a/bin/manifests/fake-non-semver.toml +++ /dev/null @@ -1,94 +0,0 @@ -# This is an artifact manifest that generates fake entries for all components. -# Some of the components have non-semver artifact versions. -# -# This is completely non-functional and is only useful for testing archive -# extraction in other parts of the repository. - -system_version = "2.0.0" - -[[artifact.gimlet_sp]] -name = "fake-gimlet-sp" -version = "2.0.0" -source = { kind = "fake", size = "1MiB" } - -[[artifact.gimlet_rot]] -name = "fake-gimlet-rot" -version = "2.0.0" -[artifact.gimlet_rot.source] -kind = "composite-rot" -archive_a = { kind = "fake", size = "512KiB" } -archive_b = { kind = "fake", size = "512KiB" } - -[[artifact.host]] -name = "fake-host" -version = "2.0.0" -[artifact.host.source] -kind = "composite-host" -gimlet_phase_1 = { kind = "fake", size = "512KiB" } -cosmo_phase_1 = { kind = "fake", size = "512KiB" } -phase_2 = { kind = "fake", size = "1MiB" } - -[[artifact.trampoline]] -name = "fake-trampoline" -version = "non-semver" -[artifact.trampoline.source] -kind = "composite-host" -gimlet_phase_1 = { kind = "fake", size = "512KiB" } -cosmo_phase_1 = { kind = "fake", size = "512KiB" } -phase_2 = { kind = "fake", size = "1MiB" } - -[[artifact.control_plane]] -name = "fake-control-plane" -version = "2.0.0" -[artifact.control_plane.source] -kind = "composite-control-plane" -zones = [ - { kind = "fake", artifact_name = "zone-1", file_name = "zone1.tar.gz", size = "1MiB" }, - { kind = "fake", artifact_name = "zone-2", file_name = "zone2.tar.gz", size = "1MiB" }, -] - -[[artifact.psc_sp]] -name = "fake-psc-sp" -version = "2.0.0" -source = { kind = "fake", size = "1MiB" } - -[[artifact.psc_rot]] -name = "fake-psc-rot" -version = "2.0.0" -[artifact.psc_rot.source] -kind = "composite-rot" -archive_a = { kind = "fake", size = "512KiB" } -archive_b = { kind = "fake", size = "512KiB" } - -[[artifact.switch_sp]] -name = "fake-switch-sp" -version = "2.0.0" -source = { kind = "fake", size = "1MiB" } - -[[artifact.switch_rot]] -name = "fake-switch-rot" -version = "2.0.0" -[artifact.switch_rot.source] -kind = "composite-rot" -archive_a = { kind = "fake", size = "512KiB" } -archive_b = { kind = "fake", size = "512KiB" } - -[[artifact.gimlet_rot_bootloader]] -name = "fake-gimlet-rot-bootloader" -version = "2.0.0" -source = { kind = "fake", size = "1MiB" } - -[[artifact.psc_rot_bootloader]] -name = "fake-psc-rot-bootloader" -version = "2.0.0" -source = { kind = "fake", size = "1MiB" } - -[[artifact.switch_rot_bootloader]] -name = "fake-switch-rot-bootloader" -version = "non-semver-2" -source = { kind = "fake", size = "1MiB" } - -[[artifact.measurement_corpus]] -name = "fake-corpus" -version = "1.0.0" -source = { kind = "fake", size ="1MiB" } diff --git a/bin/manifests/fake.toml b/bin/manifests/fake.toml deleted file mode 100644 index ae5e3f3..0000000 --- a/bin/manifests/fake.toml +++ /dev/null @@ -1,93 +0,0 @@ -# This is an artifact manifest that generates fake entries for all components. -# This is completely non-functional and is only useful for testing archive -# extraction in other parts of the repository. - -system_version = "1.0.0" - -[[artifact.gimlet_sp]] -name = "fake-gimlet-sp" -version = "1.0.0" -source = { kind = "fake", size = "1MiB" } - -[[artifact.gimlet_rot]] -name = "fake-gimlet-rot" -version = "1.0.0" -[artifact.gimlet_rot.source] -kind = "composite-rot" -archive_a = { kind = "fake", size = "512KiB" } -archive_b = { kind = "fake", size = "512KiB" } - -[[artifact.host]] -name = "fake-host" -version = "1.0.0" -[artifact.host.source] -kind = "composite-host" -gimlet_phase_1 = { kind = "fake", size = "512KiB" } -cosmo_phase_1 = { kind = "fake", size = "512KiB" } -phase_2 = { kind = "fake", size = "1MiB" } - -[[artifact.trampoline]] -name = "fake-trampoline" -version = "1.0.0" -[artifact.trampoline.source] -kind = "composite-host" -gimlet_phase_1 = { kind = "fake", size = "512KiB" } -cosmo_phase_1 = { kind = "fake", size = "512KiB" } -phase_2 = { kind = "fake", size = "1MiB" } - -[[artifact.control_plane]] -name = "fake-control-plane" -version = "1.0.0" -[artifact.control_plane.source] -kind = "composite-control-plane" -zones = [ - { kind = "fake", artifact_name = "zone-1", file_name = "zone1.tar.gz", size = "1MiB" }, - { kind = "fake", artifact_name = "zone-2", file_name = "zone2.tar.gz", size = "1MiB" }, -] - -[[artifact.psc_sp]] -name = "fake-psc-sp" -version = "1.0.0" -source = { kind = "fake", size = "1MiB" } - -[[artifact.psc_rot]] -name = "fake-psc-rot" -version = "1.0.0" -[artifact.psc_rot.source] -kind = "composite-rot" -archive_a = { kind = "fake", size = "512KiB" } -archive_b = { kind = "fake", size = "512KiB" } - -[[artifact.switch_sp]] -name = "fake-switch-sp" -version = "1.0.0" -source = { kind = "fake", size = "1MiB" } - -[[artifact.switch_rot]] -name = "fake-switch-rot" -version = "1.0.0" -[artifact.switch_rot.source] -kind = "composite-rot" -archive_a = { kind = "fake", size = "512KiB" } -archive_b = { kind = "fake", size = "512KiB" } - -[[artifact.gimlet_rot_bootloader]] -name = "fake-gimlet-rot-bootloader" -version = "1.0.0" -source = { kind = "fake", size = "1MiB" } - -[[artifact.psc_rot_bootloader]] -name = "fake-psc-rot-bootloader" -version = "1.0.0" -source = { kind = "fake", size = "1MiB" } - -[[artifact.switch_rot_bootloader]] -name = "fake-switch-rot-bootloader" -version = "1.0.0" -source = { kind = "fake", size = "1MiB" } - -[[artifact.measurement_corpus]] -name = "fake-corpus" -version = "1.0.0" -source = { kind = "fake", size ="1MiB" } - diff --git a/bin/src/build.rs b/bin/src/build.rs new file mode 100644 index 0000000..11ab4e6 --- /dev/null +++ b/bin/src/build.rs @@ -0,0 +1,58 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use anyhow::Result; +use camino::Utf8PathBuf; +use chrono::Utc; +use clap::Parser; +use semver::Version; +use tufaceous::edit::RepositoryEditor; + +use crate::sign::SignOptions; + +#[derive(Debug, Parser)] +pub struct Args { + /// Artifacts to add to the repository + /// + /// For OS images, use the directory generated by helios-build that contains + /// zfs.img, gimlet.rom, etc. + artifacts: Vec, + /// Add a standard set of fake artifacts + #[clap(long)] + fake: bool, + /// Do not generate an Installinator document + #[clap(long)] + no_installinator_document: bool, + /// ZIP archive output path + #[clap(short = 'o', long)] + output: Utf8PathBuf, + #[clap(flatten)] + sign_options: SignOptions, + /// The system version (required) + #[clap(short = 'V', long)] + system_version: Version, +} + +impl Args { + pub async fn run(self) -> Result<()> { + let mut editor = if self.fake { + RepositoryEditor::fake(self.system_version) + } else { + RepositoryEditor::new(self.system_version) + }?; + editor = editor.set_generate_installinator_document( + !self.no_installinator_document, + ); + for path in self.artifacts { + editor = editor.guess_artifact(path).await?; + } + let unsigned = editor.finish().await?; + self.sign_options + .sign(unsigned) + .await? + .write_zip_file(self.output, Utc::now()) + .await?; + Ok(()) + } +} diff --git a/bin/src/date.rs b/bin/src/date.rs deleted file mode 100644 index 52d6f4c..0000000 --- a/bin/src/date.rs +++ /dev/null @@ -1,21 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use anyhow::Result; -use chrono::{DateTime, Duration, Timelike, Utc}; - -/// Parser for datelike command line arguments. Can accept a duration (e.g. -/// "1w") or an ISO8601 timestamp. -pub(crate) fn parse_duration_or_datetime(s: &str) -> Result> { - match humantime::parse_duration(s) { - Ok(duration) => { - // Remove nanoseconds from the timestamp to keep it less - // overwhelming. `Timelike::with_nanosecond` returns None only when - // passed a value over 2 billion - let now = Utc::now().with_nanosecond(0).unwrap(); - Ok(now + Duration::from_std(duration)?) - } - Err(_) => Ok(s.parse()?), - } -} diff --git a/bin/src/dispatch.rs b/bin/src/dispatch.rs deleted file mode 100644 index a9b6274..0000000 --- a/bin/src/dispatch.rs +++ /dev/null @@ -1,200 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use anyhow::{Context, Result, bail}; -use camino::Utf8PathBuf; -use chrono::{DateTime, Utc}; -use clap::Parser; -use tufaceous_artifact::{ArtifactsDocument, KnownArtifactKind}; -use tufaceous_lib::assemble::{ArtifactManifest, OmicronRepoAssembler}; -use tufaceous_lib::{ArchiveExtractor, Key, OmicronRepo}; - -#[derive(Debug, Parser)] -pub struct Args { - #[clap(subcommand)] - command: Command, - - #[clap( - short = 'k', - long = "key", - env = "TUFACEOUS_KEY", - required = false, - global = true - )] - keys: Vec, - - #[clap(long, value_parser = crate::date::parse_duration_or_datetime, default_value = "7d", global = true)] - expiry: DateTime, - - /// TUF repository path (default: current working directory) - #[clap(short = 'r', long, global = true)] - repo: Option, -} - -impl Args { - /// Executes these arguments. - pub async fn exec(self, log: &slog::Logger) -> Result<()> { - match self.command { - Command::Assemble { - manifest_path, - output_path, - build_dir, - no_generate_key, - skip_all_present, - allow_non_semver, - no_installinator_document, - } => { - // The filename must end with "zip". - if output_path.extension() != Some("zip") { - bail!("output path `{output_path}` must end with .zip"); - } - - let manifest = ArtifactManifest::from_path(&manifest_path) - .context("error reading manifest")?; - if !allow_non_semver { - manifest.verify_all_semver()?; - } - if !skip_all_present { - manifest.verify_all_present()?; - } - - let keys = maybe_generate_keys(self.keys, no_generate_key)?; - let mut assembler = OmicronRepoAssembler::new( - log, - manifest, - keys, - self.expiry, - !no_installinator_document, - output_path, - ); - if let Some(dir) = build_dir { - assembler.set_build_dir(dir); - } - - assembler.build().await?; - - Ok(()) - } - Command::Extract { - archive_file, - dest, - no_installinator_document, - } => { - let mut extractor = ArchiveExtractor::from_path(&archive_file)?; - extractor.extract(&dest)?; - - // Now load the repository and ensure it's valid. - let repo = OmicronRepo::load_untrusted(log, &dest) - .await - .with_context(|| { - format!( - "error loading extracted repository at `{dest}` \ - (extracted files are still available)" - ) - })?; - let artifacts = - repo.read_artifacts().await.with_context(|| { - format!( - "error loading {} from extracted archive \ - at `{dest}`", - ArtifactsDocument::FILE_NAME - ) - })?; - if !no_installinator_document { - // There should be a reference to an installinator document - // within artifacts_document. - let installinator_doc_artifact = artifacts - .artifacts - .iter() - .find(|artifact| { - artifact.kind.to_known() - == Some( - KnownArtifactKind::InstallinatorDocument, - ) - }) - .context( - "could not find artifact with kind \ - `installinator_document` within artifacts.json", - )?; - - repo.read_installinator_document( - &installinator_doc_artifact.target, - ) - .await - .with_context(|| { - format!( - "error loading {} from extracted archive \ - at `{dest}`", - installinator_doc_artifact.target, - ) - })?; - } - - Ok(()) - } - } - } -} - -#[derive(Debug, Parser)] -enum Command { - /// Assembles a repository from a provided manifest. - Assemble { - /// Path to artifact manifest. - manifest_path: Utf8PathBuf, - - /// The path to write the archive to (must end with .zip). - output_path: Utf8PathBuf, - - /// Directory to use for building artifacts [default: temporary directory] - #[clap(long)] - build_dir: Option, - - /// Disable random key generation and exit if no keys are provided - #[clap(long)] - no_generate_key: bool, - - /// Skip checking to ensure all expected artifacts are present. - #[clap(long)] - skip_all_present: bool, - - /// Allow versions to be non-semver. - /// - /// Transitional option for v13 -> v14. After v14, versions will be - /// allowed to be non-semver by default. - #[clap(long)] - allow_non_semver: bool, - - /// Do not include the installinator document. - /// - /// Transitional option for v15 -> v16, meant to be used for testing. - #[clap(long)] - no_installinator_document: bool, - }, - /// Validates and extracts a repository created by the `assemble` command. - Extract { - /// The file to extract. - archive_file: Utf8PathBuf, - - /// The destination to extract the file to. - dest: Utf8PathBuf, - - /// Indicate that the file does not contain an installinator document. - #[clap(long)] - no_installinator_document: bool, - }, -} - -fn maybe_generate_keys( - keys: Vec, - no_generate_key: bool, -) -> Result> { - Ok(if !no_generate_key && keys.is_empty() { - let key = Key::generate_ed25519()?; - crate::hint::generated_key(&key); - vec![key] - } else { - keys - }) -} diff --git a/bin/src/edit.rs b/bin/src/edit.rs new file mode 100644 index 0000000..79bed69 --- /dev/null +++ b/bin/src/edit.rs @@ -0,0 +1,85 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use anyhow::Result; +use camino::Utf8PathBuf; +use chrono::Utc; +use clap::Parser; +use semver::Version; +use tufaceous::edit::RepositoryEditor; +use tufaceous_artifact::KnownArtifactTags; + +use crate::load::LoadOptions; +use crate::sign::SignOptions; + +#[derive(Debug, Parser)] +pub struct Args { + /// Artifacts to add to the repository + /// + /// For OS images, use the directory generated by helios-build that contains + /// zfs.img, gimlet.rom, etc. + #[arg(short = 'a', long, num_args(1..))] + add_artifacts: Vec, + #[clap(flatten)] + load_options: LoadOptions, + /// Do not generate a new Installinator document, and leave the existing + /// Installinator document if one exists + #[clap(long)] + no_installinator_document: bool, + /// ZIP archive output path [default: rewrite input repository] + #[clap(short = 'o', long)] + output: Option, + /// Targets to remove from the repository + /// + /// These are the paths under "repo/targets/" in a ZIP archive. The + /// list-targets command lists the target names. + #[arg(short = 'd', long, num_args(1..))] + remove_targets: Vec, + /// Input repository path + repo: Utf8PathBuf, + #[clap(flatten)] + sign_options: SignOptions, + /// Edit the system version + #[clap(short = 'V', long)] + system_version: Option, +} + +impl Args { + pub async fn run(self) -> Result<()> { + let repo = self + .load_options + .loader() + .await? + .load_zip_path(self.repo.clone(), &crate::LOG) + .await?; + let mut editor = RepositoryEditor::from_repo(&repo)? + .set_generate_installinator_document( + !self.no_installinator_document, + ); + + if !self.no_installinator_document { + editor = editor + .remove_artifacts(&KnownArtifactTags::InstallinatorDocument)?; + } + + if let Some(system_version) = self.system_version { + editor = editor.set_system_version(system_version)?; + } + for target_name in &self.remove_targets { + editor = editor.remove_target(target_name); + } + for path in self.add_artifacts { + editor = editor.guess_artifact(path).await?; + } + + let unsigned = editor.finish().await?; + let output = self.output.unwrap_or(self.repo); + self.sign_options + .sign(unsigned) + .await? + .write_zip_file(output, Utc::now()) + .await?; + Ok(()) + } +} diff --git a/bin/src/hint.rs b/bin/src/hint.rs deleted file mode 100644 index 74385e3..0000000 --- a/bin/src/hint.rs +++ /dev/null @@ -1,29 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use tufaceous_lib::Key; - -fn print_hint(hint: &str) { - for line in hint.trim().lines() { - eprintln!("{}", console::style(format!("hint: {}", line)).yellow()); - } -} - -pub(crate) fn generated_key(key: &Key) { - print_hint(&format!( - r#" -Generated a random key: - - {key} - -To modify this repository, you will need this key. Use the -k/--key -command line flag or the TUFACEOUS_KEY environment variable: - - export TUFACEOUS_KEY={key} - -To prevent this default behavior, use --no-generate-key. - "#, - key = console::style(key).italic() - )) -} diff --git a/bin/src/lib.rs b/bin/src/lib.rs deleted file mode 100644 index 65ff581..0000000 --- a/bin/src/lib.rs +++ /dev/null @@ -1,9 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -mod date; -mod dispatch; -mod hint; - -pub use dispatch::*; diff --git a/bin/src/list_targets.rs b/bin/src/list_targets.rs new file mode 100644 index 0000000..6ad8568 --- /dev/null +++ b/bin/src/list_targets.rs @@ -0,0 +1,35 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::BTreeSet; + +use anyhow::Result; +use camino::Utf8PathBuf; +use clap::Parser; + +use crate::load::LoadOptions; + +#[derive(Debug, Parser)] +pub struct Args { + #[clap(flatten)] + load_options: LoadOptions, + /// Input repository path + repo: Utf8PathBuf, +} + +impl Args { + pub async fn run(self) -> Result<()> { + let repo = self + .load_options + .loader() + .await? + .load_zip_path(self.repo.clone(), &crate::LOG) + .await?; + let target_names = repo.targets().keys().collect::>(); + for target_name in target_names { + println!("{}", target_name.raw()); + } + Ok(()) + } +} diff --git a/bin/src/load.rs b/bin/src/load.rs new file mode 100644 index 0000000..77bfc58 --- /dev/null +++ b/bin/src/load.rs @@ -0,0 +1,146 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use camino::Utf8PathBuf; +use clap::Args; +use clap::builder::ArgPredicate; +use tufaceous::ExpirationEnforcement; +use tufaceous::RepositoryLoader; +use tufaceous::error::Error; +use tufaceous::error::ErrorKind; + +#[derive(Debug, Args)] +#[cfg_attr(test, derive(PartialEq))] +pub struct LoadOptions { + /// Allow loading a repository with an expired signature. + #[clap( + long, + default_value_if("force_load", ArgPredicate::IsPresent, "true") + )] + allow_expired: bool, + + /// Blindly trust whatever root is contained in the repository + /// + /// This reads the trust root by fetching metadata/1.root.json from the + /// repository, then proceeds with normal verification. The repository must + /// be validly signed, but with this set we do not care who signed it. + #[clap( + long, + default_value_if("force_load", ArgPredicate::IsPresent, "true") + )] + blindly_trust: bool, + + /// Shorthand for --allow-expired --blindly-trust. + #[clap(short('f'), long)] + force_load: bool, + + /// Trust roots to verify the repository + /// + /// This flag is required unless --blindly-trust or --force-load is set. + #[clap( + short('r'), + long("trust-roots"), + conflicts_with("blindly_trust"), + required_unless_present_any(["blindly_trust", "force_load"]) + )] + trust_roots: Vec, +} + +impl LoadOptions { + pub async fn loader(self) -> Result { + let mut loader = RepositoryLoader::new(); + for trust_root in self.trust_roots { + let root = + tokio::fs::read(&trust_root).await.map_err(|source| { + ErrorKind::ReadFile { source, path: Some(trust_root) } + })?; + loader = loader.trust_root(root); + } + if self.allow_expired { + loader = + loader.expiration_enforcement(ExpirationEnforcement::Unsafe); + } + if self.blindly_trust { + loader = loader.unsafe_blindly_trust_repo(); + } + Ok(loader) + } +} + +#[cfg(test)] +mod tests { + use clap::Parser; + + use crate::load::LoadOptions; + + #[derive(Debug, Parser)] + struct Args { + #[clap(flatten)] + inner: LoadOptions, + } + + #[test] + fn test_force_load() { + assert_eq!( + Args::try_parse_from(["", "-f"]).unwrap().inner, + LoadOptions { + allow_expired: true, + blindly_trust: true, + force_load: true, + trust_roots: vec![], + } + ); + // ... but they default to false if --force-load is not used: + assert_eq!( + Args::try_parse_from(["", "-r", "/dev/null"]).unwrap().inner, + LoadOptions { + allow_expired: false, + blindly_trust: false, + force_load: false, + trust_roots: vec!["/dev/null".into()], + } + ); + // --force-load doesn't conflict with --trust-roots + assert_eq!( + Args::try_parse_from(["", "-f", "-r", "/dev/null"]).unwrap().inner, + LoadOptions { + allow_expired: true, + blindly_trust: true, + force_load: true, + trust_roots: vec!["/dev/null".into()], + } + ); + } + + #[test] + fn test_trust_roots_required() { + let error = Args::try_parse_from([""]).unwrap_err().to_string(); + assert!( + error.contains( + "the following required arguments were not provided:" + ) + ); + assert!(error.contains("--trust-roots ")); + + // ... but not if --blindly-trust or --force-load is set! + Args::try_parse_from(["", "--blindly-trust"]).unwrap(); + Args::try_parse_from(["", "--force-load"]).unwrap(); + } + + #[test] + fn test_trust_roots_conflicts() { + let error = Args::try_parse_from([ + "", + "--trust-roots", + "/dev/null", + "--blindly-trust", + ]) + .unwrap_err() + .to_string(); + assert!(error.contains( + "the argument '--trust-roots ' \ + cannot be used with '--blindly-trust'" + )); + } +} diff --git a/bin/src/main.rs b/bin/src/main.rs index 014817e..34812fe 100644 --- a/bin/src/main.rs +++ b/bin/src/main.rs @@ -2,34 +2,64 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +mod build; +mod edit; +mod list_targets; +mod load; +mod show_target; +mod sign; +mod verify; + +use std::sync::LazyLock; + use anyhow::Result; use clap::Parser; use slog::Drain; -use tufaceous::Args; - -#[tokio::main] -async fn main() -> Result<()> { - let log = setup_log(); - let args = Args::parse(); - args.exec(&log).await -} - -fn setup_log() -> slog::Logger { - let stderr_drain = stderr_env_drain("RUST_LOG"); - let drain = slog_async::Async::new(stderr_drain).build().fuse(); - slog::Logger::root(drain, slog::o!()) -} +use slog::Logger; -fn stderr_env_drain(env_var: &str) -> impl Drain { +static LOG: LazyLock = LazyLock::new(|| { let stderr_decorator = slog_term::TermDecorator::new().build(); let stderr_drain = slog_term::FullFormat::new(stderr_decorator).build().fuse(); let mut builder = slog_envlogger::LogBuilder::new(stderr_drain); - if let Ok(s) = std::env::var(env_var) { + if let Ok(s) = std::env::var("RUST_LOG") { builder = builder.parse(&s); } else { // Log at the info level by default. builder = builder.filter(None, slog::FilterLevel::Info); } - builder.build() + let stderr_drain = builder.build(); + let drain = slog_async::Async::new(stderr_drain).build().fuse(); + Logger::root(drain, slog::o!()) +}); + +#[derive(Debug, Parser)] +enum Command { + /// Build a new repository from a set of artifacts + Build(build::Args), + /// Add artifacts to and remove targets from a repository + /// + /// Calling this with no options will re-sign the repository with a + /// newly-generated root and overwrite it. + Edit(edit::Args), + /// List the targets in a repository + #[clap(alias = "ls")] + ListTargets(list_targets::Args), + /// Write a target to stdout + #[clap(aliases = ["cat", "show"])] + ShowTarget(show_target::Args), + /// Verify the integrity of a repository and check for issues + Verify(verify::Args), +} + +#[tokio::main] +async fn main() -> Result<()> { + let args = Command::parse(); + match args { + Command::Build(args) => args.run().await, + Command::Edit(args) => args.run().await, + Command::ListTargets(args) => args.run().await, + Command::ShowTarget(args) => args.run().await, + Command::Verify(args) => args.run().await, + } } diff --git a/bin/src/show_target.rs b/bin/src/show_target.rs new file mode 100644 index 0000000..b633198 --- /dev/null +++ b/bin/src/show_target.rs @@ -0,0 +1,43 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::io::Write; + +use anyhow::Context; +use anyhow::Result; +use camino::Utf8PathBuf; +use clap::Parser; +use futures_util::TryStreamExt; + +use crate::load::LoadOptions; + +#[derive(Debug, Parser)] +pub struct Args { + #[clap(flatten)] + load_options: LoadOptions, + /// Input repository path + repo: Utf8PathBuf, + /// Targets to output + /// + /// This is the path under "repo/targets/" in a ZIP archive. The + /// list-targets command lists the target names. + target_name: String, +} + +impl Args { + pub async fn run(self) -> Result<()> { + let repo = self + .load_options + .loader() + .await? + .load_zip_path(self.repo.clone(), &crate::LOG) + .await?; + let mut stream = repo.read_target(&self.target_name).await?; + let mut stdout = std::io::stdout().lock(); + while let Some(bytes) = stream.try_next().await? { + stdout.write_all(&bytes).context("failed to write to stdout")?; + } + Ok(()) + } +} diff --git a/bin/src/sign.rs b/bin/src/sign.rs new file mode 100644 index 0000000..91326bf --- /dev/null +++ b/bin/src/sign.rs @@ -0,0 +1,110 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use anyhow::Result; +use camino::Utf8PathBuf; +use clap::Args; +use tough::key_source::LocalKeySource; +use tufaceous::edit::SignedRepository; +use tufaceous::edit::UnsignedRepository; +use tufaceous::error::Error; +use tufaceous::error::ErrorKind; + +#[derive(Debug, Args)] +#[cfg_attr(test, derive(PartialEq))] +pub struct SignOptions { + /// Refuse to generate a signing root + #[clap(long)] + no_generate_root: bool, + + /// Path to the signing root [default: generate a root] + #[clap(long, required_if_eq("no_generate_root", "true"))] + signing_root: Option, + + /// Private signing key listed in the signing root + #[clap(long)] + key: Vec, +} + +impl SignOptions { + pub async fn sign( + self, + mut unsigned: UnsignedRepository<'_>, + ) -> Result> { + if !self.no_generate_root { + unsigned = unsigned.generate_root(); + } + if let Some(path) = self.signing_root { + let root = match tokio::fs::read(&path).await { + Ok(root) => root, + Err(source) => { + return Err(Error::from(ErrorKind::ReadFile { + source, + path: Some(path), + }) + .into()); + } + }; + unsigned = unsigned.root(root); + } + for path in self.key { + let source = LocalKeySource { path: path.into() }; + unsigned = unsigned.key(source); + } + Ok(unsigned.sign().await?) + } +} + +#[cfg(test)] +mod tests { + use clap::Parser; + + use crate::sign::SignOptions; + + #[derive(Parser)] + struct Args { + #[clap(flatten)] + inner: SignOptions, + } + + #[test] + fn test_flags() { + assert_eq!( + Args::try_parse_from([""]).unwrap().inner, + SignOptions { + no_generate_root: false, + signing_root: None, + key: vec![] + } + ); + assert!(Args::try_parse_from(["", "--no-generate-root"]).is_err()); + let mut args = vec!["", "--signing-root", "root.json"]; + assert_eq!( + Args::try_parse_from(&args).unwrap().inner, + SignOptions { + no_generate_root: false, + signing_root: Some("root.json".into()), + key: vec![], + } + ); + args.extend(["--key", "key.pem"]); + assert_eq!( + Args::try_parse_from(&args).unwrap().inner, + SignOptions { + no_generate_root: false, + signing_root: Some("root.json".into()), + key: vec!["key.pem".into()], + } + ); + args.extend(["--key", "key2.pem"]); + assert_eq!( + Args::try_parse_from(&args).unwrap().inner, + SignOptions { + no_generate_root: false, + signing_root: Some("root.json".into()), + key: vec!["key.pem".into(), "key2.pem".into()], + } + ); + } +} diff --git a/bin/src/verify.rs b/bin/src/verify.rs new file mode 100644 index 0000000..d3fdfb7 --- /dev/null +++ b/bin/src/verify.rs @@ -0,0 +1,78 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::error::Error; +use std::fmt::Display; + +use anyhow::Result; +use anyhow::ensure; +use camino::Utf8PathBuf; +use clap::Parser; +use tufaceous::CheckProblem; +use tufaceous_artifact::ArtifactHash; + +use crate::load::LoadOptions; + +#[derive(Debug, Parser)] +pub struct Args { + #[clap(flatten)] + load_options: LoadOptions, + /// Input repository path + repo: Utf8PathBuf, + /// Number of threads to use while verifying targets + #[clap(short = 'j', alias = "jobs", default_value_t = default_threads())] + threads: usize, +} + +fn default_threads() -> usize { + std::thread::available_parallelism() + .map(std::num::NonZero::get) + .unwrap_or(1) +} + +impl Args { + pub async fn run(self) -> Result<()> { + let repo = self + .load_options + .loader() + .await? + .compute_archive_sha256(true) + .v1_compatibility(true) + .load_zip_path(self.repo.clone(), &crate::LOG) + .await?; + let sha256 = ArtifactHash( + *repo.archive_sha256().expect("repo hash should be calculated"), + ); + + repo.verify_targets(self.threads).await?; + + let problems = repo.check_problems().await?; + ensure!( + problems.is_empty(), + "found compatibility problems:\n{}", + WriteProblems(&problems) + ); + + eprintln!("{}: OK, SHA256 = {sha256}", self.repo); + Ok(()) + } +} + +struct WriteProblems<'a>(&'a [CheckProblem]); + +impl Display for WriteProblems<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut nl = ""; + for problem in self.0 { + write!(f, "{nl}- {problem}")?; + let mut source = problem.source(); + while let Some(s) = source { + write!(f, ": {s}")?; + source = s.source(); + } + nl = "\n"; + } + Ok(()) + } +} diff --git a/bin/tests/integration-tests/command_tests.rs b/bin/tests/integration-tests/command_tests.rs deleted file mode 100644 index 5fac6e3..0000000 --- a/bin/tests/integration-tests/command_tests.rs +++ /dev/null @@ -1,199 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use anyhow::Result; -use assert_cmd::Command; -use dropshot::test_util::LogContext; -use dropshot::{ConfigLogging, ConfigLoggingIfExists, ConfigLoggingLevel}; -use predicates::prelude::*; -use tufaceous_lib::Key; - -#[test] -fn test_assemble_fake() -> Result<()> { - let log_config = ConfigLogging::File { - level: ConfigLoggingLevel::Trace, - path: "UNUSED".into(), - if_exists: ConfigLoggingIfExists::Fail, - }; - let logctx = LogContext::new("test_assemble_fake", &log_config); - let tempdir = tempfile::tempdir().unwrap(); - let key = Key::generate_ed25519()?; - - let archive_path = tempdir.path().join("archive.zip"); - - let mut cmd = make_cmd(&key); - cmd.args(["assemble", "manifests/fake.toml"]); - cmd.arg(&archive_path); - cmd.assert().success(); - - // Extract the archive to a new directory. - let dest_path = tempdir.path().join("dest"); - let mut cmd = make_cmd(&key); - cmd.arg("extract"); - cmd.arg(&archive_path); - cmd.arg(&dest_path); - - cmd.assert().success(); - - logctx.cleanup_successful(); - Ok(()) -} - -#[test] -fn test_assemble_fake_non_semver() -> Result<()> { - let log_config = ConfigLogging::File { - level: ConfigLoggingLevel::Trace, - path: "UNUSED".into(), - if_exists: ConfigLoggingIfExists::Fail, - }; - let logctx = LogContext::new("test_assemble_fake_non_semver", &log_config); - let tempdir = tempfile::tempdir().unwrap(); - let key = Key::generate_ed25519()?; - - let archive_path = tempdir.path().join("archive.zip"); - - let mut cmd = make_cmd(&key); - cmd.args(["assemble", "manifests/fake-non-semver.toml"]); - cmd.arg(&archive_path); - cmd.assert().failure().stderr(predicate::str::contains( - "non-semver versions found: fake-trampoline (non-semver), \ - fake-switch-rot-bootloader (non-semver-2)", - )); - - let mut cmd = make_cmd(&key); - cmd.args([ - "assemble", - "manifests/fake-non-semver.toml", - "--allow-non-semver", - ]); - cmd.arg(&archive_path); - cmd.assert().success(); - - // Extract the archive to a new directory. - let dest_path = tempdir.path().join("dest"); - let mut cmd = make_cmd(&key); - cmd.arg("extract"); - cmd.arg(&archive_path); - cmd.arg(&dest_path); - - cmd.assert().success(); - - logctx.cleanup_successful(); - Ok(()) -} - -#[test] -fn test_assemble_duplicate_zone() -> Result<()> { - let log_config = ConfigLogging::File { - level: ConfigLoggingLevel::Trace, - path: "UNUSED".into(), - if_exists: ConfigLoggingIfExists::Fail, - }; - let logctx = LogContext::new("test_assemble_duplicate_zone", &log_config); - let tempdir = tempfile::tempdir().unwrap(); - let key = Key::generate_ed25519()?; - - let archive_path = tempdir.path().join("archive.zip"); - - let mut cmd = make_cmd(&key); - cmd.args([ - "assemble", - "--skip-all-present", - // TODO: should we also check duplicate zone artifact names? - "invalid-manifests/duplicate-zone-file-name.toml", - ]); - cmd.arg(&archive_path); - cmd.assert() - .failure() - .stderr(predicate::str::contains( - r#"a deployment unit with the same kind and hash already exists in this control_plane artifact:"#, - )) - .stderr(predicate::str::contains("zone/")) - .stderr(predicate::str::contains( - r#"(existing name: zone1.tar.gz, version: 1.0.0; new name: zone1-dup.tar.gz, version: 1.0.0)"#, - )); - - logctx.cleanup_successful(); - Ok(()) -} - -#[test] -fn test_assemble_duplicate_artifact() -> Result<()> { - let log_config = ConfigLogging::File { - level: ConfigLoggingLevel::Trace, - path: "UNUSED".into(), - if_exists: ConfigLoggingIfExists::Fail, - }; - let logctx = - LogContext::new("test_assemble_duplicate_artifact", &log_config); - let tempdir = tempfile::tempdir().unwrap(); - let key = Key::generate_ed25519()?; - - let archive_path = tempdir.path().join("archive.zip"); - - let mut cmd = make_cmd(&key); - cmd.args([ - "assemble", - "--skip-all-present", - "invalid-manifests/duplicate-artifact.toml", - ]); - cmd.arg(&archive_path); - cmd.assert().failure().stderr(predicate::str::contains( - "a target named gimlet_sp-fake-gimlet-sp-1.0.0.tar.gz \ - already exists in the repository", - )); - - logctx.cleanup_successful(); - Ok(()) -} - -#[test] -fn test_assemble_duplicate_artifact_2() -> Result<()> { - let log_config = ConfigLogging::File { - level: ConfigLoggingLevel::Trace, - path: "UNUSED".into(), - if_exists: ConfigLoggingIfExists::Fail, - }; - let logctx = - LogContext::new("test_assemble_duplicate_artifact_2", &log_config); - let tempdir = tempfile::tempdir().unwrap(); - let key = Key::generate_ed25519()?; - - let archive_path = tempdir.path().join("archive.zip"); - - let mut cmd = make_cmd(&key); - cmd.args([ - "assemble", - "--skip-all-present", - "invalid-manifests/duplicate-artifact-2.toml", - ]); - cmd.arg(&archive_path); - cmd.assert() - .failure() - .stderr(predicate::str::contains("2 errors encountered")) - .stderr(predicate::str::contains( - "a deployment unit with the same kind and hash already exists \ - in this repository:", - )) - .stderr(predicate::str::contains("gimlet_sp/")) - .stderr(predicate::str::contains( - "(existing name: fake-gimlet-sp, version: 1.0.0; \ - new name: fake-gimlet-sp, version: 2.0.0)", - )) - .stderr(predicate::str::contains("switch_sp/")) - .stderr(predicate::str::contains( - "(existing name: fake-switch-sp, version: 1.0.0; \ - new name: fake-switch-sp, version: 2.0.0)", - )); - - logctx.cleanup_successful(); - Ok(()) -} - -fn make_cmd(key: &Key) -> Command { - let mut cmd = Command::cargo_bin("tufaceous").unwrap(); - cmd.env("TUFACEOUS_KEY", key.to_string()); - - cmd -} diff --git a/bin/tests/integration-tests/main.rs b/bin/tests/integration-tests/main.rs deleted file mode 100644 index fdf11b8..0000000 --- a/bin/tests/integration-tests/main.rs +++ /dev/null @@ -1,5 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -mod command_tests; diff --git a/bin/tests/manifest-tests.rs b/bin/tests/manifest-tests.rs deleted file mode 100644 index e099f31..0000000 --- a/bin/tests/manifest-tests.rs +++ /dev/null @@ -1,18 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use std::path::Path; - -use tufaceous_lib::assemble::ArtifactManifest; - -fn check_manifest(path: &Path) -> datatest_stable::Result<()> { - let path = path.try_into()?; - let manifest = - ArtifactManifest::from_path(path).expect("failed to load manifest"); - manifest.verify_all_present()?; - - Ok(()) -} - -datatest_stable::harness!(check_manifest, "manifests", r"^.*\.toml"); diff --git a/brand-metadata/Cargo.toml b/brand-metadata/Cargo.toml index 5bd92c0..5ec7cd2 100644 --- a/brand-metadata/Cargo.toml +++ b/brand-metadata/Cargo.toml @@ -6,7 +6,6 @@ license.workspace = true publish.workspace = true [dependencies] -semver.workspace = true serde.workspace = true serde_json.workspace = true tar.workspace = true diff --git a/brand-metadata/src/lib.rs b/brand-metadata/src/lib.rs index 21af0eb..0ae24f2 100644 --- a/brand-metadata/src/lib.rs +++ b/brand-metadata/src/lib.rs @@ -4,14 +4,19 @@ //! Handling of `oxide.json` metadata files in tarballs. //! -//! `oxide.json` is originally defined by the omicron1(7) zone brand, which -//! lives at . tufaceous -//! extended this format with additional archive types for identifying other -//! types of tarballs. - -use std::io::{Error, ErrorKind, Read, Result, Write}; - -use serde::{Deserialize, Serialize}; +//! `oxide.json` is defined by the omicron1(7) zone brand, which lives +//! at . tufaceous +//! previously extended this format with additional archive types for +//! identifying composite artifacts. + +use std::io::Error; +use std::io::ErrorKind; +use std::io::Read; +use std::io::Result; +use std::io::Write; + +use serde::Deserialize; +use serde::Serialize; use tufaceous_artifact::ArtifactVersion; #[derive(Clone, Debug, Deserialize, Serialize)] @@ -27,17 +32,13 @@ pub struct Metadata { t: ArchiveType, } +/// Archive types defined in helios-build-utils (part of helios-omicron-brand). #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "snake_case", tag = "t")] pub enum ArchiveType { - // Originally defined in helios-build-utils (part of helios-omicron-brand): Baseline, Layer(LayerInfo), Os, - - // tufaceous extensions: - Rot, - ControlPlane, } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -51,6 +52,7 @@ impl Metadata { Metadata { v: "1".into(), t: archive_type } } + #[expect(clippy::missing_panics_doc)] pub fn append_to_tar( &self, a: &mut tar::Builder, @@ -67,7 +69,7 @@ impl Metadata { h.set_gid(0); h.set_path("oxide.json")?; h.set_mode(0o444); - h.set_size(b.len().try_into().unwrap()); + h.set_size(b.len().try_into().expect("usize fits in u64")); h.set_mtime(mtime); h.set_cksum(); @@ -107,6 +109,16 @@ impl Metadata { } } + pub fn into_layer_info(self) -> Result { + match self.t { + ArchiveType::Layer(info) => Ok(info), + _ => Err(Error::new( + ErrorKind::InvalidData, + "archive is not the \"layer\" type", + )), + } + } + pub fn is_baseline(&self) -> bool { matches!(&self.t, ArchiveType::Baseline) } @@ -114,14 +126,6 @@ impl Metadata { pub fn is_os(&self) -> bool { matches!(&self.t, ArchiveType::Os) } - - pub fn is_rot(&self) -> bool { - matches!(&self.t, ArchiveType::Rot) - } - - pub fn is_control_plane(&self) -> bool { - matches!(&self.t, ArchiveType::ControlPlane) - } } #[cfg(test)] @@ -143,9 +147,5 @@ mod tests { r#"{"v":"1","t":"os","i":{"checksum":"42eda100ee0e3bf44b9d0bb6a836046fa3133c378cd9d3a4ba338c3ba9e56eb7","name":"ci 3a2ed5e/9d37813 2024-12-20 08:54"}}"#, ).unwrap(); assert!(metadata.is_os()); - - let metadata: Metadata = - serde_json::from_str(r#"{"v":"1","t":"control_plane"}"#).unwrap(); - assert!(metadata.is_control_plane()); } } diff --git a/clippy.toml b/clippy.toml new file mode 100644 index 0000000..d8d02c4 --- /dev/null +++ b/clippy.toml @@ -0,0 +1 @@ +doc-valid-idents = ["CoRIM", "OpenAPI", "RoT", ".."] diff --git a/lib/Cargo.toml b/lib/Cargo.toml index aa1c2d9..dc78a27 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -1,50 +1,44 @@ [package] -name = "tufaceous-lib" +name = "tufaceous" version.workspace = true edition.workspace = true license.workspace = true publish.workspace = true +rust-version = "1.94" [dependencies] -anyhow.workspace = true -async-trait.workspace = true +atomicwrites.workspace = true aws-lc-rs.workspace = true -base64.workspace = true -buf-list.workspace = true bytes.workspace = true camino.workspace = true camino-tempfile.workspace = true chrono.workspace = true +ciborium.workspace = true daft.workspace = true -debug-ignore.workspace = true +digest-io.workspace = true flate2.workspace = true -fs-err.workspace = true -futures.workspace = true +futures-util.workspace = true hex.workspace = true hubtools.workspace = true -indent_write.workspace = true -itertools.workspace = true -parse-size.workspace = true -rand.workspace = true +rats-corim.workspace = true +rawzip.workspace = true semver.workspace = true serde.workspace = true serde_json.workspace = true -serde_path_to_error.workspace = true sha2.workspace = true slog.workspace = true tar.workspace = true thiserror.workspace = true tokio.workspace = true -toml.workspace = true tough.workspace = true -tufaceous-artifact.workspace = true +tufaceous-artifact = { workspace = true, features = ["hubtools"] } tufaceous-brand-metadata.workspace = true url.workspace = true -zip.workspace = true [dev-dependencies] -dropshot.workspace = true -tokio = { workspace = true, features = ["macros", "test-util"] } +memchr.workspace = true +tokio = { workspace = true, features = ["macros"] } +tokio-test.workspace = true [lints] workspace = true diff --git a/lib/src/archive.rs b/lib/src/archive.rs deleted file mode 100644 index d209e26..0000000 --- a/lib/src/archive.rs +++ /dev/null @@ -1,297 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -//! Support for reading and writing zip archives. - -use std::fmt; -use std::io::{BufReader, BufWriter, Cursor, Read, Seek}; - -use anyhow::{Context, Result, anyhow, bail}; -use buf_list::BufList; -use bytes::Bytes; -use camino::{Utf8Component, Utf8Path, Utf8PathBuf}; -use debug_ignore::DebugIgnore; -use fs_err::File; -use zip::write::{FileOptions, SimpleFileOptions}; -use zip::{CompressionMethod, ZipArchive, ZipWriter}; - -/// A builder for TUF repo archives. -#[derive(Debug)] -pub(crate) struct ArchiveBuilder { - writer: DebugIgnore>>, - // Stored for better error messages. - output_path: Utf8PathBuf, -} - -/// Defines the base directory for TUF repo archives created by this tool. -/// -/// The usual convention is that the base dir is the name of the archive -/// (e.g. foo-1.0 for foo-1.0.zip). but just using a consistent name here -/// simplifies the code that extracts the archive. -pub const ZIP_BASE_DIR: &str = "repo"; - -impl ArchiveBuilder { - /// Creates a new `ArchiveBuilder`, writing to the given path. - pub fn new(output_path: Utf8PathBuf) -> Result { - // The filename must end with "zip". - if output_path.extension() != Some("zip") { - bail!("output path `{output_path}` must end with .zip"); - } - - let file = File::create(&output_path)?; - let writer = ZipWriter::new(BufWriter::new(file)); - Ok(Self { writer: writer.into(), output_path }) - } - - /// Writes the given path to the archive at the name `name`. - /// - /// The name has [`ZIP_BASE_DIR`] prepended to it. - pub fn write_file( - &mut self, - path: &Utf8Path, - name: &Utf8Path, - ) -> Result<()> { - let name = Utf8Path::new(ZIP_BASE_DIR).join(name); - - self.writer.start_file(name.as_str(), Self::file_options())?; - let mut reader = fs_err::File::open(path)?; - std::io::copy(&mut reader, &mut *self.writer).with_context(|| { - format!( - "error writing `{path}` to archive at `{}`", - self.output_path - ) - })?; - Ok(()) - } - - pub fn finish(self) -> Result<()> { - let Self { writer, output_path } = self; - - let zip_file = writer.0.finish().with_context(|| { - format!("error finalizing archive at `{}`", output_path) - })?; - zip_file.into_inner().with_context(|| { - format!("error writing archive at `{}`", output_path) - })?; - - Ok(()) - } - - fn file_options() -> SimpleFileOptions { - // The main purpose of the zip archive is to transmit archives that are - // already compressed, so there's no point trying to re-compress them. - FileOptions::default().compression_method(CompressionMethod::Stored) - } -} - -/// An extractor for archives created by tufaceous. -/// -/// Ideally we'd just be able to read the TUF repo out of a zip archive in -/// memory, but sadly that isn't possible today due to a missing lifetime -/// parameter on `Transport::fetch`. See [this -/// issue](https://github.com/awslabs/tough/pull/563). -#[derive(Debug)] -pub struct ArchiveExtractor { - archive: ZipArchive, -} - -impl ArchiveExtractor> { - /// Builds an extractor from the given path. - /// - /// The archive must be a zip file generated by tufaceous. - pub fn from_path(zip_path: &Utf8Path) -> Result { - let reader = BufReader::new(File::open(zip_path)?); - Self::new(reader).with_context(|| { - format!("error opening zip archive at `{zip_path}`") - }) - } -} - -impl<'a> ArchiveExtractor> { - /// Loads an archived repository from memory as borrowed bytes. - pub fn from_borrowed_bytes(archive: &'a [u8]) -> Result { - let reader = Cursor::new(archive); - Self::new(reader).context("error opening zip archive from memory") - } -} - -impl ArchiveExtractor> { - /// Loads an archived repository from memory as owned bytes. - pub fn from_owned_bytes(archive: impl Into) -> Result { - let reader = Cursor::new(archive.into()); - Self::new(reader).context("error opening zip archive from memory") - } -} - -impl<'a> ArchiveExtractor> { - /// Loads an archived repository from memory as a borrowed BufList. - pub fn from_borrowed_buf_list(archive: &'a BufList) -> Result { - let reader = buf_list::Cursor::new(archive); - Self::new(reader).context("error opening zip archive from memory") - } -} - -impl ArchiveExtractor> { - /// Loads an archived repository from memory as an owned BufList. - pub fn from_owned_buf_list(archive: BufList) -> Result { - let reader = buf_list::Cursor::new(archive); - Self::new(reader).context("error opening zip archive from memory") - } -} - -impl ArchiveExtractor -where - R: Read + Seek, -{ - /// Creates a new `ArchiveExtractor` from the given reader. - pub fn new(reader: R) -> Result> { - // Validate the archive to ensure all paths are correctly formed. - let archive = Self::validate(ZipArchive::new(reader)?)?; - - Ok(Self { archive }) - } - - fn validate(mut archive: ZipArchive) -> Result> { - for i in 0..archive.len() { - let zip_file = archive.by_index(i).with_context(|| { - format!("error reading file number `{i} from archive") - })?; - if !zip_file.is_file() { - bail!("archive must consist only of files, not directories"); - } - let path = Utf8Path::new(zip_file.name()); - validate_path(path).map_err(|error| { - anyhow!("invalid path in archive `{path}`: {error}") - })?; - } - - Ok(archive) - } - - /// Extracts this archive into the specified directory. - /// - /// Once this is completed, use - /// [`OmicronRepo::load_untrusted`](crate::OmicronRepo::load_untrusted) to - /// load the archive from `output_dir`. - /// - /// [`ZIP_BASE_DIR`] will be stripped from output paths. - pub fn extract(&mut self, output_dir: &Utf8Path) -> Result<()> { - for i in 0..self.archive.len() { - let mut zip_file = self.archive.by_index(i).with_context(|| { - format!("error reading file number `{i} from archive") - })?; - // SAFETY: file names have already been checked in `Self::validate`. - let file_name = Utf8Path::new(zip_file.name()).to_owned(); - let dest_path = output_dir.join( - file_name - .strip_prefix(ZIP_BASE_DIR) - .expect("checked in Self::validate"), - ); - // The file is in a directory. - fs_err::create_dir_all( - dest_path.parent().expect("at least 1 component"), - )?; - - let mut writer = File::create(&dest_path)?; - std::io::copy(&mut zip_file, &mut writer).with_context(|| { - format!( - "error writing `{file_name}` in archive to `{dest_path}`" - ) - })?; - } - - Ok(()) - } -} - -#[derive(Clone, Debug, Eq, PartialEq)] -enum InvalidPath<'a> { - AbsolutePath, - ExactlyBaseDir, - IncorrectBaseDir, - InvalidComponent(Utf8Component<'a>), -} - -impl fmt::Display for InvalidPath<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - InvalidPath::AbsolutePath => { - write!(f, "path is absolute -- expected relative paths") - } - InvalidPath::ExactlyBaseDir => { - write!( - f, - "path is exactly `{ZIP_BASE_DIR}` -- expected `{ZIP_BASE_DIR}/`" - ) - } - InvalidPath::IncorrectBaseDir => { - write!(f, "invalid base directory -- must be `{ZIP_BASE_DIR}`") - } - InvalidPath::InvalidComponent(component) => { - write!(f, "invalid component `{component}`") - } - } - } -} - -fn validate_path(path: &Utf8Path) -> Result<(), InvalidPath<'_>> { - if path.is_absolute() { - return Err(InvalidPath::AbsolutePath); - } - if path == ZIP_BASE_DIR { - return Err(InvalidPath::ExactlyBaseDir); - } - if !path.starts_with(ZIP_BASE_DIR) { - return Err(InvalidPath::IncorrectBaseDir); - } - - for component in path.components() { - if !matches!(component, Utf8Component::Normal(_)) { - return Err(InvalidPath::InvalidComponent(component)); - } - } - - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_validate_path() { - let valid = ["repo/foo", "repo/foo/bar", "repo/foo/./bar", "repo//foo"]; - let invalid = [ - ("repo", InvalidPath::ExactlyBaseDir), - ("repo/", InvalidPath::ExactlyBaseDir), - ("repo/.", InvalidPath::ExactlyBaseDir), - ("not-repo", InvalidPath::IncorrectBaseDir), - ("not-repo/foo", InvalidPath::IncorrectBaseDir), - ( - "repo/..", - InvalidPath::InvalidComponent(Utf8Component::ParentDir), - ), - ("/repo/foo", InvalidPath::AbsolutePath), - ]; - - for path in valid { - validate_path(Utf8Path::new(path)).unwrap_or_else(|err| { - panic!("expected path `{path}` to be valid: {err}") - }); - } - - for (path, expected) in invalid { - eprintln!("testing invalid path: `{path}`"); - let actual = match validate_path(Utf8Path::new(path)) { - Ok(()) => panic!("expected path `{path}` to be invalid"), - Err(error) => error, - }; - - assert_eq!( - actual, expected, - "for path `{path}`, InvalidPath error should match" - ); - } - } -} diff --git a/lib/src/artifact.rs b/lib/src/artifact.rs deleted file mode 100644 index 600b8a6..0000000 --- a/lib/src/artifact.rs +++ /dev/null @@ -1,638 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use std::io::{self, BufReader, Write}; -use std::path::Path; - -use anyhow::{Context, Result, bail}; -use buf_list::BufList; -use bytes::Bytes; -use camino::{Utf8Path, Utf8PathBuf}; -use fs_err::File; -use sha2::{Digest, Sha256}; -use tough::editor::RepositoryEditor; -use tufaceous_artifact::{ - ArtifactHash, ArtifactKind, ArtifactVersion, InstallinatorArtifact, - InstallinatorArtifactKind, KnownArtifactKind, -}; -use tufaceous_brand_metadata::Metadata; - -mod composite; - -pub use composite::CompositeControlPlaneArchiveBuilder; -pub use composite::CompositeEntry; -pub use composite::CompositeHostArchiveBuilder; -pub use composite::CompositeRotArchiveBuilder; -pub use composite::MtimeSource; - -use crate::assemble::ArtifactDeploymentUnits; -use crate::target::{TargetFinishWrite, TargetWriter}; - -/// The location a artifact will be obtained from. -#[derive(Clone, Debug)] -pub enum ArtifactSource { - File(Utf8PathBuf), - Memory(BufList), - // We might need to support downloading data over HTTP as well -} - -/// Describes a new artifact to be added. -pub struct AddArtifact { - kind: ArtifactKind, - name: String, - version: ArtifactVersion, - source: ArtifactSource, - deployment_units: ArtifactDeploymentUnits, -} - -impl AddArtifact { - /// Creates an [`AddArtifact`] from the provided source. - pub(crate) fn new( - kind: ArtifactKind, - name: String, - version: ArtifactVersion, - source: ArtifactSource, - deployment_units: ArtifactDeploymentUnits, - ) -> Self { - Self { kind, name, version, source, deployment_units } - } - - /// Returns the kind of artifact this is. - pub fn kind(&self) -> &ArtifactKind { - &self.kind - } - - /// Returns the name of the new artifact. - pub fn name(&self) -> &str { - &self.name - } - - /// Returns the version of the new artifact. - pub fn version(&self) -> &ArtifactVersion { - &self.version - } - - /// Returns the source for this artifact. - pub fn source(&self) -> &ArtifactSource { - &self.source - } - - /// Returns information about deployment units for this artifact. - pub fn deployment_units(&self) -> &ArtifactDeploymentUnits { - &self.deployment_units - } - - pub(crate) fn target_name(&self) -> String { - format!("{}-{}-{}.tar.gz", self.kind, self.name, self.version) - } - - /// Writes this artifact as a temporary file, returning a - /// [`TempWrittenArtifact`]. - pub(crate) fn write_temp( - &self, - targets_dir: &Utf8Path, - ) -> Result { - let target_name = self.target_name(); - let mut file = TargetWriter::new(targets_dir, &target_name)?; - self.write_to(&mut file).with_context(|| { - format!("error writing artifact `{target_name}") - })?; - let finished_file = file.finish_write(); - Ok(TempWrittenArtifact { - kind: self.kind.clone(), - name: self.name.clone(), - version: self.version.clone(), - deployment_units: self.deployment_units.clone(), - finished_file, - }) - } - - /// Writes this artifact to the specifid writer. - fn write_to(&self, writer: &mut W) -> Result<()> { - match &self.source { - ArtifactSource::File(path) => { - let mut reader = File::open(path)?; - std::io::copy(&mut reader, writer)?; - } - ArtifactSource::Memory(buf_list) => { - for chunk in buf_list { - writer.write_all(chunk)?; - } - } - } - - Ok(()) - } -} - -/// A newly-added artifact that's been written out to a temporary file. -#[must_use = "the artifact is still temporary and must be finalized"] -pub(crate) struct TempWrittenArtifact { - kind: ArtifactKind, - name: String, - version: ArtifactVersion, - deployment_units: ArtifactDeploymentUnits, - finished_file: TargetFinishWrite, -} - -impl TempWrittenArtifact { - pub(crate) fn name(&self) -> &str { - &self.name - } - - pub(crate) fn version(&self) -> &ArtifactVersion { - &self.version - } - - pub(crate) fn kind(&self) -> &ArtifactKind { - &self.kind - } - - pub(crate) fn digest(&self) -> ArtifactHash { - self.finished_file.digest() - } - - pub(crate) fn deployment_units(&self) -> &ArtifactDeploymentUnits { - &self.deployment_units - } - - /// Returns information about installinator artifacts for this newly-added - /// artifact. - pub(crate) fn installinator_artifacts( - &self, - ) -> impl Iterator + '_ { - let known = self.kind.to_known(); - - // Currently, a `TempWrittenArtifact` corresponds to zero or one - // installinator artifacts so we can just return an Option. If, in the - // future, a single `TempWrittenArtifact` corresponds to multiple - // installinator artifacts, we'd have to return a more complex iterator. - let artifact = match known { - Some(KnownArtifactKind::Host) => { - // The host phase 2 artifact is an installinator artifact. - let host_phase_2 = match &self.deployment_units { - ArtifactDeploymentUnits::SingleUnit - | ArtifactDeploymentUnits::Unknown => { - panic!( - "expected Host artifact to be Composite, found {:?}", - self.deployment_units - ); - } - ArtifactDeploymentUnits::Composite { deployment_units } => { - deployment_units - .values() - .find(|unit| { - unit.kind == ArtifactKind::HOST_PHASE_2 - }) - .unwrap_or_else(|| { - panic!( - "Host artifact must have a host phase 2 \ - deployment unit, found {:?}", - deployment_units, - ) - }) - } - }; - Some(InstallinatorArtifact { - name: host_phase_2.name.clone(), - kind: InstallinatorArtifactKind::HostPhase2, - hash: host_phase_2.hash, - }) - } - Some(KnownArtifactKind::ControlPlane) => { - Some(InstallinatorArtifact { - name: self.name.clone(), - kind: InstallinatorArtifactKind::ControlPlane, - hash: self.digest(), - }) - } - Some(KnownArtifactKind::MeasurementCorpus) => { - Some(InstallinatorArtifact { - name: self.name.clone(), - kind: InstallinatorArtifactKind::MeasurementCorpus, - hash: self.digest(), - }) - } - Some(_) | None => None, - }; - artifact.into_iter() - } - - pub(crate) fn finalize( - self, - editor: &mut RepositoryEditor, - ) -> Result { - self.finished_file.finalize(editor) - } -} - -pub(crate) fn make_filler_text_with_seed( - // composite artifact. - kind: &str, - version: &ArtifactVersion, - length: usize, - seed: &str, -) -> Vec { - // Add the kind and version to the filler text first. This ensures that - // hashes are unique by kind and version. - let mut out = Vec::with_capacity(length); - out.extend_from_slice(kind.as_bytes()); - out.extend_from_slice(b":"); - out.extend_from_slice(version.as_str().as_bytes()); - out.extend_from_slice(b":"); - out.extend_from_slice(seed.as_bytes()); - out.extend_from_slice(b":"); - let remaining = length.saturating_sub(out.len()); - out.extend( - std::iter::repeat(FILLER_TEXT).flatten().copied().take(remaining), - ); - - out -} - -pub(crate) fn make_filler_text( - // composite artifact. - kind: &str, - version: &ArtifactVersion, - length: usize, -) -> Vec { - // Add the kind and version to the filler text first. This ensures that - // hashes are unique by kind and version. - let mut out = Vec::with_capacity(length); - out.extend_from_slice(kind.as_bytes()); - out.extend_from_slice(b":"); - out.extend_from_slice(version.as_str().as_bytes()); - out.extend_from_slice(b":"); - - let remaining = length.saturating_sub(out.len()); - out.extend( - std::iter::repeat(FILLER_TEXT).flatten().copied().take(remaining), - ); - - out -} - -/// Represents host phase images. -/// -/// The host and trampoline artifacts are actually tarballs, with phase 1 and -/// phase 2 images inside them. This code extracts those images out of the -/// tarballs. -#[derive(Clone, Debug)] -pub struct HostPhaseImages { - pub gimlet_phase_1: Bytes, - pub cosmo_phase_1: Bytes, - pub phase_2: Bytes, -} - -/// File sources for extraction -/// -/// Passing three identical arguments gets confusing and error prone -pub struct HostPhaseImageSource { - pub gimlet_phase_1: W, - pub cosmo_phase_1: W, - pub phase_2: W, -} - -impl HostPhaseImages { - pub fn extract(reader: R) -> Result { - let mut gimlet_phase_1 = Vec::new(); - let mut cosmo_phase_1 = Vec::new(); - let mut phase_2 = Vec::new(); - let source = HostPhaseImageSource { - gimlet_phase_1: io::Cursor::<&mut Vec>::new( - &mut gimlet_phase_1, - ), - cosmo_phase_1: io::Cursor::<&mut Vec>::new(&mut cosmo_phase_1), - phase_2: io::Cursor::<&mut Vec>::new(&mut phase_2), - }; - - Self::extract_into(reader, source)?; - Ok(Self { - gimlet_phase_1: gimlet_phase_1.into(), - cosmo_phase_1: cosmo_phase_1.into(), - phase_2: phase_2.into(), - }) - } - - pub fn extract_into( - reader: R, - source: HostPhaseImageSource, - ) -> Result<()> { - let uncompressed = flate2::bufread::GzDecoder::new(reader); - let mut archive = tar::Archive::new(uncompressed); - - let mut oxide_json_found = false; - let mut gimlet_phase_1_writer = Some(source.gimlet_phase_1); - let mut cosmo_phase_1_writer = Some(source.cosmo_phase_1); - let mut phase_2_writer = Some(source.phase_2); - for entry in archive - .entries() - .context("error building list of entries from archive")? - { - let entry = entry.context("error reading entry from archive")?; - let path = entry - .header() - .path() - .context("error reading path from archive")?; - if path == Path::new(OXIDE_JSON_FILE_NAME) { - let json_bytes = read_entry(entry, OXIDE_JSON_FILE_NAME)?; - let metadata: Metadata = - serde_json::from_slice(&json_bytes).with_context(|| { - format!( - "error deserializing JSON from {OXIDE_JSON_FILE_NAME}" - ) - })?; - if !metadata.is_os() { - bail!( - "unexpected archive type: expected os, found {:?}", - metadata.archive_type(), - ) - } - oxide_json_found = true; - } else if path == Path::new(COSMO_HOST_PHASE_1_FILE_NAME) { - if let Some(cosmo_phase_1) = cosmo_phase_1_writer.take() { - read_entry_into( - entry, - COSMO_HOST_PHASE_1_FILE_NAME, - cosmo_phase_1, - )?; - } - } else if path == Path::new(GIMLET_HOST_PHASE_1_FILE_NAME) { - if let Some(gimlet_phase_1) = gimlet_phase_1_writer.take() { - read_entry_into( - entry, - GIMLET_HOST_PHASE_1_FILE_NAME, - gimlet_phase_1, - )?; - } - } else if path == Path::new(HOST_PHASE_2_FILE_NAME) { - if let Some(phase_2) = phase_2_writer.take() { - read_entry_into(entry, HOST_PHASE_2_FILE_NAME, phase_2)?; - } - } - - if oxide_json_found - && gimlet_phase_1_writer.is_none() - && cosmo_phase_1_writer.is_none() - && phase_2_writer.is_none() - { - break; - } - } - - let mut not_found = Vec::new(); - if !oxide_json_found { - not_found.push(OXIDE_JSON_FILE_NAME); - } - - // If we didn't `.take()` the writer out of the options, we never saw - // the expected phase1/phase2 filenames. - if cosmo_phase_1_writer.is_some() { - not_found.push(COSMO_HOST_PHASE_1_FILE_NAME); - } - if gimlet_phase_1_writer.is_some() { - not_found.push(GIMLET_HOST_PHASE_1_FILE_NAME); - } - if phase_2_writer.is_some() { - not_found.push(HOST_PHASE_2_FILE_NAME); - } - - if !not_found.is_empty() { - bail!("required files not found: {}", not_found.join(", ")) - } - - Ok(()) - } - - pub fn gimlet_phase_1_hash(&self) -> ArtifactHash { - let hash = Sha256::digest(&self.gimlet_phase_1); - ArtifactHash(hash.into()) - } - - pub fn cosmo_phase_1_hash(&self) -> ArtifactHash { - let hash = Sha256::digest(&self.cosmo_phase_1); - ArtifactHash(hash.into()) - } - - pub fn phase_2_hash(&self) -> ArtifactHash { - let hash = Sha256::digest(&self.phase_2); - ArtifactHash(hash.into()) - } -} - -fn read_entry( - entry: tar::Entry, - file_name: &str, -) -> Result { - let mut buf = Vec::new(); - read_entry_into(entry, file_name, io::Cursor::new(&mut buf))?; - Ok(buf.into()) -} - -fn read_entry_into( - mut entry: tar::Entry, - file_name: &str, - mut out: W, -) -> Result<()> { - let entry_type = entry.header().entry_type(); - if entry_type != tar::EntryType::Regular { - bail!("for {file_name}, expected regular file, found {entry_type:?}"); - } - io::copy(&mut entry, &mut out) - .with_context(|| format!("error reading {file_name} from archive"))?; - Ok(()) -} - -/// Represents RoT A/B hubris archives. -/// -/// RoT artifacts are actually tarballs, with both A and B hubris archives -/// inside them. This code extracts those archives out of the tarballs. -#[derive(Clone, Debug)] -pub struct RotArchives { - pub archive_a: Bytes, - pub archive_b: Bytes, -} - -impl RotArchives { - pub fn extract(reader: R) -> Result { - let mut archive_a = Vec::new(); - let mut archive_b = Vec::new(); - Self::extract_into( - reader, - io::Cursor::<&mut Vec>::new(&mut archive_a), - io::Cursor::<&mut Vec>::new(&mut archive_b), - )?; - Ok(Self { archive_a: archive_a.into(), archive_b: archive_b.into() }) - } - - pub fn extract_into( - reader: R, - archive_a: W, - archive_b: W, - ) -> Result<()> { - let uncompressed = flate2::bufread::GzDecoder::new(reader); - let mut archive = tar::Archive::new(uncompressed); - - let mut oxide_json_found = false; - let mut archive_a_writer = Some(archive_a); - let mut archive_b_writer = Some(archive_b); - for entry in archive - .entries() - .context("error building list of entries from archive")? - { - let entry = entry.context("error reading entry from archive")?; - let path = entry - .header() - .path() - .context("error reading path from archive")?; - if path == Path::new(OXIDE_JSON_FILE_NAME) { - let json_bytes = read_entry(entry, OXIDE_JSON_FILE_NAME)?; - let metadata: Metadata = - serde_json::from_slice(&json_bytes).with_context(|| { - format!( - "error deserializing JSON from {OXIDE_JSON_FILE_NAME}" - ) - })?; - if !metadata.is_rot() { - bail!( - "unexpected archive type: expected rot, found {:?}", - metadata.archive_type(), - ) - } - oxide_json_found = true; - } else if path == Path::new(ROT_ARCHIVE_A_FILE_NAME) { - if let Some(archive_a) = archive_a_writer.take() { - read_entry_into(entry, ROT_ARCHIVE_A_FILE_NAME, archive_a)?; - } - } else if path == Path::new(ROT_ARCHIVE_B_FILE_NAME) { - if let Some(archive_b) = archive_b_writer.take() { - read_entry_into(entry, ROT_ARCHIVE_B_FILE_NAME, archive_b)?; - } - } - - if oxide_json_found - && archive_a_writer.is_none() - && archive_b_writer.is_none() - { - break; - } - } - - let mut not_found = Vec::new(); - if !oxide_json_found { - not_found.push(OXIDE_JSON_FILE_NAME); - } - - // If we didn't `.take()` the writer out of the options, we never saw - // the expected A/B filenames. - if archive_a_writer.is_some() { - not_found.push(ROT_ARCHIVE_A_FILE_NAME); - } - if archive_b_writer.is_some() { - not_found.push(ROT_ARCHIVE_B_FILE_NAME); - } - - if !not_found.is_empty() { - bail!("required files not found: {}", not_found.join(", ")) - } - - Ok(()) - } -} - -/// Represents control plane zone images. -/// -/// The control plane artifact is actually a tarball that contains a set of zone -/// images. This code extracts those images out of the tarball. -#[derive(Clone, Debug)] -pub struct ControlPlaneZoneImages { - pub zones: Vec<(String, Bytes)>, -} - -impl ControlPlaneZoneImages { - pub fn extract(reader: R) -> Result { - let mut zones = Vec::new(); - Self::extract_into(reader, |name, reader| { - let mut buf = Vec::new(); - io::copy(reader, &mut buf)?; - zones.push((name, buf.into())); - Ok(()) - })?; - Ok(Self { zones }) - } - - pub fn extract_into(reader: R, mut handler: F) -> Result<()> - where - R: io::Read, - F: FnMut(String, &mut dyn io::Read) -> Result<()>, - { - let uncompressed = - flate2::bufread::GzDecoder::new(BufReader::new(reader)); - let mut archive = tar::Archive::new(uncompressed); - - let mut oxide_json_found = false; - let mut zone_found = false; - for entry in archive - .entries() - .context("error building list of entries from archive")? - { - let mut entry = - entry.context("error reading entry from archive")?; - let path = entry - .header() - .path() - .context("error reading path from archive")?; - if path == Path::new(OXIDE_JSON_FILE_NAME) { - let json_bytes = read_entry(entry, OXIDE_JSON_FILE_NAME)?; - let metadata: Metadata = - serde_json::from_slice(&json_bytes).with_context(|| { - format!( - "error deserializing JSON from {OXIDE_JSON_FILE_NAME}" - ) - })?; - if !metadata.is_control_plane() { - bail!( - "unexpected archive type: expected control_plane, found {:?}", - metadata.archive_type(), - ) - } - oxide_json_found = true; - } else if path.starts_with(CONTROL_PLANE_ARCHIVE_ZONE_DIRECTORY) { - if let Some(name) = path - .file_name() - .and_then(|s| s.to_str()) - .map(|s| s.to_string()) - { - handler(name, &mut entry)?; - } - zone_found = true; - } - } - - let mut not_found = Vec::new(); - if !oxide_json_found { - not_found.push(OXIDE_JSON_FILE_NAME); - } - if !not_found.is_empty() { - bail!("required files not found: {}", not_found.join(", ")) - } - if !zone_found { - bail!( - "no zone images found in `{}/`", - CONTROL_PLANE_ARCHIVE_ZONE_DIRECTORY - ); - } - - Ok(()) - } -} - -static FILLER_TEXT: &[u8; 16] = b"tufaceousfaketxt"; -static OXIDE_JSON_FILE_NAME: &str = "oxide.json"; -pub(crate) static GIMLET_HOST_PHASE_1_FILE_NAME: &str = "image/gimlet.rom"; -pub(crate) static COSMO_HOST_PHASE_1_FILE_NAME: &str = "image/cosmo.rom"; -pub(crate) static HOST_PHASE_2_FILE_NAME: &str = "image/zfs.img"; -pub(crate) static ROT_ARCHIVE_A_FILE_NAME: &str = "archive-a.zip"; -pub(crate) static ROT_ARCHIVE_B_FILE_NAME: &str = "archive-b.zip"; -static CONTROL_PLANE_ARCHIVE_ZONE_DIRECTORY: &str = "zones"; diff --git a/lib/src/artifact/composite.rs b/lib/src/artifact/composite.rs deleted file mode 100644 index 3a7eab6..0000000 --- a/lib/src/artifact/composite.rs +++ /dev/null @@ -1,220 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use std::io::{BufWriter, Write}; - -use anyhow::{Context, Result, anyhow, bail}; -use camino::Utf8Path; -use flate2::Compression; -use flate2::write::GzEncoder; -use sha2::{Digest, Sha256}; -use tufaceous_artifact::ArtifactHash; -use tufaceous_brand_metadata::{ArchiveType, Metadata}; - -use super::{ - CONTROL_PLANE_ARCHIVE_ZONE_DIRECTORY, COSMO_HOST_PHASE_1_FILE_NAME, - GIMLET_HOST_PHASE_1_FILE_NAME, HOST_PHASE_2_FILE_NAME, - ROT_ARCHIVE_A_FILE_NAME, ROT_ARCHIVE_B_FILE_NAME, -}; - -/// Represents a single entry in a composite artifact. -/// -/// A composite artifact is a tarball containing multiple artifacts. This -/// struct is intended for the insertion of one such entry into the artifact. -/// -/// At the moment it only accepts byte slices, but it could be extended to -/// support arbitrary readers in the future. -pub struct CompositeEntry<'a> { - pub data: &'a [u8], - pub mtime_source: MtimeSource, -} - -pub struct CompositeControlPlaneArchiveBuilder { - inner: CompositeTarballBuilder, -} - -impl CompositeControlPlaneArchiveBuilder { - pub fn new(writer: W, mtime_source: MtimeSource) -> Result { - let metadata = Metadata::new(ArchiveType::ControlPlane); - let inner = - CompositeTarballBuilder::new(writer, metadata, mtime_source)?; - Ok(Self { inner }) - } - - pub fn append_zone( - &mut self, - name: &str, - entry: CompositeEntry<'_>, - ) -> Result<(ArtifactHash, String)> { - let name_path = Utf8Path::new(name); - if name_path.file_name() != Some(name) { - bail!("control plane zone filenames should not contain paths"); - } - let path = - Utf8Path::new(CONTROL_PLANE_ARCHIVE_ZONE_DIRECTORY).join(name_path); - let hash = self.inner.append_file(path.as_str(), entry)?; - Ok((hash, name.to_owned())) - } - - pub fn finish(self) -> Result { - self.inner.finish() - } -} - -pub struct CompositeRotArchiveBuilder { - inner: CompositeTarballBuilder, -} - -impl CompositeRotArchiveBuilder { - pub fn new(writer: W, mtime_source: MtimeSource) -> Result { - let metadata = Metadata::new(ArchiveType::Rot); - let inner = - CompositeTarballBuilder::new(writer, metadata, mtime_source)?; - Ok(Self { inner }) - } - - pub fn append_archive_a( - &mut self, - entry: CompositeEntry<'_>, - ) -> Result { - self.inner.append_file(ROT_ARCHIVE_A_FILE_NAME, entry) - } - - pub fn append_archive_b( - &mut self, - entry: CompositeEntry<'_>, - ) -> Result { - self.inner.append_file(ROT_ARCHIVE_B_FILE_NAME, entry) - } - - pub fn finish(self) -> Result { - self.inner.finish() - } -} - -pub struct CompositeHostArchiveBuilder { - inner: CompositeTarballBuilder, -} - -impl CompositeHostArchiveBuilder { - pub fn new(writer: W, mtime_source: MtimeSource) -> Result { - let metadata = Metadata::new(ArchiveType::Os); - let inner = - CompositeTarballBuilder::new(writer, metadata, mtime_source)?; - Ok(Self { inner }) - } - - pub fn append_gimlet_phase_1( - &mut self, - entry: CompositeEntry<'_>, - ) -> Result { - self.inner.append_file(GIMLET_HOST_PHASE_1_FILE_NAME, entry) - } - - pub fn append_cosmo_phase_1( - &mut self, - entry: CompositeEntry<'_>, - ) -> Result { - self.inner.append_file(COSMO_HOST_PHASE_1_FILE_NAME, entry) - } - - pub fn append_phase_2( - &mut self, - entry: CompositeEntry<'_>, - ) -> Result { - self.inner.append_file(HOST_PHASE_2_FILE_NAME, entry) - } - - pub fn finish(self) -> Result { - self.inner.finish() - } -} - -struct CompositeTarballBuilder { - builder: tar::Builder>>, -} - -impl CompositeTarballBuilder { - fn new( - writer: W, - metadata: Metadata, - mtime_source: MtimeSource, - ) -> Result { - let mut builder = tar::Builder::new(GzEncoder::new( - BufWriter::new(writer), - Compression::fast(), - )); - metadata.append_to_tar(&mut builder, mtime_source.into_mtime())?; - Ok(Self { builder }) - } - - fn append_file( - &mut self, - path: &str, - entry: CompositeEntry<'_>, - ) -> Result { - let header = - make_tar_header(path, entry.data.len(), entry.mtime_source); - self.builder - .append(&header, entry.data) - .with_context(|| format!("error append {path:?}"))?; - let hash = Sha256::digest(entry.data); - Ok(ArtifactHash(hash.into())) - } - - fn finish(self) -> Result { - let gz_encoder = - self.builder.into_inner().context("error finalizing archive")?; - let buf_writer = - gz_encoder.finish().context("error finishing gz encoder")?; - buf_writer - .into_inner() - .map_err(|_| anyhow!("error flushing buffered archive writer")) - } -} - -fn make_tar_header( - path: &str, - size: usize, - mtime_source: MtimeSource, -) -> tar::Header { - let mtime = mtime_source.into_mtime(); - - let mut header = tar::Header::new_ustar(); - header.set_username("root").unwrap(); - header.set_uid(0); - header.set_groupname("root").unwrap(); - header.set_gid(0); - header.set_path(path).unwrap(); - header.set_size(size as u64); - header.set_mode(0o444); - header.set_entry_type(tar::EntryType::Regular); - header.set_mtime(mtime); - header.set_cksum(); - - header -} - -/// How to obtain the `mtime` field for a tar header. -#[derive(Copy, Clone, Debug)] -pub enum MtimeSource { - /// Use a fixed timestamp of zero seconds past the Unix epoch. - Zero, - - /// Use the current time. - Now, -} - -impl MtimeSource { - pub(crate) fn into_mtime(self) -> u64 { - use std::time::{SystemTime, UNIX_EPOCH}; - - match self { - Self::Zero => 0, - Self::Now => { - SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() - } - } - } -} diff --git a/lib/src/assemble/build.rs b/lib/src/assemble/build.rs deleted file mode 100644 index 947c497..0000000 --- a/lib/src/assemble/build.rs +++ /dev/null @@ -1,167 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use anyhow::{Context, Result}; -use camino::{Utf8Path, Utf8PathBuf}; -use chrono::{DateTime, Utc}; -use tough::editor::signed::SignedRole; -use tough::schema::Root; - -use crate::{ - AddArtifact, Key, OmicronRepo, OmicronRepoEditor, utils::merge_anyhow_list, -}; - -use super::ArtifactManifest; - -/// Assembles a TUF repo from a list of artifacts. -#[derive(Debug)] -pub struct OmicronRepoAssembler { - log: slog::Logger, - manifest: ArtifactManifest, - build_dir: Option, - keys: Vec, - root: Option>, - expiry: DateTime, - include_installinator_doc: bool, - output_path: Utf8PathBuf, -} - -impl OmicronRepoAssembler { - pub fn new( - log: &slog::Logger, - manifest: ArtifactManifest, - keys: Vec, - expiry: DateTime, - include_installinator_doc: bool, - output_path: Utf8PathBuf, - ) -> Self { - Self { - log: log.new(slog::o!("component" => "OmicronRepoAssembler")), - manifest, - build_dir: None, - keys, - root: None, - expiry, - include_installinator_doc, - output_path, - } - } - - pub fn set_build_dir(&mut self, build_dir: Utf8PathBuf) -> &mut Self { - self.build_dir = Some(build_dir); - self - } - - pub fn set_root_role(&mut self, root_role: SignedRole) -> &mut Self { - self.root = Some(root_role); - self - } - - pub async fn build(&self) -> Result<()> { - let (build_dir, is_temp) = match &self.build_dir { - Some(dir) => (dir.clone(), false), - None => { - // Create a new temporary directory. - let dir = camino_tempfile::Builder::new() - .prefix("tufaceous") - .tempdir()?; - // This will cause the directory to be preserved -- we're going - // to clean it up if it's successful. - let path = dir.keep(); - (path, true) - } - }; - - slog::info!(self.log, "assembling repository in `{build_dir}`"); - - match self.build_impl(&build_dir).await { - Ok(()) => { - if is_temp { - slog::debug!(self.log, "assembly successful, cleaning up"); - // Log, but otherwise ignore, errors while cleaning up. - if let Err(error) = fs_err::remove_dir_all(&build_dir) { - slog::warn!( - self.log, - "failed to clean up temporary directory {build_dir}: {error}" - ) - } - } - - slog::info!( - self.log, - "artifacts assembled and archived to `{}`", - self.output_path - ); - Ok(()) - } - Err(error) => { - slog::error!( - self.log, - "assembly failed, failing build directory preserved: `{build_dir}`" - ); - Err(error) - } - } - } - - async fn build_impl(&self, build_dir: &Utf8Path) -> Result<()> { - let root = match &self.root { - Some(root) => root.clone(), - None => { - crate::root::new_root(self.keys.clone(), self.expiry).await? - } - }; - let mut repository = OmicronRepoEditor::initialize( - build_dir.to_owned(), - root, - self.manifest.system_version.clone(), - ) - .await?; - - // Add all the artifacts. - let mut errors = Vec::new(); - - for (kind, entries) in &self.manifest.artifacts { - for data in entries { - let new_artifact = AddArtifact::new( - (*kind).into(), - data.name.clone(), - data.version.clone(), - data.source.clone(), - data.deployment_units.clone(), - ); - let res = - repository.add_artifact(&new_artifact).with_context(|| { - format!("error adding artifact with kind `{kind}`") - }); - if let Err(err) = res { - errors.push(err); - } - } - } - - if !errors.is_empty() { - return Err(merge_anyhow_list(errors)); - } - - // Write out the repository. - repository - .sign_and_finish( - self.keys.clone(), - self.expiry, - self.include_installinator_doc, - ) - .await?; - - // Now reopen the repository to archive it into a zip file. - let repo2 = OmicronRepo::load_untrusted(&self.log, build_dir) - .await - .context("error reopening repository to archive")?; - repo2 - .archive(&self.output_path) - .context("error archiving repository")?; - - Ok(()) - } -} diff --git a/lib/src/assemble/deployment_unit.rs b/lib/src/assemble/deployment_unit.rs deleted file mode 100644 index 698d397..0000000 --- a/lib/src/assemble/deployment_unit.rs +++ /dev/null @@ -1,262 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use std::{collections::BTreeMap, fmt}; - -use daft::{Diffable, Leaf}; -use thiserror::Error; -use tufaceous_artifact::{ - ArtifactHash, ArtifactHashId, ArtifactKind, ArtifactVersion, - KnownArtifactKind, -}; - -/// Information about deployment units keyed by hash ID. -/// -/// This information is used to ensure uniqueness of deployment units within a -/// particular scope. -pub type DeploymentUnitMap = BTreeMap; - -/// Information about deployment units for an artifact. -#[derive(Clone, Debug)] -pub enum ArtifactDeploymentUnits { - /// This artifact is single-unit (not composite). There is exactly one - /// deployment unit, which is the artifact itself. - SingleUnit, - - /// This is a composite artifact, with these deployment units. - Composite { deployment_units: DeploymentUnitMap }, - - /// Data was not available for this artifact. - Unknown, -} - -/// Ths scope in which deployment unit data is being gathered. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum DeploymentUnitScope { - /// Deployment units are being gathered for a single composite artifact. - Artifact { - /// The kind of the composite artifact. - composite_kind: KnownArtifactKind, - }, - - /// Deployment units are being gathered for a repository. - Repository, -} - -impl fmt::Display for DeploymentUnitScope { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - DeploymentUnitScope::Artifact { composite_kind } => { - write!(f, "{composite_kind} artifact") - } - DeploymentUnitScope::Repository => "repository".fmt(f), - } - } -} - -/// Data associated with a deployment unit. -#[derive(Clone, Debug)] -pub struct DeploymentUnitData { - /// The name of the deployment unit. - pub name: String, - - /// The version of the deployment unit. - pub version: ArtifactVersion, - - /// The kind of the deployment unit. - pub kind: ArtifactKind, - - /// The hash of the deployment unit. - pub hash: ArtifactHash, -} - -impl DeploymentUnitData { - /// Returns the [`ArtifactHashId`] of the deployment unit. - #[inline] - pub fn hash_id(&self) -> ArtifactHashId { - ArtifactHashId { kind: self.kind.clone(), hash: self.hash } - } -} - -impl fmt::Display for DeploymentUnitData { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "name: {}, version: {}", self.name, self.version) - } -} - -/// Builder for a [`DeploymentUnitMap`]. -/// -/// This builder ensures uniqueness of deployment units within a particular -/// [`DeploymentUnitScope`]. -#[derive(Clone, Debug)] -pub struct DeploymentUnitMapBuilder { - scope: DeploymentUnitScope, - // TODO: This should also check for duplicate name/version/kind. Will - // probably want to use a data structure more tailored for this. - deployment_units: DeploymentUnitMap, -} - -impl DeploymentUnitMapBuilder { - pub fn new(scope: DeploymentUnitScope) -> Self { - Self { scope, deployment_units: BTreeMap::new() } - } - - /// Starts adding a new deployment unit to `self`. - pub fn start_insert( - &mut self, - data: DeploymentUnitData, - ) -> Result, DuplicateDeploymentUnitError> { - let hash_id = data.hash_id(); - if let Some(existing) = self.deployment_units.get(&hash_id) { - return Err(DuplicateDeploymentUnitError::new_single( - self.scope, - existing.clone(), - data, - )); - }; - - Ok(NewDeploymentUnits::new_single(&mut self.deployment_units, data)) - } - - /// Starts a bulk insert of another deployment unit map into `self`. The - /// merge is not committed until `NewDeploymentUnits::commit` is called. - /// - /// Returns an error if any duplicates are found. - pub fn start_bulk_insert( - &mut self, - units: DeploymentUnitMap, - ) -> Result, DuplicateDeploymentUnitError> { - // Check that there are no duplicates. We don't expect to see any - // duplicated artifacts at all within a single artifact or repository, - // so we don't check whether the hashes are the same. - // - // In order for this check to be done now rather than at commit time, we - // rely on two things: - // - // 1. `start_bulk_insert` accepts a `&mut self` parameter. - // 2. There is no interior mutability in `DeploymentUnitMapBuilder`. - // - // Together, these two checks ensure that nothing else can modify the - // map between now and either commit or discard. - let diff = self.deployment_units.diff(&units); - - if !diff.common.is_empty() { - return Err(DuplicateDeploymentUnitError { - scope: self.scope, - duplicates: diff - .common - .into_iter() - .map(|(k, v)| (k.clone(), v.cloned())) - .collect(), - }); - } - - Ok(NewDeploymentUnits { base: &mut self.deployment_units, units }) - } - - /// Convenience method for `self.start_add_deployment_unit(..).insert()` for a - /// single deployment unit. - pub fn insert( - &mut self, - data: DeploymentUnitData, - ) -> Result<(), DuplicateDeploymentUnitError> { - self.start_insert(data)?.commit(); - Ok(()) - } - - pub fn finish_map(self) -> DeploymentUnitMap { - self.deployment_units - } - - pub fn finish_units(self) -> ArtifactDeploymentUnits { - ArtifactDeploymentUnits::Composite { - deployment_units: self.deployment_units, - } - } -} - -/// Information about new deployment units for a [`DeploymentUnitMapBuilder`]. -/// -/// This serves as a way to make new deployment units ready to be bulk-inserted -/// into the builder. -#[must_use = "NewDeploymentUnits must be committed into the builder"] -pub struct NewDeploymentUnits<'a> { - base: &'a mut DeploymentUnitMap, - units: DeploymentUnitMap, -} - -impl<'a> NewDeploymentUnits<'a> { - fn new_single( - base: &'a mut DeploymentUnitMap, - data: DeploymentUnitData, - ) -> Self { - let mut units = DeploymentUnitMap::new(); - units.insert(data.hash_id(), data); - Self { base, units } - } - - /// Inserts the deployment unit data into the builder. - pub fn commit(self) { - self.base.extend(self.units); - } -} - -#[derive(Clone, Debug, Error)] -pub struct DuplicateDeploymentUnitError { - /// The scope within which duplicates were found. - pub scope: DeploymentUnitScope, - - /// Duplicates found while inserting deployment unit data. - /// - /// For `Leaf`, `before` is the existing data and - /// `after` is the new data. - pub duplicates: BTreeMap>, -} - -impl DuplicateDeploymentUnitError { - // Note: existing and new should have the same hash_id. - fn new_single( - scope: DeploymentUnitScope, - existing: DeploymentUnitData, - new: DeploymentUnitData, - ) -> Self { - let mut duplicates = BTreeMap::new(); - let hash_id = existing.hash_id(); - duplicates.insert(hash_id, Leaf { before: existing, after: new }); - Self { scope, duplicates } - } -} - -impl fmt::Display for DuplicateDeploymentUnitError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - // For a single deployment unit, we can simply display the artifact kind and hash. - if self.duplicates.len() == 1 { - let (hash_id, data) = self.duplicates.first_key_value().unwrap(); - // XXX: should `ArtifactHashId` have a `Display` impl, or maybe a - // `.display()` or `.display_human()` method? - write!( - f, - "a deployment unit with the same kind and hash already exists in this {}:\n\ - {hash_id} (existing {}; new {})", - self.scope, data.before, data.after, - ) - } else { - writeln!( - f, - "{} deployment units with the same kind and hash already exist in this {}:", - self.duplicates.len(), - self.scope, - )?; - for (hash_id, data) in &self.duplicates { - writeln!( - f, - " - for {hash_id} (existing: {}; new: {})", - data.before, data.after, - )?; - } - - Ok(()) - } - } -} diff --git a/lib/src/assemble/manifest.rs b/lib/src/assemble/manifest.rs deleted file mode 100644 index 8fb425f..0000000 --- a/lib/src/assemble/manifest.rs +++ /dev/null @@ -1,1019 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use std::collections::{BTreeMap, BTreeSet}; -use std::io::BufReader; -use std::str::FromStr; -use std::{fmt, fs}; - -use anyhow::{Context, Result, bail, ensure}; -use camino::{Utf8Path, Utf8PathBuf}; -use itertools::Itertools; -use parse_size::parse_size; -use semver::Version; -use serde::{Deserialize, Serialize}; -use tufaceous_artifact::{ArtifactKind, ArtifactVersion, KnownArtifactKind}; - -use crate::assemble::{DeploymentUnitData, DeploymentUnitScope}; -use crate::{ - ArtifactSource, COSMO_HOST_PHASE_1_FILE_NAME, - CompositeControlPlaneArchiveBuilder, CompositeEntry, - CompositeHostArchiveBuilder, CompositeRotArchiveBuilder, - GIMLET_HOST_PHASE_1_FILE_NAME, HOST_PHASE_2_FILE_NAME, HostPhaseImages, - MtimeSource, ROT_ARCHIVE_A_FILE_NAME, ROT_ARCHIVE_B_FILE_NAME, - make_filler_text, make_filler_text_with_seed, -}; - -use super::{ArtifactDeploymentUnits, DeploymentUnitMapBuilder}; - -static FAKE_MANIFEST_TOML: &str = - include_str!("../../../bin/manifests/fake.toml"); - -/// A list of components in a TUF repo representing a single update. -#[derive(Clone, Debug)] -pub struct ArtifactManifest { - pub system_version: Version, - pub artifacts: BTreeMap>, -} - -impl ArtifactManifest { - /// Reads a manifest in from a TOML file. - pub fn from_path(path: &Utf8Path) -> Result { - let input = fs_err::read_to_string(path)?; - let base_dir = path - .parent() - .with_context(|| format!("path `{path}` did not have a parent"))?; - Self::from_str(base_dir, &input) - } - - /// Deserializes a manifest from an input string. - pub fn from_str(base_dir: &Utf8Path, input: &str) -> Result { - let manifest = DeserializedManifest::from_str(input)?; - Self::from_deserialized(base_dir, manifest) - } - - /// Creates a manifest from a [`DeserializedManifest`]. - pub fn from_deserialized( - base_dir: &Utf8Path, - manifest: DeserializedManifest, - ) -> Result { - // Replace all paths in the deserialized manifest with absolute ones, - // and do some processing to support flexible manifests: - // - // 1. assemble any composite artifacts from their pieces - // 2. replace any "fake" artifacts with in-memory buffers - // - // Currently both of those transformations produce - // `ArtifactSource::Memory(_)` variants (i.e., composite and fake - // artifacts all sit in-memory until we're done with the manifest), - // which puts some limits on how large the inputs to the manifest can - // practically be. If this becomes onerous, we could instead write the - // transformed artifacts to temporary files. - // - // We do some additional error checking here to make sure the - // `CompositeZZZ` variants are only used with their corresponding - // `KnownArtifactKind`s. It would be nicer to enforce this more - // statically and let serde do these checks, but that seems relatively - // tricky in comparison to these checks. - - Ok(ArtifactManifest { - system_version: manifest.system_version, - artifacts: manifest - .artifacts - .into_iter() - .map(|(kind, entries)| { - Self::parse_deserialized_entries(base_dir, kind, entries) - }) - .collect::>()?, - }) - } - - fn parse_deserialized_entries( - base_dir: &Utf8Path, - kind: KnownArtifactKind, - entries: Vec, - ) -> Result<(KnownArtifactKind, Vec)> { - let entries = entries - .into_iter() - .map(|artifact_data| { - let (source, deployment_units) = match artifact_data.source { - DeserializedArtifactSource::File { path } => { - let path = base_dir.join(&path); - - // Host images are actually composite artifacts, and we - // need to treat them that way for installinator. - let deployment_units = - if kind == KnownArtifactKind::Host { - let file = - fs::File::open(&path).with_context(|| { - format!( - "error opening host image at `{path}`" - ) - })?; - let reader = BufReader::new(file); - let images = HostPhaseImages::extract(reader)?; - - let mut data_builder = - DeploymentUnitMapBuilder::new( - DeploymentUnitScope::Artifact { - composite_kind: kind, - }, - ); - data_builder - .insert(DeploymentUnitData { - name: COSMO_HOST_PHASE_1_FILE_NAME - .to_owned(), - version: artifact_data.version.clone(), - kind: ArtifactKind::COSMO_HOST_PHASE_1, - hash: images.cosmo_phase_1_hash(), - }) - .expect("unique kind"); - data_builder - .insert(DeploymentUnitData { - name: GIMLET_HOST_PHASE_1_FILE_NAME - .to_owned(), - version: artifact_data.version.clone(), - kind: ArtifactKind::GIMLET_HOST_PHASE_1, - hash: images.gimlet_phase_1_hash(), - }) - .expect("unique kind"); - - data_builder - .insert(DeploymentUnitData { - name: HOST_PHASE_2_FILE_NAME.to_owned(), - version: artifact_data.version.clone(), - kind: ArtifactKind::HOST_PHASE_2, - hash: images.phase_2_hash(), - }) - .expect("unique kind"); - - data_builder.finish_units() - } else { - // It would be nice to extract other kinds of - // composite artifacts here, but (a) we don't - // have a need for that in this case and (b) the - // code for that currently lives in omicron's - // update-common. - ArtifactDeploymentUnits::SingleUnit - }; - - (ArtifactSource::File(path), deployment_units) - } - DeserializedArtifactSource::Fake { size, data_version } => { - // This test-only environment variable is used to - // simulate two artifacts with different - // name/version/kind but the same hash. - let data_version = data_version - .as_ref() - .unwrap_or(&artifact_data.version); - let fake_data = - FakeDataAttributes::new(kind, data_version) - .make_data(size as usize); - ( - ArtifactSource::Memory(fake_data.into()), - ArtifactDeploymentUnits::SingleUnit, - ) - } - DeserializedArtifactSource::CompositeHost { - gimlet_phase_1, - cosmo_phase_1, - phase_2, - } => { - ensure!( - matches!( - kind, - KnownArtifactKind::Host - | KnownArtifactKind::Trampoline - ), - "`composite_host` source cannot be used with \ - artifact kind {kind:?}" - ); - - let mtime_source = if gimlet_phase_1.is_fake() - && cosmo_phase_1.is_fake() - && phase_2.is_fake() - { - // Ensure stability of fake artifacts. - MtimeSource::Zero - } else { - MtimeSource::Now - }; - - let mut builder = CompositeHostArchiveBuilder::new( - Vec::new(), - mtime_source, - )?; - let cosmo_phase_1_hash = cosmo_phase_1.with_entry( - FakeDataAttributes::new_with_seed( - kind, - &artifact_data.version, - "cosmo".to_string(), - ), - |entry| builder.append_cosmo_phase_1(entry), - )?; - let gimlet_phase_1_hash = gimlet_phase_1.with_entry( - FakeDataAttributes::new_with_seed( - kind, - &artifact_data.version, - "gimlet".to_string(), - ), - |entry| builder.append_gimlet_phase_1(entry), - )?; - - let phase_2_hash = phase_2.with_entry( - FakeDataAttributes::new( - kind, - &artifact_data.version, - ), - |entry| builder.append_phase_2(entry), - )?; - let source = - ArtifactSource::Memory(builder.finish()?.into()); - - let mut data_builder = DeploymentUnitMapBuilder::new( - DeploymentUnitScope::Artifact { - composite_kind: kind, - }, - ); - data_builder - .insert(DeploymentUnitData { - name: GIMLET_HOST_PHASE_1_FILE_NAME.to_owned(), - version: artifact_data.version.clone(), - kind: ArtifactKind::GIMLET_HOST_PHASE_1, - hash: gimlet_phase_1_hash, - }) - .expect("unique kind"); - - data_builder - .insert(DeploymentUnitData { - name: COSMO_HOST_PHASE_1_FILE_NAME.to_owned(), - version: artifact_data.version.clone(), - kind: ArtifactKind::COSMO_HOST_PHASE_1, - hash: cosmo_phase_1_hash, - }) - .expect("unique kind"); - data_builder - .insert(DeploymentUnitData { - name: HOST_PHASE_2_FILE_NAME.to_owned(), - version: artifact_data.version.clone(), - kind: ArtifactKind::HOST_PHASE_2, - hash: phase_2_hash, - }) - .expect("unique kind"); - - (source, data_builder.finish_units()) - } - DeserializedArtifactSource::CompositeRot { - archive_a, - archive_b, - } => { - let (a_kind, b_kind) = - kind.rot_a_and_b_kinds().context( - "`composite_rot` source cannot be used with \ - non-RoT artifact kind", - )?; - - let mtime_source = - if archive_a.is_fake() && archive_b.is_fake() { - // Ensure stability of fake artifacts. - MtimeSource::Zero - } else { - MtimeSource::Now - }; - - let mut builder = CompositeRotArchiveBuilder::new( - Vec::new(), - mtime_source, - )?; - let archive_a_hash = archive_a.with_entry( - FakeDataAttributes::new( - kind, - &artifact_data.version, - ), - |entry| builder.append_archive_a(entry), - )?; - let archive_b_hash = archive_b.with_entry( - FakeDataAttributes::new( - kind, - &artifact_data.version, - ), - |entry| builder.append_archive_b(entry), - )?; - - let mut data_builder = DeploymentUnitMapBuilder::new( - DeploymentUnitScope::Artifact { - composite_kind: kind, - }, - ); - data_builder - .insert(DeploymentUnitData { - name: ROT_ARCHIVE_A_FILE_NAME.to_owned(), - version: artifact_data.version.clone(), - kind: a_kind, - hash: archive_a_hash, - }) - .expect("unique kind in empty map"); - data_builder - .insert(DeploymentUnitData { - name: ROT_ARCHIVE_B_FILE_NAME.to_owned(), - version: artifact_data.version.clone(), - kind: b_kind, - hash: archive_b_hash, - }) - .expect("unique kind in empty map"); - - ( - ArtifactSource::Memory(builder.finish()?.into()), - data_builder.finish_units(), - ) - } - DeserializedArtifactSource::CompositeControlPlane { - zones, - } => { - ensure!( - kind == KnownArtifactKind::ControlPlane, - "`composite_control_plane` source cannot be \ - used with artifact kind {kind:?}" - ); - - // Ensure stability of fake artifacts. - let mtime_source = if zones.iter().all(|z| z.is_fake()) - { - MtimeSource::Zero - } else { - MtimeSource::Now - }; - - let data = Vec::new(); - let mut builder = - CompositeControlPlaneArchiveBuilder::new( - data, - mtime_source, - )?; - - let zone_kind = - ArtifactKind::from(KnownArtifactKind::Zone); - let mut data_builder = DeploymentUnitMapBuilder::new( - DeploymentUnitScope::Artifact { - composite_kind: kind, - }, - ); - - for zone in zones { - let (hash, name) = zone.with_name_and_entry( - &artifact_data.version, - |name, entry| builder.append_zone(name, entry), - )?; - data_builder.insert(DeploymentUnitData { - name: name.to_owned(), - version: artifact_data.version.clone(), - kind: zone_kind.clone(), - hash, - })?; - } - ( - ArtifactSource::Memory(builder.finish()?.into()), - data_builder.finish_units(), - ) - } - }; - let data = ArtifactData { - name: artifact_data.name, - version: artifact_data.version, - source, - deployment_units, - }; - Ok(data) - }) - .collect::>()?; - Ok((kind, entries)) - } - - /// Returns a fake manifest. Useful for testing. - pub fn new_fake() -> Self { - // The base directory doesn't matter for fake manifests. - Self::from_str(".".into(), FAKE_MANIFEST_TOML) - .expect("the fake manifest is a valid manifest") - } - - /// Checks that all versions are valid semver. - pub fn verify_all_semver(&self) -> Result<()> { - let mut non_semver = Vec::new(); - for artifacts in self.artifacts.values() { - for artifact in artifacts { - if artifact.version.as_str().parse::().is_err() { - non_semver.push(artifact); - } - } - } - - if !non_semver.is_empty() { - bail!( - "non-semver versions found: {}", - non_semver.iter().map(|d| d.display()).join(", "), - ); - } - Ok(()) - } - - /// Checks that all expected artifacts are present, returning an error with - /// details if any artifacts are missing. - pub fn verify_all_present(&self) -> Result<()> { - let all_artifacts: BTreeSet<_> = KnownArtifactKind::iter() - .filter(|k| { - // Installinator documents are generated by tufaceous and should - // not be part of the manifest. - !matches!( - k, - KnownArtifactKind::Zone - | KnownArtifactKind::InstallinatorDocument - | KnownArtifactKind::MeasurementCorpus - ) - }) - .collect(); - let present_artifacts: BTreeSet<_> = - self.artifacts.keys().copied().collect(); - - let missing = &all_artifacts - &present_artifacts; - if !missing.is_empty() { - bail!( - "manifest has missing artifacts: {}", - itertools::join(missing, ", ") - ); - } - - Ok(()) - } -} - -#[derive(Debug)] -struct FakeDataAttributes<'a> { - kind: KnownArtifactKind, - version: &'a ArtifactVersion, - seed: String, -} - -impl<'a> FakeDataAttributes<'a> { - fn new(kind: KnownArtifactKind, version: &'a ArtifactVersion) -> Self { - Self { kind, version, seed: "".to_string() } - } - - fn new_with_seed( - kind: KnownArtifactKind, - version: &'a ArtifactVersion, - seed: String, - ) -> Self { - Self { kind, version, seed } - } - - fn make_data(&self, size: usize) -> Vec { - use hubtools::{CabooseBuilder, HubrisArchiveBuilder}; - - let board = match self.kind { - KnownArtifactKind::Host | KnownArtifactKind::Trampoline => { - // need to use the extra seed to be able to generate different gimlet vs - // cosmo images - return make_filler_text_with_seed( - &self.kind.to_string(), - self.version, - size, - &self.seed, - ); - } - // non-Hubris artifacts: just make fake data - KnownArtifactKind::ControlPlane | KnownArtifactKind::Zone => { - return make_filler_text( - &self.kind.to_string(), - self.version, - size, - ); - } - KnownArtifactKind::InstallinatorDocument => { - panic!( - "fake manifest should not have an installinator document" - ); - } - KnownArtifactKind::MeasurementCorpus => { - return make_filler_text( - &self.kind.to_string(), - self.version, - size, - ); - } - - // In production, all the bootloaders and RoTs claim to have the - // same board (currently: `oxide-rot-1`). Let's do that here too. - KnownArtifactKind::GimletRotBootloader - | KnownArtifactKind::PscRotBootloader - | KnownArtifactKind::SwitchRotBootloader - | KnownArtifactKind::GimletRot - | KnownArtifactKind::PscRot - | KnownArtifactKind::SwitchRot => "SimRot", - - // SP artifacts: build a fake archive (SimGimletSp and - // SimGimletRot are used by sp-sim) - KnownArtifactKind::GimletSp => "SimGimletSp", - KnownArtifactKind::SwitchSp => "SimSidecarSp", - KnownArtifactKind::PscSp => "SimPscSp", - }; - - let caboose = { - // We use a fake git commit that contains `self.kind` to ensure that - // the artifacts we produce are distinct for each `kind`, even if - // all the other caboose fields are identical. - let mut builder = CabooseBuilder::default() - .git_commit(format!("this-is-a-fake-{}", self.kind)) - .board(board) - .name(board) - .version(self.version.to_string()); - - if let Some(sign) = self.kind.fake_artifact_hubris_sign() { - builder = builder.sign(sign); - } - - builder.build() - }; - - let mut builder = HubrisArchiveBuilder::with_fake_image(); - builder.write_caboose(caboose.as_slice()).unwrap(); - builder.build_to_vec().unwrap() - } -} - -/// Information about an individual artifact. -#[derive(Clone, Debug)] -pub struct ArtifactData { - pub name: String, - pub version: ArtifactVersion, - pub source: ArtifactSource, - pub deployment_units: ArtifactDeploymentUnits, -} - -impl ArtifactData { - /// Returns a displayer for the name and version of the artifact. - pub fn display(&self) -> ArtifactDataDisplay<'_> { - ArtifactDataDisplay { data: self } - } -} - -pub struct ArtifactDataDisplay<'a> { - data: &'a ArtifactData, -} - -impl fmt::Display for ArtifactDataDisplay<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{} ({})", self.data.name, self.data.version) - } -} - -/// Deserializable version of [`ArtifactManifest`]. -/// -/// Since manifests require a base directory to be deserialized properly, -/// we don't expose the `Deserialize` impl on `ArtifactManifest, forcing -/// consumers to go through [`ArtifactManifest::from_path`] or -/// [`ArtifactManifest::from_str`]. -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] -#[serde(rename_all = "snake_case")] -pub struct DeserializedManifest { - pub system_version: Version, - #[serde(rename = "artifact")] - pub artifacts: BTreeMap>, -} - -impl DeserializedManifest { - pub fn from_path(path: &Utf8Path) -> Result { - let input = fs_err::read_to_string(path)?; - Self::from_str(&input).with_context(|| { - format!("error deserializing manifest from {path}") - }) - } - - pub fn to_toml(&self) -> Result { - toml::to_string(self).context("error serializing manifest to TOML") - } - - /// For fake manifests, applies a set of changes to them. - /// - /// Intended for testing. - pub fn apply_tweaks(&mut self, tweaks: &[ManifestTweak]) -> Result<()> { - for tweak in tweaks { - match tweak { - ManifestTweak::SystemVersion(version) => { - self.system_version = version.clone(); - } - ManifestTweak::ArtifactVersion { kind, version } => { - let entries = - self.artifacts.get_mut(kind).with_context(|| { - format!( - "manifest does not have artifact kind \ - {kind}", - ) - })?; - for entry in entries { - entry.version = version.clone(); - } - } - ManifestTweak::ArtifactSize { kind, size_delta } => { - let entries = - self.artifacts.get_mut(kind).with_context(|| { - format!( - "manifest does not have artifact kind \ - {kind}", - ) - })?; - - for entry in entries { - entry.source.apply_size_delta(*size_delta)?; - } - } - ManifestTweak::ArtifactDataVersion { kind, data_version } => { - let entries = - self.artifacts.get_mut(kind).with_context(|| { - format!( - "manifest does not have artifact kind \ - {kind}", - ) - })?; - - for entry in entries { - entry - .source - .apply_data_version(data_version.as_ref())?; - } - } - } - } - - Ok(()) - } - - /// Returns the fake manifest. - pub fn fake() -> Self { - Self::from_str(FAKE_MANIFEST_TOML).unwrap() - } - - /// Returns a version of the fake manifest with a set of changes applied. - /// - /// This is primarily intended for testing. - pub fn tweaked_fake(tweaks: &[ManifestTweak]) -> Self { - let mut manifest = Self::fake(); - manifest - .apply_tweaks(tweaks) - .expect("builtin fake manifest should accept all tweaks"); - - manifest - } -} - -impl FromStr for DeserializedManifest { - type Err = anyhow::Error; - - fn from_str(input: &str) -> Result { - let de = toml::Deserializer::new(input); - serde_path_to_error::deserialize(de) - .context("error deserializing manifest") - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] -#[serde(rename_all = "snake_case")] -pub struct DeserializedArtifactData { - pub name: String, - pub version: ArtifactVersion, - pub source: DeserializedArtifactSource, -} - -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] -#[serde(tag = "kind", rename_all = "kebab-case")] -pub enum DeserializedArtifactSource { - File { - path: Utf8PathBuf, - }, - Fake { - #[serde(deserialize_with = "deserialize_byte_size")] - size: u64, - /// The internal version to use while constructing the fake artifact - /// data. - /// - /// If not set, the artifact's version is used. - #[serde(default)] - data_version: Option, - }, - CompositeHost { - gimlet_phase_1: DeserializedFileArtifactSource, - cosmo_phase_1: DeserializedFileArtifactSource, - phase_2: DeserializedFileArtifactSource, - }, - CompositeRot { - archive_a: DeserializedFileArtifactSource, - archive_b: DeserializedFileArtifactSource, - }, - CompositeControlPlane { - zones: Vec, - }, -} - -impl DeserializedArtifactSource { - fn apply_size_delta(&mut self, size_delta: i64) -> Result<()> { - match self { - DeserializedArtifactSource::File { .. } => { - bail!("cannot apply size delta to `file` source") - } - DeserializedArtifactSource::Fake { size, data_version: _ } => { - *size = (*size).saturating_add_signed(size_delta); - Ok(()) - } - DeserializedArtifactSource::CompositeHost { - gimlet_phase_1, - cosmo_phase_1, - phase_2, - } => { - gimlet_phase_1.apply_size_delta(size_delta)?; - cosmo_phase_1.apply_size_delta(size_delta)?; - phase_2.apply_size_delta(size_delta)?; - Ok(()) - } - DeserializedArtifactSource::CompositeRot { - archive_a, - archive_b, - } => { - archive_a.apply_size_delta(size_delta)?; - archive_b.apply_size_delta(size_delta)?; - Ok(()) - } - DeserializedArtifactSource::CompositeControlPlane { zones } => { - for zone in zones { - zone.apply_size_delta(size_delta)?; - } - Ok(()) - } - } - } - - fn apply_data_version( - &mut self, - new_data_version: Option<&ArtifactVersion>, - ) -> Result<()> { - match self { - DeserializedArtifactSource::File { .. } => { - bail!("cannot apply data version to `file` source") - } - DeserializedArtifactSource::Fake { data_version, .. } => { - *data_version = new_data_version.cloned(); - Ok(()) - } - DeserializedArtifactSource::CompositeHost { .. } => { - bail!( - "cannot yet apply data version to `composite_host` source" - ) - } - DeserializedArtifactSource::CompositeRot { .. } => { - bail!("cannot yet apply data version to `composite_rot` source") - } - DeserializedArtifactSource::CompositeControlPlane { .. } => { - bail!( - "cannot yet apply data version to `composite_control_plane` source" - ) - } - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum DeserializedFileArtifactSource { - File { - path: Utf8PathBuf, - }, - Fake { - #[serde(deserialize_with = "deserialize_byte_size")] - size: u64, - }, -} - -impl DeserializedFileArtifactSource { - fn is_fake(&self) -> bool { - matches!(self, DeserializedFileArtifactSource::Fake { .. }) - } - - fn with_entry(&self, fake_attr: FakeDataAttributes, f: F) -> Result - where - F: FnOnce(CompositeEntry<'_>) -> Result, - { - let (data, mtime_source) = match self { - DeserializedFileArtifactSource::File { path } => { - let data = std::fs::read(path) - .with_context(|| format!("failed to read {path}"))?; - // For now, always use the current time as the source. (Maybe - // change this to use the mtime on disk in the future?) - (data, MtimeSource::Now) - } - DeserializedFileArtifactSource::Fake { size } => { - (fake_attr.make_data(*size as usize), MtimeSource::Zero) - } - }; - let entry = CompositeEntry { data: &data, mtime_source }; - f(entry) - } - - fn apply_size_delta(&mut self, size_delta: i64) -> Result<()> { - match self { - DeserializedFileArtifactSource::File { .. } => { - bail!("cannot apply size delta to `file` source") - } - DeserializedFileArtifactSource::Fake { size } => { - *size = (*size).saturating_add_signed(size_delta); - Ok(()) - } - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum DeserializedControlPlaneZoneSource { - File { - path: Utf8PathBuf, - #[serde(skip_serializing_if = "Option::is_none")] - file_name: Option, - }, - #[serde(rename_all = "snake_case")] - Fake { - artifact_name: String, - file_name: String, - #[serde(deserialize_with = "deserialize_byte_size")] - size: u64, - }, -} - -impl DeserializedControlPlaneZoneSource { - fn is_fake(&self) -> bool { - matches!(self, DeserializedControlPlaneZoneSource::Fake { .. }) - } - - fn with_name_and_entry( - &self, - version: &ArtifactVersion, - f: F, - ) -> Result - where - F: FnOnce(&str, CompositeEntry<'_>) -> Result, - { - let (name, data, mtime_source) = match self { - DeserializedControlPlaneZoneSource::File { path, file_name } => { - let data = std::fs::read(path) - .with_context(|| format!("failed to read {path}"))?; - let name = file_name - .as_deref() - .or_else(|| path.file_name()) - .with_context(|| { - format!("zone path missing file name: {path}") - })?; - // For now, always use the current time as the source. (Maybe - // change this to use the mtime on disk in the future?) - (name.to_owned(), data, MtimeSource::Now) - } - DeserializedControlPlaneZoneSource::Fake { - artifact_name, - file_name, - size, - } => { - use flate2::{Compression, write::GzEncoder}; - use tufaceous_brand_metadata::{ - ArchiveType, LayerInfo, Metadata, - }; - - let mut tar = tar::Builder::new(GzEncoder::new( - Vec::new(), - Compression::fast(), - )); - - let metadata = Metadata::new(ArchiveType::Layer(LayerInfo { - pkg: artifact_name.clone(), - version: version.clone(), - })); - metadata.append_to_tar(&mut tar, 0)?; - - let mut h = tar::Header::new_ustar(); - h.set_entry_type(tar::EntryType::Regular); - h.set_path("fake")?; - h.set_mode(0o444); - h.set_size(*size); - h.set_mtime(0); - h.set_cksum(); - tar.append( - &h, - make_filler_text(artifact_name, version, *size as usize) - .as_slice(), - )?; - - let data = tar.into_inner()?.finish()?; - (file_name.clone(), data, MtimeSource::Zero) - } - }; - let entry = CompositeEntry { data: &data, mtime_source }; - f(&name, entry) - } - - fn apply_size_delta(&mut self, size_delta: i64) -> Result<()> { - match self { - DeserializedControlPlaneZoneSource::File { .. } => { - bail!("cannot apply size delta to `file` source") - } - DeserializedControlPlaneZoneSource::Fake { size, .. } => { - (*size) = (*size).saturating_add_signed(size_delta); - Ok(()) - } - } - } -} -/// A change to apply to a manifest. -#[derive(Clone, Debug)] -pub enum ManifestTweak { - /// Update the system version. - SystemVersion(Version), - - /// Update the versions for this artifact. - ArtifactVersion { kind: KnownArtifactKind, version: ArtifactVersion }, - - /// Update the size of this fake artifact. - ArtifactSize { kind: KnownArtifactKind, size_delta: i64 }, - - /// Update the data version of this fake artifact. - /// - /// This version is typically the same as the artifact version, but it can - /// be changed for testing purposes. - ArtifactDataVersion { - kind: KnownArtifactKind, - /// Setting this to `None` resets the data version to the artifact - /// version. - data_version: Option, - }, -} - -fn deserialize_byte_size<'de, D>(deserializer: D) -> Result -where - D: serde::Deserializer<'de>, -{ - // Attempt to deserialize the size as either a string or an integer. - - struct Visitor; - - impl serde::de::Visitor<'_> for Visitor { - type Value = u64; - - fn expecting( - &self, - formatter: &mut std::fmt::Formatter, - ) -> std::fmt::Result { - formatter - .write_str("a string representing a byte size or an integer") - } - - fn visit_str(self, value: &str) -> Result - where - E: serde::de::Error, - { - parse_size(value).map_err(|_| { - serde::de::Error::invalid_value( - serde::de::Unexpected::Str(value), - &self, - ) - }) - } - - // TOML uses i64, not u64 - fn visit_i64(self, value: i64) -> Result - where - E: serde::de::Error, - { - Ok(value as u64) - } - - fn visit_u64(self, value: u64) -> Result - where - E: serde::de::Error, - { - Ok(value) - } - } - - deserializer.deserialize_any(Visitor) -} - -#[cfg(test)] -mod tests { - use super::*; - - // Ensure that the fake manifest roundtrips after serialization and - // deserialization. - #[test] - fn fake_roundtrip() { - let manifest = DeserializedManifest::fake(); - let toml = toml::to_string(&manifest).unwrap(); - let deserialized = DeserializedManifest::from_str(&toml) - .expect("fake manifest is a valid manifest"); - assert_eq!(manifest, deserialized); - } -} diff --git a/lib/src/assemble/mod.rs b/lib/src/assemble/mod.rs deleted file mode 100644 index 51542e2..0000000 --- a/lib/src/assemble/mod.rs +++ /dev/null @@ -1,11 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -mod build; -mod deployment_unit; -mod manifest; - -pub use build::*; -pub use deployment_unit::*; -pub use manifest::*; diff --git a/lib/src/edit/editor.rs b/lib/src/edit/editor.rs new file mode 100644 index 0000000..8c70923 --- /dev/null +++ b/lib/src/edit/editor.rs @@ -0,0 +1,623 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::BTreeMap; +use std::collections::HashMap; +use std::num::NonZero; +use std::sync::Arc; + +use camino::Utf8Path; +use camino::Utf8PathBuf; +use semver::Version; +use tokio::sync::Semaphore; +use tokio::task::JoinSet; +use tufaceous_artifact::ArtifactHash; +use tufaceous_artifact::ArtifactVersion; +use tufaceous_artifact::InstallinatorDocument; +use tufaceous_artifact::KnownArtifactTags; +use tufaceous_artifact::Metadata; +use tufaceous_artifact::OsVariant; +use tufaceous_artifact::RotBootloaderTags; +use tufaceous_artifact::RotSlot; +use tufaceous_artifact::RotTags; +use tufaceous_artifact::SpTags; + +use crate::Repository; +use crate::edit::UnsignedRepository; +use crate::edit::input::Input; +use crate::edit::input::Output; +use crate::edit::source::BytesSource; +use crate::edit::source::FileSource; +use crate::edit::source::RepositorySource; +use crate::edit::source::Target; +use crate::edit::source::TargetSource; +use crate::error::Error; +use crate::error::ErrorKind; +use crate::schema::ArtifactSchema; +use crate::schema::ArtifactSetSchema; + +#[derive(Debug, Clone)] +#[must_use] +pub struct RepositoryEditor<'a> { + system_version: Version, + artifact_version: ArtifactVersion, + generate_installinator_document: bool, + targets: HashMap>>, + artifacts: BTreeMap, + metadata: BTreeMap, + max_threads: NonZero, +} + +impl<'a> RepositoryEditor<'a> { + /// Create an empty repository editor. + pub fn new(system_version: Version) -> Result { + Ok(Self { + artifact_version: ArtifactVersion::new(system_version.to_string())?, + system_version, + generate_installinator_document: true, + targets: HashMap::new(), + artifacts: BTreeMap::new(), + metadata: BTreeMap::new(), + max_threads: std::thread::available_parallelism() + .unwrap_or(NonZero::::MIN), + }) + } + + /// Change the system version of the repository. + pub fn set_system_version( + self, + system_version: Version, + ) -> Result { + Ok(Self { + artifact_version: ArtifactVersion::new(system_version.to_string())?, + system_version, + ..self + }) + } + + /// Sets whether an Installinator document should be generated based on the + /// artifacts in the repository. + /// + /// Defaults to `true`. The document is generated during + /// [`RepositoryEditor::finish`]. + pub fn set_generate_installinator_document( + self, + generate_installinator_document: bool, + ) -> Self { + Self { generate_installinator_document, ..self } + } + + /// Add a measurement corpus to the repository. + pub async fn add_measurement_corpus( + self, + path: impl Into, + ) -> Result { + let source = FileSource::open(path.into()).await?; + let input = Input::measurement_corpus(source, None).await?; + self.insert_input(input) + } + + /// Add a fake measurement corpus to the repository. + /// + /// `hashes` specifies the number of SHA256 hashes to list in the CoRIM + /// document. This can be used to create different documents at the same + /// version. + pub fn add_fake_measurement_corpus( + self, + hashes: usize, + version: ArtifactVersion, + ) -> Result { + let input = Input::fake_measurement_corpus(hashes, version, None)?; + self.insert_input(input) + } + + /// Add an OS image to the repository. + /// + /// `output_dir` is a path to the output directory for `helios-build image` + /// (the `-o` argument). This directory contains `cosmo.rom`, `gimlet.rom`, + /// and `zfs.img`. + pub async fn add_os_image_dir( + self, + variant: OsVariant, + output_dir: impl AsRef, + ) -> Result { + let input = Input::os_images( + variant, + output_dir.as_ref(), + None, + self.artifact_version.clone(), + ) + .await?; + self.insert_input(input) + } + + /// Add a fake OS image to the repository. + pub fn add_fake_os_image(self, variant: OsVariant) -> Result { + let input = + Input::fake_os_images(variant, self.artifact_version.clone(), None); + self.insert_input(input) + } + + /// Add a Root of Trust Hubris archive to the repository. + /// + /// Tags are automatically determined based on the image caboose, except + /// for `slot` which must be specified. + pub async fn add_rot_archive( + self, + rot_slot: RotSlot, + path: impl Into, + ) -> Result { + let source = FileSource::open(path.into()).await?; + self.insert_input(Input::rot_archive(source, None, rot_slot).await?) + } + + /// Add a Root of Trust Bootloader Hubris archive to the repository. + /// + /// Tags are automatically determined based on the image caboose. + pub async fn add_rot_bootloader_archive( + self, + path: impl Into, + ) -> Result { + let source = FileSource::open(path.into()).await?; + self.insert_input(Input::rot_bootloader_archive(source, None).await?) + } + + /// Add a Service Processor Hubris archive to the repository. + /// + /// Tags are automatically determined based on the image caboose. + pub async fn add_sp_archive( + self, + path: impl Into, + ) -> Result { + let source = FileSource::open(path.into()).await?; + self.insert_input(Input::sp_archive(source, None).await?) + } + + /// Add a fake Root of Trust Hubris archive to the repository. + /// + /// This will generate a fake Hubris archive with the appropriate tags. + pub fn add_fake_rot_archive(self, tags: RotTags) -> Result { + let input = + Input::fake_rot_archive(tags, self.artifact_version.clone(), None)?; + self.insert_input(input) + } + + /// Add a fake Root of Trust Bootloader Hubris archive to the repository. + /// + /// This will generate a fake Hubris archive with the appropriate tags. + pub fn add_fake_rot_bootloader_archive( + self, + tags: RotBootloaderTags, + ) -> Result { + let input = Input::fake_rot_bootloader_archive( + tags, + self.artifact_version.clone(), + None, + )?; + self.insert_input(input) + } + + /// Add a fake Service Processor Hubris archive to the repository. + /// + /// This will generate a fake Hubris archive with the appropriate tags. + pub fn add_fake_sp_archive(self, tags: SpTags) -> Result { + let input = + Input::fake_sp_archive(tags, self.artifact_version.clone(), None)?; + self.insert_input(input) + } + + /// Add a zone image to the repository. + /// + /// The `zone_name` tag is automatically determined based on the layer + /// metadata (the `oxide.json` file that starts zone tarballs). + pub async fn add_zone_image( + self, + path: impl Into, + ) -> Result { + self.insert_input(Input::zone_image(path.into()).await?) + } + + /// Add a fake zone image to the repository. + /// + /// This will generate a tarball containing a matching `oxide.json` layer + /// metadata file. + pub fn add_fake_zone_image( + self, + zone_name: String, + file_name: String, + ) -> Result { + let input = Input::fake_zone_image( + zone_name, + file_name, + self.artifact_version.clone(), + None, + )?; + self.insert_input(input) + } + + /// Attempt to guess the artifact kind at `path` and add it to the + /// repository. + /// + /// This should only be used as a convenience method to human users. + /// Automation should not be making any guesses. + pub async fn guess_artifact( + self, + path: impl Into, + ) -> Result { + let input = + Input::guess(path.into(), self.artifact_version.clone()).await?; + self.insert_input(input) + } + + fn insert_input(mut self, input: Input) -> Result + where + T: Into>, + { + for output in input.outputs()? { + if let Some(artifact) = output.to_artifact_schema() { + insert_artifact(&mut self.artifacts, artifact)?; + } + self.targets + .entry(output.target_name) + .or_default() + .push(output.source.into()); + } + Ok(self) + } + + /// Add a non-artifact target to the repository. + /// + /// This target will be part of the signed set of files in the repository, + /// but will not be copied onto sleds for use by the control plane. This is + /// intended for ancillary files that are useful to systems other than the + /// control plane. + pub async fn add_extra_target( + mut self, + target_name: impl Into, + path: impl Into, + ) -> Result { + self.targets + .entry(target_name.into()) + .or_default() + .push(FileSource::open(path.into()).await?.into()); + Ok(self) + } + + /// Remove artifacts matching the given tags. + pub fn remove_artifacts( + mut self, + filter: &KnownArtifactTags, + ) -> Result { + let filter = + filter.to_tags().map_err(ErrorKind::ConvertKnownTagsToMap)?; + let removed = self + .artifacts + .extract_if(.., |_, artifact| artifact.tags == filter); + for (target_name, _) in removed { + self.targets.remove(&target_name); + } + Ok(self) + } + + /// Remove a target with the given target name. + pub fn remove_target(mut self, target_name: impl AsRef) -> Self { + let target_name = target_name.as_ref(); + self.targets.remove(target_name); + self.artifacts.remove(target_name); + self + } + + /// Create a fake repository for testing purposes. + pub fn fake(system_version: Version) -> Result { + let mut editor = Self::new(system_version)?; + let version = editor.artifact_version.clone(); + for input in Input::fake(&version, None)? { + editor = editor.insert_input(input)?; + } + Ok(editor) + } + + /// Create a fake repository for testing purposes where the system version, + /// artifact versions, and the versions interior to the artifact data are + /// potentially different. + /// + /// The Installinator document, if generated, always uses `system_version` + /// for the artifact and interior versions. + pub fn inconsistent_fake( + system_version: Version, + artifact_version: &ArtifactVersion, + interior_version: &ArtifactVersion, + ) -> Result { + let mut editor = Self::new(system_version)?; + for input in Input::fake(artifact_version, Some(interior_version))? { + editor = editor.insert_input(input)?; + } + Ok(editor) + } + + /// Set the repository-level metadata. + pub fn set_metadata(mut self, metadata: &Metadata) -> Result { + self.metadata = + metadata.to_map().map_err(ErrorKind::ConvertMetadataToMap)?; + Ok(self) + } + + /// Create a `RepositoryEditor` from a loaded repository. + /// + /// This creates an editor with references to all of the artifacts and + /// targets in the original repository. + pub fn from_repo(repo: &'a Repository) -> Result { + Self::new(repo.system_version().clone())?.import_repo(repo) + } + + /// Import all of the artifacts and targets from a repository into this + /// editor. + pub fn import_repo(mut self, repo: &'a Repository) -> Result { + for artifact in repo.to_artifact_schema()? { + insert_artifact(&mut self.artifacts, artifact)?; + } + for (target_name, target) in repo.targets() { + if target_name.raw() == ArtifactSetSchema::TARGET_NAME { + continue; + } + self.targets.entry(target_name.raw().to_owned()).or_default().push( + RepositorySource { + repo, + target_name: target_name.raw().to_owned(), + length: target.length, + sha256: target.hashes.sha256.to_vec(), + } + .into(), + ); + } + self.metadata = repo.metadata().clone(); + Ok(self) + } + + /// Set the maximum number of concurrent threads used for hashing targets + /// when `finish` is called. + /// + /// Defaults to [`std::thread::available_parallelism`] (or `1` if that + /// fails). + pub fn set_max_threads( + self, + max_threads: impl Into>, + ) -> Self { + Self { max_threads: max_threads.into(), ..self } + } + + /// Finalize the artifacts and targets, returning an [`UnsignedRepository`]. + pub async fn finish(self) -> Result, Error> { + let mut artifacts = self.artifacts; + + // Ensure each set of tags is unique, except for known tag sets that + // we expect to be non-unique. + let mut seen_tags = HashMap::new(); + for (target_name, artifact) in &artifacts { + if let Some(tags) = artifact.known_tags() + && tags.allow_multiple_artifacts() + { + continue; + } + + if let Some(first_target_name) = + seen_tags.insert(&artifact.tags, target_name) + { + return Err(ErrorKind::DisallowedTagCollision { + first_target_name: first_target_name.clone(), + second_target_name: target_name.clone(), + tags: artifact.tags.clone(), + } + .into()); + } + } + + // Collect all the sha256 hashes and lengths for each source. For file + // and fake sources, we want to calculate the hashes in parallel, so + // we spawn their calculation tasks on a JoinSet. Sources from borrowed + // repositories can't be moved into a task, but we already know their + // hash. + let semaphore = Arc::new(Semaphore::new(self.max_threads.get())); + let mut all_targets = Vec::new(); + let mut tasks = JoinSet::new(); + for (target_name, sources) in self.targets { + for source in sources { + let target_name = target_name.clone(); + let semaphore = semaphore.clone(); + match source { + TargetSource::Bytes(source) => { + tasks.spawn(async move { + let Ok(_permit) = semaphore.acquire().await else { + unreachable!("we never close the semaphore"); + }; + let target = source.into_target().await; + Ok::<_, Error>((target_name, target)) + }); + } + TargetSource::File(source) => { + tasks.spawn(async move { + let Ok(_permit) = semaphore.acquire().await else { + unreachable!("we never close the semaphore"); + }; + let target = source.into_target().await?; + Ok((target_name, target)) + }); + } + TargetSource::Repository(source) => { + all_targets.push((target_name, source.into_target()?)); + } + } + } + } + while let Some(result) = tasks.join_next().await { + all_targets.push(result??); + } + // Fold the targets back into a map, checking that duplicates have the + // same sha256 hash and length. + let mut targets = TargetMap::default(); + for (target_name, target) in all_targets { + targets.insert(target_name, target)?; + } + + if self.generate_installinator_document { + let output = generate_installinator_document( + artifacts.values().filter_map(|artifact| { + let target = targets.0.get(&artifact.target_name)?; + Some((artifact, target.sha256)) + }), + self.artifact_version.clone(), + )?; + let Some(artifact) = output.to_artifact_schema() else { + unreachable!( + "generate_installinator_document returns an artifact" + ); + }; + insert_artifact(&mut artifacts, artifact)?; + targets.insert( + output.target_name, + output.source.into_target().await, + )?; + } + + let document = ArtifactSetSchema { + system_version: self.system_version, + artifacts: artifacts.into_values().collect(), + metadata: self.metadata, + }; + let target = BytesSource::json(&document) + .map_err(ErrorKind::SerializeArtifacts)? + .into_target() + .await; + targets.insert(ArtifactSetSchema::TARGET_NAME.to_owned(), target)?; + Ok(UnsignedRepository::from_targets(targets.0)) + } +} + +#[derive(Debug, Default)] +struct TargetMap<'a>(BTreeMap>); + +impl<'a> TargetMap<'a> { + fn insert( + &mut self, + target_name: String, + target: Target<'a>, + ) -> Result<(), Error> { + if let Some(existing) = self.0.get(&target_name) { + if existing.length == target.length + && existing.sha256 == target.sha256 + { + if existing.source.cost() <= target.source.cost() { + return Ok(()); + } + } else { + return Err( + ErrorKind::TargetNameCollision { target_name }.into() + ); + } + } + self.0.insert(target_name, target); + Ok(()) + } +} + +pub(crate) fn generate_installinator_document( + artifacts: impl Iterator, ArtifactHash)>, + version: ArtifactVersion, +) -> Result, Error> { + let target_name = format!("installinator_document-{version}.json"); + let mut document = InstallinatorDocument::empty(version.clone()); + for (artifact, hash) in artifacts { + let artifact = artifact.as_ref(); + if let Some(artifact) = crate::util::installinator_artifact( + artifact.tags.clone(), + hash, + &artifact.target_name, + ) { + document.artifacts.insert(artifact); + } + } + let source = BytesSource::json(&document) + .map_err(ErrorKind::SerializeInstallinator)?; + Output::new( + target_name, + version, + &KnownArtifactTags::InstallinatorDocument, + source, + ) +} + +fn insert_artifact( + artifacts: &mut BTreeMap, + artifact: ArtifactSchema, +) -> Result<(), Error> { + if let Some(existing) = artifacts.get(&artifact.target_name) + && existing != &artifact + { + return Err(ErrorKind::TargetNameCollision { + target_name: artifact.target_name, + } + .into()); + } + artifacts.insert(artifact.target_name.clone(), artifact); + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use bytes::Bytes; + use chrono::Utc; + use futures_util::TryStreamExt; + use semver::Version; + use tufaceous_artifact::ArtifactVersion; + + use crate::RepositoryLoader; + use crate::edit::RepositoryEditor; + use crate::edit::source::BytesSource; + use crate::error::Error; + use crate::schema::ArtifactSchema; + + /// Test that all of the repository creation and loading machinery works + /// when an artifact is zero bytes. (This otherwise wouldn't be tested as + /// none of the fake artifacts are empty.) + #[tokio::test] + async fn empty_artifact() -> Result<(), Error> { + let log = slog::Logger::root(slog::Discard, slog::o!()); + let mut editor = RepositoryEditor::new(Version::new(1, 0, 0))? + .set_generate_installinator_document(false); + // Manually add an empty artifact. + let target_name = "empty".to_string(); + editor + .targets + .entry(target_name.clone()) + .or_default() + .push(BytesSource::new(Bytes::new()).into()); + editor.artifacts.insert( + target_name.clone(), + ArtifactSchema { + target_name, + version: ArtifactVersion::new_const("1.0.0"), + tags: BTreeMap::new(), + }, + ); + + let signed = editor.finish().await?.generate_root().sign().await?; + let zip = signed.write_zip(Vec::new(), Utc::now()).await?; + let repo = RepositoryLoader::new() + .trust_root(signed.root()) + .load_zip_buffer(zip, &log) + .await?; + let artifacts = repo.artifacts().iter().collect::>(); + assert_eq!(artifacts.len(), 1); + let data = repo + .read_artifact(artifacts[0]) + .await? + .map_ok(|bytes| bytes.to_vec()) + .try_concat() + .await?; + assert!(data.is_empty()); + Ok(()) + } +} diff --git a/lib/src/edit/fake.rs b/lib/src/edit/fake.rs new file mode 100644 index 0000000..cab8288 --- /dev/null +++ b/lib/src/edit/fake.rs @@ -0,0 +1,155 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::HashMap; + +use tufaceous_artifact::ArtifactSet; +use tufaceous_artifact::ArtifactVersion; +use tufaceous_artifact::OsVariant; +use tufaceous_artifact::RotBootloaderTags; +use tufaceous_artifact::RotKeyTableHash; +use tufaceous_artifact::RotSlot; +use tufaceous_artifact::RotTags; +use tufaceous_artifact::SpTags; + +use crate::edit::generate_installinator_document; +use crate::edit::input::Input; +use crate::edit::source::BytesSource; +use crate::error::Error; +use crate::schema::ArtifactSchema; + +const FAKE_SIGN: &str = + "0000000000000000000000000000000000000000000000000000000000000000"; + +const FAKE_ZONES: [(&str, &str); 11] = [ + ("clickhouse", "clickhouse.tar.gz"), + ("clickhouse_keeper", "clickhouse_keeper.tar.gz"), + ("clickhouse_server", "clickhouse_server.tar.gz"), + ("cockroachdb", "cockroachdb.tar.gz"), + ("crucible-zone", "crucible.tar.gz"), + ("crucible-pantry-zone", "crucible_pantry.tar.gz"), + ("external-dns", "external_dns.tar.gz"), + ("internal-dns", "internal_dns.tar.gz"), + ("ntp", "ntp.tar.gz"), + ("nexus", "nexus.tar.gz"), + ("oximeter", "oximeter.tar.gz"), +]; + +pub trait ArtifactSetExt: Sized { + fn fake(version: ArtifactVersion) -> Result; +} + +impl ArtifactSetExt for ArtifactSet { + fn fake(version: ArtifactVersion) -> Result { + let mut artifacts = HashMap::new(); + for input in Input::fake(&version, None)? { + for output in input.outputs()? { + artifacts.extend(output.into_artifact()); + } + } + artifacts.extend( + generate_installinator_document( + artifacts.iter().map(|(target_name, artifact)| { + ( + ArtifactSchema { + target_name: target_name.clone(), + version: artifact.version.clone(), + tags: artifact.tags.clone(), + }, + artifact.hash, + ) + }), + version, + )? + .into_artifact(), + ); + Ok(artifacts.into_values().collect()) + } +} + +impl Input { + pub(crate) fn fake( + version: &ArtifactVersion, + interior_version: Option<&ArtifactVersion>, + ) -> Result, Error> { + let mut inputs = Vec::new(); + for hashes in [4, 16] { + inputs.push(Self::fake_measurement_corpus( + hashes, + version.clone(), + interior_version, + )?); + } + for variant in [OsVariant::Host, OsVariant::Recovery] { + inputs.push(Self::fake_os_images( + variant, + version.clone(), + interior_version, + )); + } + for slot in [RotSlot::A, RotSlot::B] { + inputs.push(Self::fake_rot_archive( + RotTags { + rot_board: "SimRot".into(), + rot_rkth: Some(RotKeyTableHash::new(FAKE_SIGN)), + rot_slot: slot, + }, + version.clone(), + interior_version, + )?); + } + inputs.push(Self::fake_rot_bootloader_archive( + RotBootloaderTags { + rot_board: "SimRot".into(), + rot_rkth: Some(RotKeyTableHash::new(FAKE_SIGN)), + }, + version.clone(), + interior_version, + )?); + for board in ["SimGimletSp", "SimCosmoSp", "SimSidecarSp", "SimPscSp"] { + inputs.push(Self::fake_sp_archive( + SpTags { sp_board: board.into() }, + version.clone(), + interior_version, + )?); + } + for (zone_name, file_name) in FAKE_ZONES { + inputs.push(Self::fake_zone_image( + zone_name.into(), + file_name.into(), + version.clone(), + interior_version.cloned(), + )?); + } + Ok(inputs) + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use semver::Version; + use tufaceous_artifact::ArtifactSet; + use tufaceous_artifact::ArtifactVersion; + + use crate::Repository; + use crate::edit::ArtifactSetExt; + + #[tokio::test] + async fn fake_artifacts_equals_fake_repo() { + let log = slog::Logger::root(slog::Discard, slog::o!()); + let system_version = Version::new(1, 0, 0); + let version = ArtifactVersion::new(system_version.to_string()).unwrap(); + + let artifacts = ArtifactSet::fake(version).unwrap(); + // sleep 1 second to ensure any embedded timestamps would be different. + // we could ostensibly use `tokio::time::Instant` throughout the code + // base but that wouldn't take into account third-party libraries we use + // to generate fake artifacts. + tokio::time::sleep(Duration::from_secs(1)).await; + let repo = Repository::fake(system_version, &log).await.unwrap(); + assert_eq!(&artifacts, repo.artifacts()); + } +} diff --git a/lib/src/edit/guess.rs b/lib/src/edit/guess.rs new file mode 100644 index 0000000..b2a5c11 --- /dev/null +++ b/lib/src/edit/guess.rs @@ -0,0 +1,77 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::ops::ControlFlow; + +use bytes::Bytes; +use bytes::BytesMut; +use camino::Utf8PathBuf; +use tokio::fs::File; +use tokio::io::AsyncReadExt; +use tufaceous_artifact::ArtifactVersion; + +use crate::edit::input::Input; +use crate::edit::source::FileSource; +use crate::edit::source::TargetSource; +use crate::error::Error; +use crate::error::ErrorKind; +use crate::error::try_path; + +/// The input to guess methods: the first chunk of bytes from the file, and the +/// `FileSource` for converting into an [`Input`] if the guess method knows what +/// it is. +pub(crate) struct GuessInput { + pub(crate) file_start: Bytes, + pub(crate) source: FileSource, +} + +/// A guess method can: +/// +/// - say it knows what an input is, diverging the control flow: +/// `Ok(ControlFlow::Break(input))` +/// - say it doesn't know what an input is, returning the input and continuing +/// the control flow: `Ok(ControlFlow::Continue(guess_input))` +/// - return an error: `Err(error)` +pub(crate) type GuessResult = + Result>, GuessInput>, Error>; + +impl Input> { + pub(crate) async fn guess( + path: Utf8PathBuf, + version: ArtifactVersion, + ) -> Result { + if let Some(result) = Self::guess_os_images(&path, &version).await { + return result; + } + + let mut file = try_path!(File::open(&path).await, OpenFile, path); + let mut buf = BytesMut::zeroed(4096); + let n = try_path!(file.read(&mut buf).await, ReadFile, path); + if n == 0 { + // we're not going to try to guess an empty file + return Err(ErrorKind::GuessArtifact { path }.into()); + } + buf.truncate(n); + + let guess_input = GuessInput { + file_start: buf.freeze(), + source: FileSource::from_file(file.into_std().await, path.clone()), + }; + let guess_input = + match Self::guess_measurement_corpus(guess_input).await? { + ControlFlow::Break(input) => return Ok(input), + ControlFlow::Continue(guess_input) => guess_input, + }; + let guess_input = match Self::guess_hubris_archive(guess_input).await? { + ControlFlow::Break(input) => return Ok(input), + ControlFlow::Continue(guess_input) => guess_input, + }; + match Self::guess_zone_image(guess_input)? { + ControlFlow::Break(input) => Ok(input), + ControlFlow::Continue(_) => { + Err(ErrorKind::GuessArtifact { path }.into()) + } + } + } +} diff --git a/lib/src/edit/hubris_archive.rs b/lib/src/edit/hubris_archive.rs new file mode 100644 index 0000000..178ef6e --- /dev/null +++ b/lib/src/edit/hubris_archive.rs @@ -0,0 +1,237 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::ops::ControlFlow; + +use hubtools::Caboose; +use hubtools::CabooseBuilder; +use hubtools::HubrisArchiveBuilder; +use tufaceous_artifact::ArtifactVersion; +use tufaceous_artifact::ReadCabooseError; +use tufaceous_artifact::RotBootloaderTags; +use tufaceous_artifact::RotKeyTableHash; +use tufaceous_artifact::RotSlot; +use tufaceous_artifact::RotTags; +use tufaceous_artifact::SpTags; + +use crate::edit::guess::GuessInput; +use crate::edit::guess::GuessResult; +use crate::edit::input::Input; +use crate::edit::source::BytesSource; +use crate::edit::source::FileSource; +use crate::edit::source::TargetSource; +use crate::error::Error; +use crate::error::ErrorKind; +use crate::error::try_path; + +impl Input> { + pub(crate) async fn rot_archive( + mut source: FileSource, + caboose: Option, + slot: RotSlot, + ) -> Result { + let caboose = match caboose { + Some(caboose) => caboose, + None => source.read_hubris_caboose().await?, + }; + let tags = try_path!( + RotTags::from_caboose(&caboose, slot), + ReadCaboose, + source.path() + ); + let version = try_path!( + tag_helper(caboose.version(), "VERS"), + ReadCaboose, + source.path() + ) + .parse()?; + Ok(Self::Rot { source: source.into(), tags, version }) + } + + pub(crate) async fn rot_bootloader_archive( + mut source: FileSource, + caboose: Option, + ) -> Result { + let caboose = match caboose { + Some(caboose) => caboose, + None => source.read_hubris_caboose().await?, + }; + let tags = try_path!( + RotBootloaderTags::from_caboose(&caboose), + ReadCaboose, + source.path() + ); + let version = try_path!( + tag_helper(caboose.version(), "VERS"), + ReadCaboose, + source.path() + ); + Ok(Self::RotBootloader { + source: source.into(), + tags, + version: version.parse()?, + }) + } + + pub(crate) async fn sp_archive( + mut source: FileSource, + caboose: Option, + ) -> Result { + let caboose = match caboose { + Some(caboose) => caboose, + None => source.read_hubris_caboose().await?, + }; + let tags = try_path!( + SpTags::from_caboose(&caboose), + ReadCaboose, + source.path() + ); + let name = try_path!( + tag_helper(caboose.name(), "NAME"), + ReadCaboose, + source.path() + ); + let version = try_path!( + tag_helper(caboose.version(), "VERS"), + ReadCaboose, + source.path() + ); + Ok(Self::Sp { + source: source.into(), + tags, + name, + version: version.parse()?, + }) + } + + pub(crate) async fn guess_hubris_archive( + mut input: GuessInput, + ) -> GuessResult { + if !input.file_start.starts_with(b"PK\x03\x04") { + return Ok(ControlFlow::Continue(input)); + } + let archive = match input.source.read_hubris_archive().await { + Ok(archive) => archive, + Err(err) => { + if matches!(*err, ErrorKind::ReadHubrisArchive { .. }) { + // seemingly not a valid Hubris archive, keep guessing + return Ok(ControlFlow::Continue(input)); + } + // otherwise, probably an IO error + return Err(err); + } + }; + let Ok(caboose) = archive.read_caboose() else { + // seemingly not a valid Hubris archive, keep guessing + return Ok(ControlFlow::Continue(input)); + }; + // HACK: We are reading the `image-name` file in the archive, which + // appears to be "a" or "b" if it's an RoT image, "default" if it's + // an SP image, and nonexistent if it's an RoT bootloader image. This + // seems fragile. Ideally this can be in the caboose someday (see + // sprot-release#74). + match archive.image_name().as_deref() { + Ok("a") => { + Self::rot_archive(input.source, Some(caboose), RotSlot::A).await + } + Ok("b") => { + Self::rot_archive(input.source, Some(caboose), RotSlot::B).await + } + Ok("default") => { + Self::sp_archive(input.source, Some(caboose)).await + } + Err(hubtools::Error::MissingFile(_, _)) => { + Self::rot_bootloader_archive(input.source, Some(caboose)).await + } + _ => return Ok(ControlFlow::Continue(input)), + } + .map(ControlFlow::Break) + } +} + +impl Input { + pub(crate) fn fake_rot_archive( + tags: RotTags, + version: ArtifactVersion, + interior_version: Option<&ArtifactVersion>, + ) -> Result { + let data = CabooseData { + board: &tags.rot_board, + rkth: &tags.rot_rkth, + commit: "this-is-a-fake-rot", + version: interior_version.unwrap_or(&version), + }; + let source = data.generate_fake_archive()?; + Ok(Input::Rot { source, tags, version }) + } + + pub(crate) fn fake_rot_bootloader_archive( + tags: RotBootloaderTags, + version: ArtifactVersion, + interior_version: Option<&ArtifactVersion>, + ) -> Result { + let data = CabooseData { + board: &tags.rot_board, + rkth: &tags.rot_rkth, + commit: "this-is-a-fake-rot-bootloader", + version: interior_version.unwrap_or(&version), + }; + let source = data.generate_fake_archive()?; + Ok(Input::RotBootloader { source, tags, version }) + } + + pub(crate) fn fake_sp_archive( + tags: SpTags, + version: ArtifactVersion, + interior_version: Option<&ArtifactVersion>, + ) -> Result { + let data = CabooseData { + board: &tags.sp_board, + rkth: &None, + commit: "this-is-a-fake-sp", + version: interior_version.unwrap_or(&version), + }; + let source = data.generate_fake_archive()?; + Ok(Input::Sp { source, name: tags.sp_board.clone(), tags, version }) + } +} + +struct CabooseData<'a> { + board: &'a str, + rkth: &'a Option, + commit: &'static str, + version: &'a ArtifactVersion, +} + +impl CabooseData<'_> { + fn generate_fake_archive(self) -> Result { + let mut builder = CabooseBuilder::default() + .board(self.board) + .name(self.board) + .git_commit(self.commit) + .version(self.version.to_string()); + if let Some(rkth) = self.rkth { + builder = builder.sign(rkth.as_str()); + } + let caboose = builder.build(); + + let mut builder = HubrisArchiveBuilder::with_fake_image(); + builder + .write_caboose(caboose.as_slice()) + .map_err(ErrorKind::GenerateFakeHubrisArchive)?; + let vec = builder + .build_to_vec() + .map_err(ErrorKind::GenerateFakeHubrisArchive)?; + Ok(BytesSource::new(vec)) + } +} + +fn tag_helper( + value: Result<&[u8], hubtools::CabooseError>, + tag: &'static str, +) -> Result { + let s = std::str::from_utf8(value?) + .map_err(|source| ReadCabooseError::Utf8 { tag, source })?; + Ok(s.to_owned()) +} diff --git a/lib/src/edit/input.rs b/lib/src/edit/input.rs new file mode 100644 index 0000000..e982f1d --- /dev/null +++ b/lib/src/edit/input.rs @@ -0,0 +1,251 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Indirection layer for logical sets of artifacts/targets. +//! +//! In the usual case, a file in a repository ("target" in TUF terms) is an +//! artifact that can be thought of as logically independent. The exception to +//! this general rule is OS images, which logically couple several phase 1 ROMs, +//! a phase 2 ZFS image, and extra non-artifact targets that are kept around for +//! archival purposes. +//! +//! An [`Input`] is a logical set of targets and their metadata, which can be +//! cheaply converted into a set of [`Output`]s. Each `Output` is the +//! [source data][crate::edit::source], the target name within the repository, +//! and if the target is an artifact, its version and tags. +//! +//! This indirection exists to make the behavior in the rest of the editor +//! module more consistent, as well as provide a single place where the +//! generated target names for artifacts are defined. This keeps target names +//! consistent across all real and fake repositories. Additionally, because +//! none of these methods are `async`, it is possible to generate a list of fake +//! artifacts in non-async code that will always be consistent with the set of +//! artifacts in a generated fake repository. + +use std::collections::BTreeMap; + +use camino::Utf8PathBuf; +use sha2::Digest; +use sha2::Sha256; +use tufaceous_artifact::Artifact; +use tufaceous_artifact::ArtifactHash; +use tufaceous_artifact::ArtifactVersion; +use tufaceous_artifact::KnownArtifactTags; +use tufaceous_artifact::OsBoard; +use tufaceous_artifact::OsPhase1Tags; +use tufaceous_artifact::OsPhase2Tags; +use tufaceous_artifact::OsVariant; +use tufaceous_artifact::RotBootloaderTags; +use tufaceous_artifact::RotKeyTableHash; +use tufaceous_artifact::RotTags; +use tufaceous_artifact::SpTags; +use tufaceous_artifact::ZoneTags; + +use crate::COSMO_PHASE_1_PATH; +use crate::GIMLET_PHASE_1_PATH; +use crate::PHASE_2_PATH; +use crate::edit::source::BytesSource; +use crate::error::Error; +use crate::error::ErrorKind; +use crate::schema::ArtifactSchema; + +#[derive(Debug)] +pub(crate) enum Input { + MeasurementCorpus { + source: Source, + corim_id: String, + sha256: ArtifactHash, + version: ArtifactVersion, + }, + OsImages { + cosmo_phase_1: Source, + gimlet_phase_1: Source, + phase_2: Source, + extra_targets: BTreeMap, + os_variant: OsVariant, + version: ArtifactVersion, + }, + Rot { + source: Source, + tags: RotTags, + version: ArtifactVersion, + }, + RotBootloader { + source: Source, + tags: RotBootloaderTags, + version: ArtifactVersion, + }, + Sp { + source: Source, + tags: SpTags, + name: String, + version: ArtifactVersion, + }, + Zone { + source: Source, + file_name: String, + tags: ZoneTags, + version: ArtifactVersion, + }, +} + +impl Input { + pub(crate) fn outputs(self) -> Result>, Error> { + Ok(match self { + Input::MeasurementCorpus { source, corim_id, sha256, version } => { + let target_name = + format!("measurements/{corim_id}-{sha256}.cbor",); + let tags = KnownArtifactTags::MeasurementCorpus; + vec![Output::new(target_name, version, &tags, source)?] + } + Input::OsImages { + cosmo_phase_1, + gimlet_phase_1, + phase_2, + extra_targets, + os_variant, + version, + } => { + let base = Utf8PathBuf::from(format!("os-{os_variant}")); + let mut vec = Vec::with_capacity(3 + extra_targets.len()); + vec.push(Output::new( + base.join(COSMO_PHASE_1_PATH).into(), + version.clone(), + &OsPhase1Tags { os_board: OsBoard::COSMO, os_variant } + .into(), + cosmo_phase_1, + )?); + vec.push(Output::new( + base.join(GIMLET_PHASE_1_PATH).into(), + version.clone(), + &OsPhase1Tags { os_board: OsBoard::GIMLET, os_variant } + .into(), + gimlet_phase_1, + )?); + vec.push(Output::new( + base.join(PHASE_2_PATH).into(), + version, + &OsPhase2Tags { os_variant }.into(), + phase_2, + )?); + for (file_name, source) in extra_targets { + vec.push(Output::extra( + base.join(file_name).into(), + source, + )); + } + vec + } + Input::Rot { source, tags, version } => { + let target_name = format!( + "rot/{board}-{rkth}-{version}-slot-{slot}.zip", + board = tags.rot_board, + rkth = tags + .rot_rkth + .as_ref() + .and_then(RotKeyTableHash::friendly_ca_name) + .unwrap_or("unsigned"), + slot = tags.rot_slot + ); + vec![Output::new(target_name, version, &tags.into(), source)?] + } + Input::RotBootloader { source, tags, version } => { + let target_name = format!( + "rot-bootloader/{board}-{rkth}-{version}.zip", + board = tags.rot_board, + rkth = tags + .rot_rkth + .as_ref() + .and_then(RotKeyTableHash::friendly_ca_name) + .unwrap_or("unsigned"), + ); + vec![Output::new(target_name, version, &tags.into(), source)?] + } + Input::Sp { source, tags, name, version } => { + let target_name = format!("sp/{name}-{version}.zip"); + if tags.sp_board.as_str() == name { + vec![Output::new( + target_name, + version, + &tags.into(), + source, + )?] + } else { + // This is likely a lab image. As of writing these are + // stored in the TUF repo for manufacturing but are + // explicitly ignored by the control plane, as they can + // never be used in an actual rack. The current thinking is + // that they will eventually no longer need to be in the TUF + // repo. Add these as an extra target, not an artifact. + vec![Output::extra(target_name, source)] + } + } + Input::Zone { source, file_name, tags, version } => { + let target_name = format!("zones/{file_name}"); + vec![Output::new(target_name, version, &tags.into(), source)?] + } + }) + } +} + +#[derive(Debug)] +pub(crate) struct Output { + pub(crate) target_name: String, + pub(crate) source: Source, + artifact_data: Option, +} + +#[derive(Debug)] +pub(crate) struct ArtifactData { + tags: BTreeMap, + version: ArtifactVersion, +} + +impl Output { + pub(crate) fn new( + target_name: String, + version: ArtifactVersion, + tags: &KnownArtifactTags, + source: Source, + ) -> Result { + let tags = tags.to_tags().map_err(ErrorKind::ConvertKnownTagsToMap)?; + Ok(Output { + target_name, + source, + artifact_data: Some(ArtifactData { tags, version }), + }) + } + + fn extra(target_name: String, source: Source) -> Self { + Output { target_name, source, artifact_data: None } + } + + pub(crate) fn to_artifact_schema(&self) -> Option { + let data = self.artifact_data.as_ref()?; + Some(ArtifactSchema { + target_name: self.target_name.clone(), + version: data.version.clone(), + tags: data.tags.clone(), + }) + } +} + +impl Output { + pub(crate) fn into_artifact(self) -> Option<(String, Artifact)> { + let data = self.artifact_data?; + let mut hasher = Sha256::new(); + for bytes in self.source.iter_bytes() { + hasher.update(bytes); + } + Some(( + self.target_name, + Artifact { + version: data.version, + tags: data.tags, + hash: ArtifactHash(hasher.finalize().0), + length: self.source.length(), + }, + )) + } +} diff --git a/lib/src/edit/key.rs b/lib/src/edit/key.rs new file mode 100644 index 0000000..ef256ad --- /dev/null +++ b/lib/src/edit/key.rs @@ -0,0 +1,62 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::sync::Arc; + +use aws_lc_rs::error::Unspecified; +use aws_lc_rs::rand::SecureRandom; +use aws_lc_rs::signature::Ed25519KeyPair; +use tough::async_trait; +use tough::key_source::KeySource; +use tough::schema::key::Key; +use tough::sign::Sign; + +use crate::error::Error; +use crate::error::ErrorKind; + +#[derive(Debug, Clone)] +pub struct Ed25519Key(Arc); + +impl Ed25519Key { + pub fn generate() -> Result { + Ok(Self(Arc::new( + Ed25519KeyPair::generate() + .map_err(|Unspecified| ErrorKind::Ed25519Generate)?, + ))) + } +} + +#[async_trait] +impl Sign for Ed25519Key { + fn tuf_key(&self) -> Key { + self.0.tuf_key() + } + + async fn sign( + &self, + msg: &[u8], + rng: &(dyn SecureRandom + Sync), + ) -> Result, Box> + { + Sign::sign(self.0.as_ref(), msg, rng).await + } +} + +#[async_trait] +impl KeySource for Ed25519Key { + async fn as_sign( + &self, + ) -> Result, Box> + { + Ok(Box::new(self.clone())) + } + + async fn write( + &self, + _value: &str, + _key_id_hex: &str, + ) -> Result<(), Box> { + Ok(()) + } +} diff --git a/lib/src/edit/measurement_corpus.rs b/lib/src/edit/measurement_corpus.rs new file mode 100644 index 0000000..59ff511 --- /dev/null +++ b/lib/src/edit/measurement_corpus.rs @@ -0,0 +1,147 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::ops::ControlFlow; + +use rats_corim::Corim; +use rats_corim::CorimBuilder; +use sha2::Digest; +use sha2::Sha256; +use tufaceous_artifact::ArtifactHash; +use tufaceous_artifact::ArtifactVersion; + +use crate::edit::guess::GuessInput; +use crate::edit::guess::GuessResult; +use crate::edit::input::Input; +use crate::edit::source::BytesSource; +use crate::edit::source::FileSource; +use crate::edit::source::TargetSource; +use crate::error::Error; +use crate::error::ErrorKind; +use crate::error::try_path; + +impl Input> { + pub(crate) async fn measurement_corpus( + mut source: FileSource, + corim: Option, + ) -> Result { + let corim = if let Some(corim) = corim { + corim + } else { + let v = source.read_to_end().await?; + try_path!(Corim::from_bytes(v.as_slice()), ReadCorim, source.path()) + }; + let sha256 = source.sha256().await?; + let version = + try_path!(corim.get_version(), ReadCorim, source.path()).parse()?; + Ok(Self::MeasurementCorpus { + source: source.into(), + corim_id: corim.id, + sha256, + version, + }) + } + + pub(crate) async fn guess_measurement_corpus( + input: GuessInput, + ) -> GuessResult { + if !matches!(input.file_start[0], 0xa0..=0xbf /* CBOR map */) { + return Ok(ControlFlow::Continue(input)); + } + let corim = + match ciborium::from_reader::(&mut &*input.file_start) { + Ok(corim) => Some(corim), + Err(ciborium::de::Error::Io(source)) => { + if source.kind() == std::io::ErrorKind::UnexpectedEof { + // This was plausibly a CoRIM manifest until we hit the + // end of the buffer, indicating a very high likelihood + // that if we read the entire thing it'd still be a + // CoRIM manifest. + None + } else { + return Err(ErrorKind::ReadFile { + source, + path: Some(input.source.path().to_owned()), + } + .into()); + } + } + Err(_) => { + return Ok(ControlFlow::Continue(input)); + } + }; + Self::measurement_corpus(input.source, corim) + .await + .map(ControlFlow::Break) + } +} + +impl Input { + pub(crate) fn fake_measurement_corpus( + hashes: usize, + version: ArtifactVersion, + interior_version: Option<&ArtifactVersion>, + ) -> Result { + let mut builder = CorimBuilder::new(); + builder.vendor("fake-vendor".to_string()); + builder.id("fake-measurement-id".to_string()); + builder.tag_id("fake-tag-id".to_string()); + builder.version(interior_version.unwrap_or(&version).to_string()); + for i in 0..hashes { + builder.add_hash(format!("layer{i}"), 10, vec![0; 32]); + } + let corim = builder + .build() + .map_err(ErrorKind::GenerateFakeMeasurementCorpus)?; + let bytes = corim + .to_vec() + .map_err(ErrorKind::SerializeFakeMeasurementCorpus)?; + Ok(Input::MeasurementCorpus { + corim_id: corim.id, + sha256: ArtifactHash(Sha256::digest(&bytes).0), + source: BytesSource::new(bytes), + version, + }) + } +} + +#[cfg(test)] +mod tests { + use camino::Utf8Path; + use futures_util::TryStreamExt; + use futures_util::pin_mut; + + use crate::edit::guess::GuessInput; + use crate::edit::input::Input; + use crate::edit::source::FileSource; + use crate::error::Error; + + #[tokio::test] + async fn guess_partial_input() -> Result<(), Error> { + const TRUNCATE_LEN: usize = 16; + + let path = Utf8Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/data/corim-all-sp-v1.0.55.cbor"); + let metadata = tokio::fs::metadata(&path).await.unwrap(); + assert!(usize::try_from(metadata.len()).unwrap() > TRUNCATE_LEN); + + let source = FileSource::open(path).await?; + // Read the first chunk from the stream, but truncate it to ensure + // we hit the match arm where we correctly guess that this was an + // incomplete read. + let file_start = { + let stream = source.stream(); + pin_mut!(stream); + stream.try_next().await?.unwrap().split_to(TRUNCATE_LEN) + }; + let input = + Input::guess_measurement_corpus(GuessInput { file_start, source }) + .await? + .break_value() + .unwrap(); + assert!(matches!(input, Input::MeasurementCorpus { .. })); + + Ok(()) + } +} diff --git a/lib/src/edit/mod.rs b/lib/src/edit/mod.rs new file mode 100644 index 0000000..f66d762 --- /dev/null +++ b/lib/src/edit/mod.rs @@ -0,0 +1,27 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +mod editor; +mod fake; +mod guess; +mod hubris_archive; +mod input; +mod key; +mod measurement_corpus; +mod os_images; +mod root; +mod sign; +mod source; +mod zone_image; + +pub use editor::*; +pub use fake::*; +pub use key::*; +pub use root::*; +pub use sign::*; + +const KIB: u64 = 1024; +const MIB: u64 = 1024 * KIB; +#[expect(clippy::unreadable_literal)] +const OXIDE_BOOT_MAGIC: [u8; 4] = 0x1DEB0075_u32.to_le_bytes(); diff --git a/lib/src/edit/os_images.rs b/lib/src/edit/os_images.rs new file mode 100644 index 0000000..4c3a01c --- /dev/null +++ b/lib/src/edit/os_images.rs @@ -0,0 +1,282 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::BTreeMap; +use std::path::Path; + +use bytes::BufMut; +use bytes::BytesMut; +use camino::Utf8Path; +use camino::Utf8PathBuf; +use flate2::read::GzDecoder; +use futures_util::TryStreamExt; +use tokio::fs::File; +use tokio::io::AsyncReadExt; +use tufaceous_artifact::ArtifactVersion; +use tufaceous_artifact::OsVariant; + +use crate::COSMO_PHASE_1_PATH; +use crate::GIMLET_PHASE_1_PATH; +use crate::PHASE_2_PATH; +use crate::edit::KIB; +use crate::edit::MIB; +use crate::edit::OXIDE_BOOT_MAGIC; +use crate::edit::input::Input; +use crate::edit::source::BytesSource; +use crate::edit::source::FileSource; +use crate::edit::source::TargetSource; +use crate::error::Error; +use crate::error::ErrorKind; +use crate::error::try_path; + +impl Input> { + pub(crate) async fn os_images( + os_variant: OsVariant, + dir: &Utf8Path, + phase_2: Option, + version: ArtifactVersion, + ) -> Result { + let cosmo_phase_1 = + FileSource::open(dir.join(COSMO_PHASE_1_PATH)).await?.into(); + let gimlet_phase_1 = + FileSource::open(dir.join(GIMLET_PHASE_1_PATH)).await?.into(); + let phase_2 = match phase_2 { + Some(phase_2) => phase_2, + None => FileSource::open(dir.join(PHASE_2_PATH)).await?, + }; + + // If `os.tar.gz` is present, read the `image/*.txt` files from it + // as extra sources. Stop once we get to `zfs.img`, which comes after + // the additional metadata. (If a GzDecoder was seekable we could + // conceivably skip past any files we don't want to read, but alas.) + // https://github.com/oxidecomputer/helios/blob/f145457f6ccb13a139b8d93408b9b4de5db57bd6/tools/helios-build/src/main.rs#L1881-L1885 + let tarball_path = dir.join("os.tar.gz"); + let mut extra_targets = match tokio::fs::File::open(&tarball_path).await + { + Ok(file) => { + let file = file.into_std().await; + tokio::task::spawn_blocking(move || { + read_os_tarball_metadata_blocking(file, tarball_path) + }) + .await?? + } + Err(source) if source.kind() == std::io::ErrorKind::NotFound => { + // We might be reading from an unpacked Tufaceous archive; look + // for any `*.txt` files and include them. + let mut extra_targets = BTreeMap::new(); + let mut read_dir = crate::util::read_dir(dir.into()).await?; + while let Some(entry) = read_dir.try_next().await? { + if entry.path().extension() != Some("txt") { + continue; + } + let file_name = entry.file_name().to_owned(); + let source = FileSource::open(entry.into_path()).await?; + extra_targets.insert(file_name, source.into()); + } + extra_targets + } + Err(source) => { + return Err(ErrorKind::OpenFile { + source, + path: Some(tarball_path), + } + .into()); + } + }; + for path in ["unix.z", "cpio.z"] { + if let Some(source) = FileSource::try_open(dir.join(path)).await? { + extra_targets.insert(path.into(), source.into()); + } + } + + Ok(Self::OsImages { + cosmo_phase_1, + gimlet_phase_1, + phase_2: phase_2.into(), + extra_targets, + os_variant, + version, + }) + } + + pub(crate) async fn guess_os_images( + path: &Utf8Path, + version: &ArtifactVersion, + ) -> Option> { + let phase_2_path = path.join(PHASE_2_PATH); + let mut file = match File::open(&phase_2_path).await { + Ok(file) => file, + Err(source) => { + if matches!( + source.kind(), + std::io::ErrorKind::NotFound + | std::io::ErrorKind::NotADirectory + ) { + return None; + } + return Some(Err(ErrorKind::OpenFile { + source, + path: Some(phase_2_path), + } + .into())); + } + }; + // Read the header block from the image and guess whether it's a + // recovery image based on the image name. + let mut buf = [0; 4096]; + if let Err(source) = file.read_exact(&mut buf).await { + if source.kind() == std::io::ErrorKind::UnexpectedEof { + // The read was fine, the file is just too small to be a real + // OS image. + return None; + } + return Some(Err(ErrorKind::ReadFile { + source, + path: Some(phase_2_path), + } + .into())); + } + if !buf.starts_with(&OXIDE_BOOT_MAGIC) { + return None; + } + // see https://github.com/oxidecomputer/boot-image-tools/blob/main/src/diskimage.rs + let image_name = &buf[200..328]; + let variant = if image_name.starts_with(b"recovery") { + OsVariant::Recovery + } else { + OsVariant::Host + }; + + let phase_2 = + FileSource::from_file(file.into_std().await, phase_2_path); + Some( + Self::os_images(variant, path, Some(phase_2), version.clone()) + .await, + ) + } +} + +fn read_os_tarball_metadata_blocking( + file: std::fs::File, + tarball_path: Utf8PathBuf, +) -> Result>, Error> { + let mut metadata = BTreeMap::new(); + let mut archive = tar::Archive::new(GzDecoder::new(file)); + for entry in try_path!(archive.entries(), ReadFile, tarball_path) { + let mut entry = try_path!(entry, ReadFile, tarball_path); + if entry.header().entry_type() != tar::EntryType::Regular { + continue; + } + let path = try_path!(entry.path(), ReadFile, tarball_path); + let Some(parent) = path.parent() else { continue }; + if parent != "image" { + continue; + } + let Some(extension) = path.extension() else { continue }; + if extension != "txt" { + break; + } + let file_name = Path::new( + path.file_name() + .expect("a path with an extension must have a file name"), + ); + let file_name = try_path!( + Utf8PathBuf::try_from(file_name.to_owned()) + .map_err(camino::FromPathBufError::into_io_error), + ReadFile, + tarball_path + ); + let mut writer = BytesMut::new().writer(); + try_path!( + std::io::copy(&mut entry, &mut writer), + ReadFile, + tarball_path + ); + let source = BytesSource::new(writer.into_inner().freeze()); + metadata.insert(file_name.into(), source.into()); + } + Ok(metadata) +} + +impl Input { + pub(crate) fn fake_os_images( + os_variant: OsVariant, + version: ArtifactVersion, + interior_version: Option<&ArtifactVersion>, + ) -> Self { + let interior_version = interior_version.unwrap_or(&version); + let cosmo_phase_1 = BytesSource::fake_padded( + format!( + "cosmo {os_variant} OS phase 1 image version {interior_version}\n" + ), + MIB, + ); + let gimlet_phase_1 = BytesSource::fake_padded( + format!( + "gimlet {os_variant} OS phase 1 image version {interior_version}\n" + ), + MIB, + ); + + let mut phase_2_bytes = BytesMut::with_capacity(4096); + phase_2_bytes.put(OXIDE_BOOT_MAGIC.as_slice()); // uint32_t odh_magic; + phase_2_bytes.put_u32_le(2); // uint32_t odh_version; + // The only defined ODH_FLAG is: + // #define ODH_FLAG_COMPRESSED 0x1 + // but we are not compressing any fake images. (Normally the host image + // is "uncompressed" -- a raw ZFS image with compressed contents -- and + // the recovery image is compressed.) + phase_2_bytes.put_u64_le(0); // uint64_t odh_flags; + phase_2_bytes.put_u64_le(4 * MIB); // uint64_t odh_data_size; + phase_2_bytes.put_u64_le(4 * MIB); // uint64_t odh_image_size; + phase_2_bytes.put_u64_le(1 << 32); // uint64_t odh_target_size; + // #define OXBOOT_CSUMLEN_SHA256 32 + // uint8_t odh_sha256[OXBOOT_CSUMLEN_SHA256]; + phase_2_bytes.put( + // head -c $((4 * 1024 * 1024)) "ci".as_bytes(), + OsVariant::Recovery => "recovery".as_bytes(), + }); + phase_2_bytes.put(" fake123/789fake 1986-12-28 01:23".as_bytes()); + // rest of header is zeroes, which will be written out by `fake_padded` + let phase_2 = BytesSource::fake_padded(phase_2_bytes, 4096 + 4 * MIB); + + let mut extra_targets = BTreeMap::new(); + extra_targets.insert( + String::from("unix.z"), + BytesSource::fake_padded( + format!("{os_variant} OS unix.z version {interior_version}\n"), + 64 * KIB, + ), + ); + extra_targets.insert( + String::from("cpio.z"), + BytesSource::fake_padded( + format!("{os_variant} OS cpio.z version {interior_version}\n"), + 256 * KIB, + ), + ); + + Self::OsImages { + cosmo_phase_1, + gimlet_phase_1, + phase_2, + extra_targets, + os_variant, + version, + } + } +} diff --git a/lib/src/edit/root.rs b/lib/src/edit/root.rs new file mode 100644 index 0000000..da35189 --- /dev/null +++ b/lib/src/edit/root.rs @@ -0,0 +1,140 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::HashMap; +use std::fmt::Display; +use std::num::NonZero; + +use aws_lc_rs::rand::SystemRandom; +use chrono::DateTime; +use chrono::Utc; +use serde::Deserialize; +use serde::Serialize; +use tough::editor::signed::SignedRole; +use tough::key_source::KeySource; +use tough::schema; +use tough::schema::KeyHolder; +use tough::schema::RoleKeys; +use tough::schema::RoleType; +use tough::schema::Signed; + +use crate::error::Error; +use crate::error::ErrorKind; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Root { + inner: Signed, + buffer: String, +} + +impl Root { + pub async fn generate( + keys: &[Box], + expires: DateTime, + ) -> Result { + let inner = generate_root(keys, expires).await?.signed().clone(); + let mut buffer = serde_json::to_string_pretty(&inner) + .map_err(ErrorKind::SerializeRoot)?; + buffer.push('\n'); + Ok(Self { inner, buffer }) + } + + pub fn verify_self_signed(&self) -> Result<(), Error> { + self.inner + .signed + .verify_role(&self.inner) + .map_err(ErrorKind::RoleVerify)?; + Ok(()) + } + + pub fn as_str(&self) -> &str { + &self.buffer + } +} + +impl Display for Root { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.buffer) + } +} + +impl Serialize for Root { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + self.inner.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for Root { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let inner = >::deserialize(deserializer)?; + let mut buffer = serde_json::to_string_pretty(&inner) + .map_err(serde::de::Error::custom)?; + buffer.push('\n'); + Ok(Self { inner, buffer }) + } +} + +pub(crate) async fn generate_root( + keys: &[Box], + expires: DateTime, +) -> Result, Error> { + let mut root = schema::Root { + spec_version: "1.0.0".into(), + consistent_snapshot: false, + version: NonZero::::MIN, + expires, + keys: HashMap::new(), + roles: HashMap::new(), + _extra: HashMap::new(), + }; + + for key in keys { + let key = key.as_sign().await.map_err(ErrorKind::ToughKey)?.tuf_key(); + root.keys.insert(key.key_id().map_err(ErrorKind::KeyId)?, key); + } + for role_type in [ + RoleType::Root, + RoleType::Snapshot, + RoleType::Targets, + RoleType::Timestamp, + ] { + root.roles.insert( + role_type, + RoleKeys { + keyids: root.keys.keys().cloned().collect(), + threshold: NonZero::::MIN, + _extra: HashMap::new(), + }, + ); + } + + Ok(SignedRole::new( + root.clone(), + &KeyHolder::Root(root), + keys, + &SystemRandom::new(), + ) + .await?) +} + +#[cfg(test)] +mod tests { + use chrono::Utc; + + use crate::edit::Ed25519Key; + use crate::edit::Root; + + #[tokio::test] + async fn generate_verify() { + let key = Ed25519Key::generate().unwrap(); + let root = Root::generate(&[Box::new(key)], Utc::now()).await.unwrap(); + root.verify_self_signed().unwrap(); + } +} diff --git a/lib/src/edit/sign.rs b/lib/src/edit/sign.rs new file mode 100644 index 0000000..9188312 --- /dev/null +++ b/lib/src/edit/sign.rs @@ -0,0 +1,489 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::BTreeMap; +use std::collections::HashMap; +use std::io::Write; +use std::num::NonZero; +use std::time::Duration; + +use atomicwrites::AtomicFile; +use atomicwrites::OverwriteBehavior; +use bytes::Bytes; +use camino::Utf8Path; +use camino::Utf8PathBuf; +use chrono::DateTime; +use chrono::SubsecRound; +use chrono::Utc; +use flate2::Compression; +use flate2::write::DeflateEncoder; +use futures_util::StreamExt; +use futures_util::TryStreamExt; +use rawzip::CompressionMethod; +use rawzip::ZipArchiveWriter; +use rawzip::time::UtcDateTime; +use tokio::sync::mpsc; +use tokio::task::JoinHandle; +use tough::key_source::KeySource; +use tough::schema::Root; +use tough::schema::Signed; + +use crate::edit::Ed25519Key; +use crate::edit::OXIDE_BOOT_MAGIC; +use crate::edit::source::FileSource; +use crate::edit::source::Target; +use crate::edit::source::TargetSource; +use crate::error::Error; +use crate::error::ErrorKind; +use crate::error::try_path; + +pub(crate) const DEFAULT_VALIDITY: Duration = + Duration::from_secs(60 * 60 * 24 * 7 /* 1 week */); + +#[derive(Debug)] +#[must_use] +pub struct UnsignedRepository<'a> { + targets: BTreeMap>, + root: Option, + keys: Vec>, + snapshot_version: NonZero, + snapshot_expires: DateTime, + targets_version: NonZero, + targets_expires: DateTime, + timestamp_version: NonZero, + timestamp_expires: DateTime, +} + +#[derive(Debug)] +enum RequestedRoot { + Root(Vec), + Generate, +} + +impl<'a> UnsignedRepository<'a> { + pub(crate) fn from_targets(targets: BTreeMap>) -> Self { + let now = Utc::now().trunc_subsecs(0); + let version = NonZero::try_from(now.timestamp()) + .and_then(NonZero::::try_from) + .unwrap_or(NonZero::::MIN); + let expires = now + DEFAULT_VALIDITY; + + Self { + targets, + root: None, + keys: Vec::new(), + snapshot_version: version, + snapshot_expires: expires, + targets_version: version, + targets_expires: expires, + timestamp_version: version, + timestamp_expires: expires, + } + } + + pub fn root(self, root: impl AsRef<[u8]>) -> Self { + Self { root: Some(RequestedRoot::Root(root.as_ref().to_vec())), ..self } + } + + pub fn key(mut self, key: impl KeySource + 'static) -> Self { + self.keys.push(Box::new(key)); + self + } + + pub fn generate_root(self) -> Self { + Self { root: Some(RequestedRoot::Generate), ..self } + } + + pub fn snapshot_version(self, snapshot_version: NonZero) -> Self { + Self { snapshot_version, ..self } + } + + pub fn snapshot_expires(self, snapshot_expires: DateTime) -> Self { + Self { snapshot_expires, ..self } + } + + pub fn targets_version(self, targets_version: NonZero) -> Self { + Self { targets_version, ..self } + } + + pub fn targets_expires(self, targets_expires: DateTime) -> Self { + Self { targets_expires, ..self } + } + + pub fn timestamp_version(self, timestamp_version: NonZero) -> Self { + Self { timestamp_version, ..self } + } + + pub fn timestamp_expires(self, timestamp_expires: DateTime) -> Self { + Self { timestamp_expires, ..self } + } + + pub async fn sign(mut self) -> Result, Error> { + let (root, consistent_snapshot) = match self.root { + Some(RequestedRoot::Root(root)) => { + let parsed_root: Signed = + serde_json::from_slice(&root) + .map_err(ErrorKind::ParseSigningRoot)?; + (root, parsed_root.signed.consistent_snapshot) + } + Some(RequestedRoot::Generate) => { + if self.keys.is_empty() { + self.keys.push(Box::new(Ed25519Key::generate()?)); + } + let expires = self + .snapshot_expires + .min(self.targets_expires) + .min(self.timestamp_expires); + let root = + crate::edit::generate_root(&self.keys, expires).await?; + ( + root.buffer().clone(), + root.signed().signed.consistent_snapshot, + ) + } + None => return Err(ErrorKind::NoSigningRoot.into()), + }; + + let tempdir = tokio::task::spawn_blocking(camino_tempfile::tempdir) + .await? + .map_err(ErrorKind::CreateTempDir)?; + + // tough's RepositoryEditor can only be constructed with a path to a + // root role, which is mildly silly; we need to write the root out to a + // temporary directory so it can read it back in again. + let root_path = tempdir.path().join("root.json"); + try_path!( + tokio::fs::write(&root_path, &root).await, + WriteFile, + root_path + ); + let mut editor = + tough::editor::RepositoryEditor::new(&root_path).await?; + editor + .snapshot_version(self.snapshot_version) + .snapshot_expires(self.snapshot_expires) + .targets_version(self.targets_version)? + .targets_expires(self.targets_expires)? + .timestamp_version(self.timestamp_version) + .timestamp_expires(self.timestamp_expires); + + let mut sources = BTreeMap::new(); + for (target_name, Target { length, sha256, source }) in self.targets { + let path = if consistent_snapshot { + format!("{}.{target_name}", hex::encode(sha256)) + } else { + target_name.clone() + }; + let hashes = tough::schema::Hashes { + sha256: sha256.0.to_vec().into(), + _extra: HashMap::new(), + }; + let target = tough::schema::Target { + length, + hashes, + custom: HashMap::new(), + _extra: HashMap::new(), + }; + editor.add_target(target_name.as_str(), target)?; + + sources.insert((FilePrefix::Targets, path), source); + } + + let signed = editor.sign(&self.keys).await?; + // We can't read the metadata directly, so we again need to write it to + // our temporary directory and then read the data back in. + let metadata_dir = tempdir.path().join("metadata"); + signed.write(&metadata_dir).await?; + let mut read_dir = crate::util::read_dir(metadata_dir).await?; + while let Some(entry) = read_dir.try_next().await? { + // This is opening a file within the tempdir that is about to be + // deleted. This is expected to be fine. + let source = + FileSource::open(entry.path().to_owned()).await?.into(); + sources.insert( + (FilePrefix::Metadata, entry.file_name().into()), + source, + ); + } + + Ok(SignedRepository { root, sources }) + } +} + +#[derive(Debug)] +pub struct SignedRepository<'a> { + root: Vec, + sources: BTreeMap<(FilePrefix, String), TargetSource<'a>>, +} + +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] +enum FilePrefix { + Metadata, + Targets, +} + +impl SignedRepository<'_> { + pub fn root(&self) -> &[u8] { + &self.root + } + + pub async fn write_zip( + &self, + writer: W, + modification_time: DateTime, + ) -> Result { + let (tx, rx) = mpsc::channel(1); + let task = tokio::task::spawn_blocking(move || { + blocking_write_task(writer, rx, modification_time) + }); + self.write_zip_impl(tx, task, None).await + } + + pub async fn write_zip_file( + &self, + path: impl AsRef, + modification_time: DateTime, + ) -> Result<(), Error> { + let path = path.as_ref(); + let file = AtomicFile::new(path, OverwriteBehavior::AllowOverwrite); + let (tx, rx) = mpsc::channel(1); + let task = tokio::task::spawn_blocking(move || { + file.write(|file| { + blocking_write_task(file, rx, modification_time).map(|_| ()) + })?; + Ok(()) + }); + self.write_zip_impl(tx, task, Some(path)).await + } + + async fn write_zip_impl( + &self, + tx: mpsc::Sender, + task: JoinHandle>, + archive_path: Option<&Utf8Path>, + ) -> Result { + 'outer: for ((prefix, name), source) in &self.sources { + let prefix = Utf8Path::new(match prefix { + FilePrefix::Metadata => "repo/metadata", + FilePrefix::Targets => "repo/targets", + }); + let mut stream = source.stream(); + let first_chunk = stream.try_next().await?; + let compression = first_chunk + .as_deref() + .map_or(Compression::none(), deflate_heuristic); + + let Ok(()) = tx + .send(ZipWriterMessage::StartFile { + name: prefix.join(name), + compression, + }) + .await + else { + break 'outer; + }; + + let mut stream = + futures_util::stream::iter(first_chunk).map(Ok).chain(stream); + while let Some(chunk) = stream.try_next().await? { + let Ok(()) = + tx.send(ZipWriterMessage::WriteFileBytes(chunk)).await + else { + break 'outer; + }; + } + let Ok(()) = tx.send(ZipWriterMessage::FinishFile).await else { + break 'outer; + }; + } + tx.send(ZipWriterMessage::FinishArchive).await.ok(); + task.await?.map_err(|error| { + match error { + ZipWriterError::Zip(source) => ErrorKind::WriteZip { + source, + archive_path: archive_path.map(Utf8Path::to_owned), + }, + ZipWriterError::StateMachine(reason) => { + ErrorKind::WriteZipStateMachine { reason } + } + } + .into() + }) + } +} + +fn deflate_heuristic(buf: &[u8]) -> Compression { + if buf.is_empty() { + // probably an empty file! + Compression::none() + } else if buf.starts_with(b"\x1f\x8b") { + // gzip, e.g. illumos zone tarball + Compression::none() + } else if buf.starts_with(b"\x78") + && let Some([x, y]) = &buf.get(..2) + && u16::from_be_bytes([*x, *y]) % 31 == 0 + { + // zlib + if y & 0xc0 == 0 { + // buf is zlib-compressed at level 0, which is not compressed at + // all, so we should compress this + Compression::best() + } else { + Compression::none() + } + } else if buf.starts_with(b"PK\x03\x04") { + // ZIP archive, e.g. hubris archive. not necessarily compressed based on + // this heuristic alone but in our case it's very likely. + Compression::none() + } else if buf.starts_with(&OXIDE_BOOT_MAGIC) { + // oxide phase 2 OS image. images are zlib-compressed after the header + // if the least-significant bit of the flags starting at byte 8 is set: + if buf.get(8).is_some_and(|b| *b & 0x1 == 0x1) { + Compression::none() + } else { + // not zlib-compressed, but the ZFS dataset likely has compression + // enabled. use fast compression to deflate unused blocks + Compression::fast() + } + } else { + Compression::best() + } +} + +enum ZipWriterMessage { + FinishArchive, + StartFile { name: Utf8PathBuf, compression: Compression }, + WriteFileBytes(Bytes), + FinishFile, +} + +enum ZipWriterError { + Zip(rawzip::Error), + StateMachine(&'static str), +} + +impl From for ZipWriterError { + fn from(error: rawzip::Error) -> Self { + ZipWriterError::Zip(error) + } +} + +impl From for ZipWriterError { + fn from(error: std::io::Error) -> Self { + ZipWriterError::Zip(error.into()) + } +} + +impl From> for ZipWriterError { + fn from(error: atomicwrites::Error) -> Self { + match error { + atomicwrites::Error::User(error) => error, + atomicwrites::Error::Internal(error) => error.into(), + } + } +} + +fn blocking_write_task( + writer: W, + mut rx: mpsc::Receiver, + modification_time: DateTime, +) -> Result { + let mut archive = ZipArchiveWriter::new(writer); + loop { + let message = rx.blocking_recv().ok_or( + ZipWriterError::StateMachine("unexpected end of message stream"), + )?; + let (name, compression) = match message { + ZipWriterMessage::FinishArchive => { + return Ok(archive.finish()?); + } + ZipWriterMessage::StartFile { name, compression } => { + (name, compression) + } + ZipWriterMessage::WriteFileBytes(_) + | ZipWriterMessage::FinishFile => { + return Err(ZipWriterError::StateMachine( + "expected StartFile or FinishArchive", + )); + } + }; + + let (mut entry, config) = archive + .new_file(name.as_str()) + .compression_method(if compression == Compression::none() { + CompressionMethod::Store + } else { + CompressionMethod::Deflate + }) + .last_modified(UtcDateTime::from_unix( + modification_time.timestamp(), + )) + .unix_permissions(0o644) + .start()?; + let encoder = ZipEncoder::new(&mut entry, compression); + let mut writer = config.wrap(encoder); + + loop { + let message = + rx.blocking_recv().ok_or(ZipWriterError::StateMachine( + "unexpected end of message stream", + ))?; + match message { + ZipWriterMessage::WriteFileBytes(bytes) => { + writer.write_all(&bytes)?; + } + ZipWriterMessage::FinishFile => { + let (encoder, output) = writer.finish()?; + encoder.finish()?; + entry.finish(output)?; + break; + } + ZipWriterMessage::FinishArchive + | ZipWriterMessage::StartFile { .. } => { + return Err(ZipWriterError::StateMachine( + "expected WriteFileBytes or FinishFile", + )); + } + } + } + } +} + +enum ZipEncoder { + Store(W), + Deflate(DeflateEncoder), +} + +impl ZipEncoder { + fn new(writer: W, compression: Compression) -> Self { + if compression == Compression::none() { + Self::Store(writer) + } else { + Self::Deflate(DeflateEncoder::new(writer, compression)) + } + } + + fn finish(self) -> std::io::Result { + match self { + Self::Store(writer) => Ok(writer), + Self::Deflate(encoder) => encoder.finish(), + } + } +} + +impl Write for ZipEncoder { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + match self { + ZipEncoder::Store(writer) => writer.write(buf), + ZipEncoder::Deflate(writer) => writer.write(buf), + } + } + + fn flush(&mut self) -> std::io::Result<()> { + match self { + ZipEncoder::Store(writer) => writer.flush(), + ZipEncoder::Deflate(writer) => writer.flush(), + } + } +} diff --git a/lib/src/edit/source.rs b/lib/src/edit/source.rs new file mode 100644 index 0000000..0858b6c --- /dev/null +++ b/lib/src/edit/source.rs @@ -0,0 +1,369 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Indirection layer for reading target data from varying sources. +//! +//! When generating a repository, the data for each target could come from +//! one of several places. In the normal case it comes from a file on disk +//! ([`FileSource`]). If it is particularly small or is fake it might be an +//! in-memory representation ([`BytesSource`]). If we're editing an existing +//! repository, all of the previous repository's artifacts are treated as a +//! pointer to the open repository ([`RepositorySource`]). +//! +//! [`TargetSource`] is an enum that covers all three of these possible sources, +//! and has static dispatch for reading the underlying source as a `Stream`. +//! +//! Each of the three concrete sources has an `into_target` method which returns +//! a [`Target`], which is the `TargetSource` along with the target's length and +//! SHA-256 checksum. + +use std::convert::Infallible; +use std::pin::Pin; +use std::sync::Arc; +use std::sync::Mutex; + +use bytes::Bytes; +use bytes::BytesMut; +use camino::Utf8Path; +use camino::Utf8PathBuf; +use futures_util::Stream; +use futures_util::StreamExt; +use futures_util::TryStreamExt; +use futures_util::stream; +use hubtools::Caboose; +use hubtools::RawHubrisArchive; +use rawzip::FileReader; +use rawzip::ReaderAt; +use serde::Serialize; +use sha2::Digest; +use sha2::Sha256; +use tokio::fs::File; +use tufaceous_artifact::ArtifactHash; + +use crate::Repository; +use crate::error::Error; +use crate::error::ErrorKind; +use crate::error::try_path; + +#[derive(Debug)] +pub(crate) struct Target<'a> { + pub(crate) length: u64, + pub(crate) sha256: ArtifactHash, + pub(crate) source: TargetSource<'a>, +} + +#[derive(Debug, Clone)] +pub(crate) enum TargetSource<'a> { + Bytes(BytesSource), + File(FileSource), + Repository(RepositorySource<'a>), +} + +impl TargetSource<'_> { + /// A relative indication of how computationally expensive it is to read + /// from this source, given two sources of the data. + /// + /// For example, reading from a file on disk is preferred from reading a + /// target out of an opened repository. + pub(crate) fn cost(&self) -> usize { + match self { + TargetSource::Bytes(BytesSource { fake_length: None, .. }) => 0, + TargetSource::Bytes(_) => 1, + TargetSource::File(_) => 2, + TargetSource::Repository(_) => 3, + } + } + + pub(crate) fn stream( + &self, + ) -> Pin> + Send + '_>> { + match self { + TargetSource::Bytes(source) => { + Box::pin(source.stream().err_into::()) + } + TargetSource::File(source) => Box::pin(source.stream()), + TargetSource::Repository(source) => Box::pin(source.stream()), + } + } +} + +impl From for TargetSource<'_> { + fn from(source: BytesSource) -> Self { + TargetSource::Bytes(source) + } +} + +impl From for TargetSource<'_> { + fn from(source: FileSource) -> Self { + TargetSource::File(source) + } +} + +impl<'a> From> for TargetSource<'a> { + fn from(source: RepositorySource<'a>) -> Self { + TargetSource::Repository(source) + } +} + +#[derive(Debug, Clone)] +pub(crate) struct BytesSource { + bytes: Bytes, + fake_length: Option, + sha256: Option, +} + +impl BytesSource { + pub(crate) fn new(bytes: impl Into) -> Self { + Self { bytes: bytes.into(), fake_length: None, sha256: None } + } + + pub(crate) fn json( + data: &T, + ) -> Result { + let mut s = serde_json::to_string_pretty(data)?; + s.push('\n'); + Ok(Self { bytes: s.into(), fake_length: None, sha256: None }) + } + + pub(crate) fn fake_padded(prefix: impl Into, length: u64) -> Self { + Self { bytes: prefix.into(), fake_length: Some(length), sha256: None } + } + + pub(crate) fn iter_bytes(&self) -> impl Iterator + 'static { + let mut bytes = self.bytes.clone(); + let mut remaining = match self.fake_length { + Some(length) => { + if let Ok(length) = usize::try_from(length) { + bytes.truncate(length); + } + length - usize64!(bytes.len()) + } + None => 0, + }; + std::iter::once(bytes).chain(std::iter::from_fn(move || { + static ZERO: &[u8] = &[0; 8192]; + if remaining == 0 { + None + } else { + let end = remaining.min(8192); + remaining -= end; + let end = usize::try_from(end).expect("8192 <= usize::MAX"); + Some(Bytes::from_static(&ZERO[..end])) + } + })) + } + + pub(crate) fn stream( + &self, + ) -> impl Stream> + 'static { + stream::iter(self.iter_bytes().map(Ok)) + } + + pub(crate) fn length(&self) -> u64 { + self.fake_length.unwrap_or_else(|| usize64!(self.bytes.len())) + } + + async fn sha256(&mut self) -> ArtifactHash { + if let Some(sha256) = self.sha256 { + return sha256; + } + let mut stream = self.stream(); + let mut hasher = Sha256::new(); + while let Some(Ok(bytes)) = stream.next().await { + hasher.update(&bytes); + } + *self.sha256.insert(ArtifactHash(hasher.finalize().0)) + } + + pub(crate) async fn into_target(mut self) -> Target<'static> { + Target { + length: self.length(), + sha256: self.sha256().await, + source: self.into(), + } + } +} + +#[derive(Debug, Clone)] +pub(crate) struct FileSource { + inner: Arc, +} + +#[derive(Debug)] +struct FileSourceInner { + file: FileReader, + path: Utf8PathBuf, + length_sha256: Mutex>, +} + +impl FileSource { + pub(crate) async fn open(path: Utf8PathBuf) -> Result { + let file = try_path!(File::open(&path).await, OpenFile, path); + Ok(Self::from_file(file.into_std().await, path)) + } + + pub(crate) async fn try_open( + path: Utf8PathBuf, + ) -> Result, Error> { + let file = match File::open(&path).await { + Ok(file) => file, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + return Ok(None); + } + Err(err) => try_path!(Err(err), OpenFile, path), + }; + Ok(Some(Self::from_file(file.into_std().await, path))) + } + + pub(crate) fn from_file(file: std::fs::File, path: Utf8PathBuf) -> Self { + Self { + inner: Arc::new(FileSourceInner { + file: file.into(), + path, + length_sha256: Mutex::new(None), + }), + } + } + + pub(crate) fn path(&self) -> &Utf8Path { + &self.inner.path + } + + pub(crate) async fn into_target( + mut self, + ) -> Result, Error> { + let inner = { + let guard = self.inner.length_sha256.lock().expect("poisoned"); + *guard + }; + let (length, sha256) = match inner { + Some(inner) => inner, + None => self.read_impl(None).await?, + }; + Ok(Target { length, sha256, source: self.into() }) + } + + async fn read_impl( + &mut self, + mut vec: Option<&mut Vec>, + ) -> Result<(u64, ArtifactHash), Error> { + let (length, hasher) = self + .stream() + .try_fold( + (0u64, Sha256::new()), + |(mut length, mut hasher), bytes| { + length += usize64!(bytes.len()); + hasher.update(&bytes); + if let Some(vec) = vec.as_mut() { + vec.extend_from_slice(&bytes); + } + std::future::ready(Ok((length, hasher))) + }, + ) + .await?; + let sha256 = ArtifactHash(hasher.finalize().0); + Ok({ + let mut guard = self.inner.length_sha256.lock().expect("poisoned"); + *guard.insert((length, sha256)) + }) + } + + pub(crate) async fn sha256(&mut self) -> Result { + let inner = { + let guard = self.inner.length_sha256.lock().expect("poisoned"); + *guard + }; + let (_, sha256) = match inner { + Some(inner) => inner, + None => self.read_impl(None).await?, + }; + Ok(sha256) + } + + pub(crate) async fn read_to_end(&mut self) -> Result, Error> { + let mut vec = Vec::new(); + self.read_impl(Some(&mut vec)).await?; + Ok(vec) + } + + pub(crate) async fn read_hubris_archive( + &mut self, + ) -> Result { + Ok(try_path!( + RawHubrisArchive::from_vec(self.read_to_end().await?), + ReadHubrisArchive, + &self.inner.path + )) + } + + pub(crate) async fn read_hubris_caboose( + &mut self, + ) -> Result { + Ok(try_path!( + self.read_hubris_archive().await?.read_caboose(), + ReadHubrisArchive, + &self.inner.path + )) + } + + pub(crate) fn stream(&self) -> impl Stream> { + let inner = self.inner.clone(); + crate::mpsc_stream::mpsc_stream(None, move |tx| { + let mut buf = BytesMut::with_capacity(8192); + let mut offset = 0; + loop { + if buf.capacity() == 0 { + buf.reserve(8192); + } + buf.resize(buf.capacity(), 0); + match inner.file.read_at(&mut buf, offset) { + Ok(n) => { + buf.truncate(n); + } + Err(source) => { + let err = ErrorKind::ReadFile { + source, + path: Some(inner.path.clone()), + }; + return tx.blocking_send(Err(err.into())); + } + } + + let bytes = buf.split().freeze(); + if bytes.is_empty() { + return Ok(()); + } + offset += usize64!(bytes.len()); + tx.blocking_send(Ok(bytes))?; + } + }) + } +} + +#[derive(Debug, Clone)] +pub(crate) struct RepositorySource<'a> { + pub(crate) repo: &'a Repository, + pub(crate) target_name: String, + pub(crate) length: u64, + pub(crate) sha256: Vec, +} + +impl<'a> RepositorySource<'a> { + pub(crate) fn into_target(self) -> Result, Error> { + let sha256 = match self.sha256.as_slice().try_into() { + Ok(sha256) => ArtifactHash(sha256), + Err(_) => { + return Err(ErrorKind::InvalidHashLength { + target_name: self.target_name, + } + .into()); + } + }; + Ok(Target { length: self.length, sha256, source: self.into() }) + } + + pub(crate) fn stream(&self) -> impl Stream> { + stream::once(self.repo.read_target(&self.target_name)).try_flatten() + } +} diff --git a/lib/src/edit/zone_image.rs b/lib/src/edit/zone_image.rs new file mode 100644 index 0000000..0b34e60 --- /dev/null +++ b/lib/src/edit/zone_image.rs @@ -0,0 +1,110 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::ops::ControlFlow; + +use bytes::BufMut; +use bytes::BytesMut; +use camino::Utf8PathBuf; +use flate2::Compression; +use flate2::read::GzDecoder; +use flate2::write::GzEncoder; +use tufaceous_artifact::ArtifactVersion; +use tufaceous_artifact::ZoneTags; +use tufaceous_brand_metadata::ArchiveType; +use tufaceous_brand_metadata::LayerInfo; +use tufaceous_brand_metadata::Metadata; + +use crate::edit::guess::GuessInput; +use crate::edit::guess::GuessResult; +use crate::edit::input::Input; +use crate::edit::source::BytesSource; +use crate::edit::source::FileSource; +use crate::edit::source::TargetSource; +use crate::error::Error; +use crate::error::ErrorKind; +use crate::error::try_path; + +impl Input> { + pub(crate) async fn zone_image(path: Utf8PathBuf) -> Result { + let file = + try_path!(tokio::fs::File::open(&path).await, OpenFile, path); + let file_name = path + .file_name() + .expect("a path to an opened file must have a file name") + .to_string(); + let (file, layer_info) = crate::util::read_zone_layer_info( + file.into_std().await, + path.clone(), + ) + .await?; + let source = FileSource::from_file(file, path); + Ok(Self::Zone { + source: source.into(), + file_name, + tags: ZoneTags { zone_name: layer_info.pkg }, + version: layer_info.version, + }) + } + + #[expect(clippy::unnecessary_wraps)] + pub(crate) fn guess_zone_image(input: GuessInput) -> GuessResult { + // `oxide.json` is the first file of a zone image and is relatively + // small, so it should be contained entirely within the first 4K of the + // compressed tarball. + let mut archive = tar::Archive::new(GzDecoder::new(&*input.file_start)); + // The tar crate returns std::io::Error so it's difficult to + // differentiate between "I couldn't read the file" and "the file isn't + // a tar archive", but at this point in guessing we're unlikely to have + // OS-level IO issues. + let Ok(layer_info) = Metadata::read_from_tar(&mut archive) + .and_then(Metadata::into_layer_info) + else { + return Ok(ControlFlow::Continue(input)); + }; + let Some(file_name) = input.source.path().file_name() else { + return Ok(ControlFlow::Continue(input)); + }; + let file_name = file_name.to_string(); + Ok(ControlFlow::Break(Self::Zone { + source: input.source.into(), + file_name, + tags: ZoneTags { zone_name: layer_info.pkg }, + version: layer_info.version, + })) + } +} + +impl Input { + pub(crate) fn fake_zone_image( + zone_name: String, + file_name: String, + version: ArtifactVersion, + interior_version: Option, + ) -> Result { + let mut archive = tar::Builder::new(GzEncoder::new( + BytesMut::new().writer(), + Compression::best(), + )); + let metadata = Metadata::new(ArchiveType::Layer(LayerInfo { + pkg: zone_name.clone(), + version: interior_version.unwrap_or_else(|| version.clone()), + })); + metadata + .append_to_tar(&mut archive, 0) + .map_err(ErrorKind::GenerateFakeZoneImage)?; + let bytes = archive + .into_inner() + .and_then(GzEncoder::finish) + .map_err(ErrorKind::GenerateFakeZoneImage)? + .into_inner() + .freeze(); + Ok(Input::Zone { + source: BytesSource::new(bytes), + file_name, + tags: ZoneTags { zone_name }, + version, + }) + } +} diff --git a/lib/src/error.rs b/lib/src/error.rs new file mode 100644 index 0000000..6843a72 --- /dev/null +++ b/lib/src/error.rs @@ -0,0 +1,510 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::BTreeMap; +use std::convert::Infallible; +use std::error::Error as _; +use std::fmt; +use std::fmt::Debug; +use std::fmt::Display; +use std::ops::Deref; +use std::ops::Range; + +use crate::ZipTransportError; +use camino::Utf8PathBuf; +use tough::TransportErrorKind; +use tufaceous_artifact::Artifact; +use tufaceous_artifact::DisplayTags; + +macro_rules! try_path { + ($result:expr, $kind:ident, $path:expr) => { + $crate::error::try_path!($result, $kind, path: $path) + }; + + ($result:expr, $kind:ident, $field:ident: $path:expr) => { + match $result { + Ok(value) => value, + Err(source) => { + return Err( + ErrorKind::$kind { source, $field: $path.into() }.into() + ) + } + } + }; +} +pub(crate) use try_path; + +#[derive(Debug, thiserror::Error)] +#[error(transparent)] +pub struct Error(pub Box); + +impl Deref for Error { + type Target = ErrorKind; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum ErrorKind { + #[error(transparent)] + Fetch(tough::TransportError), + #[error(transparent)] + Join(#[from] tokio::task::JoinError), + #[error(transparent)] + Tough(tough::error::Error), + #[error("error while manipulating key")] + ToughKey(#[source] Box), + + #[error( + "failed to read end of central directory of zip archive{archive_path}", + archive_path = SpacePath(archive_path), + )] + ReadZipEocd { + #[source] + source: rawzip::Error, + archive_path: Option, + }, + #[error( + "failed to read central directory file header of \ + zip archive{archive_path}", + archive_path = SpacePath(archive_path), + )] + ReadZipCdfh { + #[source] + source: rawzip::Error, + archive_path: Option, + }, + #[error( + "failed to read local file header of zip archive{archive_path}", + archive_path = SpacePath(archive_path), + )] + ReadZipLocal { + #[source] + source: rawzip::Error, + archive_path: Option, + }, + #[error( + "failed to write zip archive{archive_path}", + archive_path = SpacePath(archive_path), + )] + WriteZip { + #[source] + source: rawzip::Error, + archive_path: Option, + }, + #[error( + "zip writer state machine violation: {reason} \ + (please report this logic error in Tufaceous)" + )] + WriteZipStateMachine { reason: &'static str }, + #[error( + "zip archive{archive_path}'s end of central directory record \ + expects {expected} entries, but found {actual} entries", + archive_path = SpacePath(archive_path), + actual = OrMore(actual), + )] + ZipEntryCount { + expected: u64, + actual: Option, + archive_path: Option, + }, + #[error( + "zip archive{archive_path} has overlapping compressed data ranges: \ + {earlier_path:?} ({earlier:?}) and {later_path:?} ({later:?})", + archive_path = SpacePath(archive_path), + earlier_path = DebugByteString(earlier_path), + later_path = DebugByteString(later_path), + )] + ZipOverlappingRanges { + earlier_path: Vec, + earlier: Range, + later_path: Vec, + later: Range, + archive_path: Option, + }, + #[error( + "zip archive{archive_path} path name mismatch between central directory \ + and local header ({central:?} != {local:?})", + archive_path = SpacePath(archive_path), + central = DebugByteString(central), + local = DebugByteString(local), + )] + ZipPathMismatch { + central: Vec, + local: Vec, + archive_path: Option, + }, + #[error( + "zip archive{archive_path} compressed data range for {file_path:?} \ + ({data_range:?}) overruns central directory", + archive_path = SpacePath(archive_path), + file_path = DebugByteString(file_path), + )] + ZipRangeOverrun { + file_path: Vec, + data_range: Range, + archive_path: Option, + }, + + #[error("failed to create temporary directory")] + CreateTempDir(#[source] std::io::Error), + #[error("failed to create temporary file")] + CreateTempFile(#[source] std::io::Error), + #[error("failed to open file{path}", path = SpacePath(path))] + OpenFile { + #[source] + source: std::io::Error, + path: Option, + }, + #[error("failed to read directory{path}", path = SpacePath(path))] + ReadDir { + #[source] + source: std::io::Error, + path: Option, + }, + #[error("failed to read from file{path}", path = SpacePath(path))] + ReadFile { + #[source] + source: std::io::Error, + path: Option, + }, + #[error("failed to seek in file{path}", path = SpacePath(path))] + SeekFile { + #[source] + source: std::io::Error, + path: Option, + }, + #[error("failed to write to file{path}", path = SpacePath(path))] + WriteFile { + #[source] + source: std::io::Error, + path: Option, + }, + #[error("failed to read stream")] + ReadStream(#[source] Box), + #[error("stream exceeded length limit ({limit})")] + StreamLimit { limit: u64 }, + + #[error("failed to read hubris archive {path}")] + ReadHubrisArchive { + #[source] + source: hubtools::Error, + path: Utf8PathBuf, + }, + #[error("failed to read caboose from {path}")] + ReadCaboose { + #[source] + source: tufaceous_artifact::ReadCabooseError, + path: Utf8PathBuf, + }, + #[error("failed to generate fake hubris archive")] + GenerateFakeHubrisArchive(#[source] hubtools::Error), + #[error("failed to read CoRIM manifest {path}")] + ReadCorim { + #[source] + source: rats_corim::Error, + path: Utf8PathBuf, + }, + #[error("failed to generate fake measurement corpus")] + GenerateFakeMeasurementCorpus(#[source] rats_corim::Error), + #[error("failed to serialize fake measurement corpus")] + SerializeFakeMeasurementCorpus(#[source] rats_corim::Error), + #[error("failed to generate fake zone image")] + GenerateFakeZoneImage(#[source] std::io::Error), + + #[error("unspecified failure when generating ed25519 key")] + Ed25519Generate, + #[error("failed to calculate TUF key ID")] + KeyId(#[source] tough::schema::Error), + #[error("failed to serialize root role")] + SerializeRoot(#[source] serde_json::Error), + #[error("role verification failed")] + RoleVerify(#[source] tough::schema::Error), + + #[error("metadata base URL unset")] + MetadataBaseUrlUnset, + #[error("targets base URL unset")] + TargetsBaseUrlUnset, + #[error("failed to join {url} onto {base}")] + UrlJoin { + #[source] + source: url::ParseError, + url: &'static str, + base: String, + }, + #[error("no trust roots provided to load repository")] + NoTrustRoots, + #[error("maximum root size {max_size} exceeded")] + MaxRootSizeExceeded { max_size: u64 }, + #[error( + "artifact ({tags} version={version} hash={hash} length={length}) \ + not found", + tags = .0.display_tags(), + version = .0.version, + hash = .0.hash, + length = .0.length, + )] + ArtifactNotFound(Artifact), + #[error("target {target_name} not found")] + TargetNotFound { target_name: String }, + #[error("failed to parse target {target}")] + ParseTargetJson { + #[source] + source: serde_json::Error, + target: String, + }, + #[error("unsafe target name {target_name}")] + UnsafeTargetName { target_name: String }, + + #[error(transparent)] + ArtifactVersion(#[from] tufaceous_artifact::ArtifactVersionError), + #[error("failed to guess what kind of artifact {path} is")] + GuessArtifact { path: Utf8PathBuf }, + #[error( + "failed to convert metadata structure into mapping \ + (please report this logic error in Tufaceous)" + )] + ConvertMetadataToMap(#[source] serde_json::Error), + #[error("target name collision on {target_name}")] + TargetNameCollision { target_name: String }, + #[error( + "artifacts {first_target_name} and {second_target_name} + have the same tags {tags}, which is not allowed", + tags = DisplayTags::from(.tags), + )] + DisallowedTagCollision { + first_target_name: String, + second_target_name: String, + tags: BTreeMap, + }, + #[error("target {target_name} has invalid SHA-256 hash length")] + InvalidHashLength { target_name: String }, + #[error("failed to serialize artifacts document")] + SerializeArtifacts(#[source] serde_json::Error), + #[error("failed to serialize Installinator document")] + SerializeInstallinator(#[source] serde_json::Error), + #[error("no root provided to sign repository")] + NoSigningRoot, + #[error("failed to parse signing root")] + ParseSigningRoot(#[source] serde_json::Error), + + #[error( + "failed to convert known tags structure into mapping \ + (please report this logic error in Tufaceous)" + )] + ConvertKnownTagsToMap(#[source] serde_json::Error), + #[error("failed to read composite artifact {target}")] + ReadCompositeArtifact { + #[source] + source: std::io::Error, + target: String, + }, + #[error("failed to read oxide.json from {path}")] + ReadZoneOxideJson { + #[source] + source: std::io::Error, + path: Utf8PathBuf, + }, + #[error("importing v1 repository to an editor is not supported")] + ImportV1Repo, +} + +impl ErrorKind { + /// Returns `true` if the error is due to a problem loading or reading + /// a repository, where retrying the operation with the same input would + /// result in the same error. + /// + /// Note that errors that can return `true` here can also happen during + /// other operations, such as editing or signing repositories. + pub fn is_repository_error(&self) -> bool { + match self { + ErrorKind::ZipEntryCount { .. } + | ErrorKind::ZipOverlappingRanges { .. } + | ErrorKind::ZipPathMismatch { .. } + | ErrorKind::ZipRangeOverrun { .. } + | ErrorKind::StreamLimit { .. } + | ErrorKind::ReadHubrisArchive { .. } + | ErrorKind::ReadCaboose { .. } + | ErrorKind::MetadataBaseUrlUnset + | ErrorKind::TargetsBaseUrlUnset + | ErrorKind::UrlJoin { .. } + | ErrorKind::NoTrustRoots + | ErrorKind::MaxRootSizeExceeded { .. } + | ErrorKind::ArtifactNotFound { .. } + | ErrorKind::TargetNotFound { .. } + | ErrorKind::ParseTargetJson { .. } + | ErrorKind::UnsafeTargetName { .. } + | ErrorKind::ArtifactVersion(_) + | ErrorKind::ConvertKnownTagsToMap(_) + | ErrorKind::ReadCompositeArtifact { .. } + | ErrorKind::ReadZoneOxideJson { .. } => true, + + ErrorKind::Join(_) + | ErrorKind::ToughKey(_) + | ErrorKind::WriteZip { .. } + | ErrorKind::WriteZipStateMachine { .. } + | ErrorKind::CreateTempDir(_) + | ErrorKind::CreateTempFile(_) + | ErrorKind::OpenFile { .. } + | ErrorKind::ReadDir { .. } + | ErrorKind::ReadFile { .. } + | ErrorKind::SeekFile { .. } + | ErrorKind::WriteFile { .. } + | ErrorKind::ReadStream(_) + | ErrorKind::GenerateFakeHubrisArchive(_) + | ErrorKind::GenerateFakeMeasurementCorpus(_) + | ErrorKind::ReadCorim { .. } + | ErrorKind::SerializeFakeMeasurementCorpus(_) + | ErrorKind::GenerateFakeZoneImage(_) + | ErrorKind::Ed25519Generate + | ErrorKind::KeyId(_) + | ErrorKind::SerializeRoot(_) + | ErrorKind::RoleVerify(_) + | ErrorKind::GuessArtifact { .. } + | ErrorKind::ConvertMetadataToMap(_) + | ErrorKind::TargetNameCollision { .. } + | ErrorKind::DisallowedTagCollision { .. } + | ErrorKind::InvalidHashLength { .. } + | ErrorKind::SerializeArtifacts(_) + | ErrorKind::SerializeInstallinator(_) + | ErrorKind::NoSigningRoot + | ErrorKind::ParseSigningRoot(_) + | ErrorKind::ImportV1Repo => false, + + ErrorKind::Fetch(error) => { + // A transport error might be due to a broken repository (e.g. + // HTTP Not Found, ZIP CRC mismatch), but it might also be due + // to a retryable problem (e.g. HTTP timeout). We will try and + // classify errors we can introspect but otherwise we'll return + // false. + match error.kind() { + TransportErrorKind::UnsupportedUrlScheme + | TransportErrorKind::FileNotFound => return true, + _ => {} + } + if let Some(source) = error.source().and_then(|source| { + source.downcast_ref::() + }) { + return match source { + ZipTransportError::UrlJoin { .. } + | ZipTransportError::CompressionMethod(_) + | ZipTransportError::Duplicate + | ZipTransportError::FileNotFound + | ZipTransportError::IsADirectory + | ZipTransportError::IsASymlink => true, + + ZipTransportError::Io(_) + | ZipTransportError::Join(_) => false, + + ZipTransportError::Zip(source) => { + // All of rawzip's errors are related to broken zip + // files, except for IO errors. + !matches!(source.kind(), rawzip::ErrorKind::IO(_)) + } + }; + } + false + } + + ErrorKind::Tough(error) => { + // tough's errors are... tough to classify. We have a pretty + // simple heuristic: if any error in the source chain is + // `std::io::Error`, return false. Otherwise, return true. + for source in std::iter::successors(error.source(), |source| { + (*source).source() + }) { + if source.downcast_ref::().is_some() { + return false; + } + } + true + } + + ErrorKind::ReadZipEocd { source, .. } + | ErrorKind::ReadZipCdfh { source, .. } + | ErrorKind::ReadZipLocal { source, .. } => { + // All of rawzip's errors are related to broken zip files, + // except for IO errors. + !matches!(source.kind(), rawzip::ErrorKind::IO(_)) + } + } + } +} + +impl> From for Error { + fn from(kind: T) -> Self { + Error(Box::new(kind.into())) + } +} + +impl From for ErrorKind { + fn from(error: tough::error::Error) -> Self { + if let tough::error::Error::Transport { source, .. } = error { + ErrorKind::Fetch(source) + } else { + ErrorKind::Tough(error) + } + } +} + +impl From for ErrorKind { + fn from(error: Infallible) -> Self { + match error {} + } +} + +struct SpacePath<'a>(&'a Option); + +impl Display for SpacePath<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.0 { + Some(path) => write!(f, " {path}"), + None => Ok(()), + } + } +} + +struct OrMore<'a>(&'a Option); + +impl Display for OrMore<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.0 { + Some(value) => write!(f, "{value}"), + None => write!(f, "more"), + } + } +} + +pub(crate) struct DebugByteString<'a>(pub(crate) &'a [u8]); + +impl Debug for DebugByteString<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if let Ok(s) = std::str::from_utf8(self.0) { + write!(f, "{s:?}") + } else { + write!(f, "b\"{}\"", self.0.escape_ascii()) + } + } +} + +#[cfg(test)] +mod tests { + use crate::error::Error; + use crate::error::ErrorKind; + + #[test] + fn error_display_chain_doesnt_repeat() { + let err = Error::from(ErrorKind::OpenFile { + source: std::io::Error::from(std::io::ErrorKind::NotFound), + path: Some("/nowhere/in/particular".into()), + }); + assert_eq!( + crate::util::error_chain(&err), + "failed to open file /nowhere/in/particular: entity not found" + ); + } +} diff --git a/lib/src/key.rs b/lib/src/key.rs deleted file mode 100644 index 2703d82..0000000 --- a/lib/src/key.rs +++ /dev/null @@ -1,84 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use std::fmt::{self, Display}; -use std::str::FromStr; - -use anyhow::{Result, bail}; -use aws_lc_rs::rand::SystemRandom; -use aws_lc_rs::signature::Ed25519KeyPair; -use base64::{Engine, engine::general_purpose::URL_SAFE}; -use tough::async_trait; -use tough::key_source::KeySource; -use tough::sign::{Sign, SignKeyPair}; - -pub(crate) fn boxed_keys(keys: Vec) -> Vec> { - keys.into_iter().map(|k| Box::new(k) as Box).collect() -} - -#[derive(Debug, Clone)] -pub enum Key { - Ed25519 { pkcs8: Vec }, -} - -impl Key { - pub fn generate_ed25519() -> Result { - let pkcs8 = Ed25519KeyPair::generate_pkcs8(&SystemRandom::new())?; - Ok(Key::Ed25519 { pkcs8: pkcs8.as_ref().to_vec() }) - } - - fn as_sign_key_pair(&self) -> Result { - match self { - Key::Ed25519 { pkcs8 } => { - Ok(SignKeyPair::ED25519(Ed25519KeyPair::from_pkcs8(pkcs8)?)) - } - } - } - - pub(crate) fn as_tuf_key(&self) -> Result { - Ok(self.as_sign_key_pair()?.tuf_key()) - } -} - -#[async_trait] -impl KeySource for Key { - async fn as_sign( - &self, - ) -> Result, Box> - { - Ok(Box::new(self.as_sign_key_pair()?)) - } - - async fn write( - &self, - _value: &str, - _key_id_hex: &str, - ) -> Result<(), Box> { - Err("cannot write key back to key source".into()) - } -} - -impl FromStr for Key { - type Err = anyhow::Error; - - fn from_str(s: &str) -> Result { - match s.split_once(':') { - Some(("ed25519", base64)) => { - Ok(Key::Ed25519 { pkcs8: URL_SAFE.decode(base64)? }) - } - Some((kind, _)) => bail!("Invalid key source kind: {}", kind), - None => bail!("Invalid key source (format is `kind:data`)"), - } - } -} - -impl Display for Key { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Key::Ed25519 { pkcs8 } => { - write!(f, "ed25519:{}", URL_SAFE.encode(pkcs8)) - } - } - } -} diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 9dc1854..e457ad9 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -2,16 +2,35 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -mod archive; -mod artifact; -pub mod assemble; -mod key; -mod repository; -pub mod root; -mod target; -mod utils; +/// This library very commonly needs to convert from usize (Rust's standard +/// length type) to u64 (standard for file sizes in operating system APIs). This +/// is fallible so that Rust can be adapted to systems with pointer sizes larger +/// than 64 bits. In 2026 these systems do not appear to exist nor appear to be +/// on the horizon. +/// +/// This is a shorthand to assert that a value is a usize, then convert and +/// panic on failure; the failure is impossible and elided on 64-bit or lower +/// systems. +macro_rules! usize64 { + ($usize:expr) => {{ + let x: usize = $usize; + u64::try_from(x).expect("usize fits in u64") + }}; +} -pub use archive::*; -pub use artifact::*; -pub use key::*; -pub use repository::*; +pub mod edit; +pub mod error; +mod loader; +mod mpsc_stream; +mod repo; +mod schema; +mod util; +mod zip_transport; + +pub use loader::*; +pub use repo::*; +pub use zip_transport::*; + +pub(crate) const COSMO_PHASE_1_PATH: &str = "cosmo.rom"; +pub(crate) const GIMLET_PHASE_1_PATH: &str = "gimlet.rom"; +pub(crate) const PHASE_2_PATH: &str = "zfs.img"; diff --git a/lib/src/loader.rs b/lib/src/loader.rs new file mode 100644 index 0000000..c18f989 --- /dev/null +++ b/lib/src/loader.rs @@ -0,0 +1,372 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::fmt::Debug; +use std::io::Seek; + +use bytes::Bytes; +use camino::Utf8PathBuf; +use digest_io::IoWrapper; +use futures_util::Stream; +use futures_util::TryStreamExt; +use futures_util::pin_mut; +use sha2::Digest; +use sha2::Sha256; +use slog::Logger; +use tokio::io::AsyncWriteExt; +pub use tough::ExpirationEnforcement; +pub use tough::Limits; +use url::Url; + +use crate::Repository; +use crate::ZipTransport; +use crate::error::Error; +use crate::error::ErrorKind; +use crate::error::try_path; + +#[derive(Debug, Clone, Default)] +#[must_use] +pub struct RepositoryLoader { + compute_archive_sha256: bool, + expiration_enforcement: ExpirationEnforcement, + limits: Limits, + metadata_base_url: Option, + targets_base_url: Option, + trust_store: TrustStore, + v1_compatibility: bool, +} + +#[derive(Debug, Clone)] +enum TrustStore { + UnsafeBlindTrust, + Store(Vec>), +} + +impl Default for TrustStore { + fn default() -> Self { + Self::Store(Vec::new()) + } +} + +impl RepositoryLoader { + /// Constructs a new `RepositoryLoader`. + /// + /// This is the same as `Repository::loader()`. + pub fn new() -> Self { + Self::default() + } + + /// Set whether to compute and store the SHA256 digest of the archive, if an + /// archive is used to load the repository. + /// + /// Only affects [`Self::load_zip_buffer`], [`Self::load_zip_path`], + /// [`Self::load_zip_file`], and [`Self::load_zip_stream`]. + pub fn compute_archive_sha256(self, compute_archive_sha256: bool) -> Self { + Self { compute_archive_sha256, ..self } + } + + /// Set whether metadata expiration times are enforced. + pub fn expiration_enforcement( + self, + expiration_enforcement: ExpirationEnforcement, + ) -> Self { + Self { expiration_enforcement, ..self } + } + + /// Set limits used while fetching repository metadata. + pub fn limits(self, limits: Limits) -> Self { + Self { limits, ..self } + } + + /// Set the metadata base URL. + pub fn metadata_base_url(self, metadata_base_url: Url) -> Self { + Self { metadata_base_url: Some(metadata_base_url), ..self } + } + + /// Set the targets base URL. + pub fn targets_base_url(self, targets_base_url: Url) -> Self { + Self { targets_base_url: Some(targets_base_url), ..self } + } + + /// Add a trusted root role to the trust store. + pub fn trust_root(mut self, trust_root: impl AsRef<[u8]>) -> Self { + match self.trust_store { + TrustStore::Store(ref mut store) => { + store.push(trust_root.as_ref().into()); + } + TrustStore::UnsafeBlindTrust => { + self.trust_store = + TrustStore::Store(vec![trust_root.as_ref().into()]); + } + } + self + } + + /// Add additional trusted root roles to the trust store. + pub fn trust_roots( + mut self, + trust_roots: impl IntoIterator>, + ) -> Self { + let iter = trust_roots.into_iter().map(|root| root.as_ref().into()); + match self.trust_store { + TrustStore::Store(ref mut store) => { + store.extend(iter); + } + TrustStore::UnsafeBlindTrust => { + self.trust_store = TrustStore::Store(iter.collect()); + } + } + self + } + + /// Blindly trust whatever root is contained in the repository. + /// + /// This reads the trust root by fetching `metadata/1.root.json` from the + /// repository, then proceeds with normal verification. The repository must + /// be validly signed, but with this set we do not care who signed it. + /// + /// Calling this method removes any trust roots added via + /// [`Self::trust_root`] and [`Self::trust_roots`], and calling either of + /// those methods to add a trust root reverts the effect of this method. + pub fn unsafe_blindly_trust_repo(self) -> Self { + Self { trust_store: TrustStore::UnsafeBlindTrust, ..self } + } + + /// Enable compatibility with v1-format repositories. + /// + /// If a v1-format repository is encountered, composite artifacts are + /// extracted into temporary files in [`std::env::temp_dir()`]. Artifacts + /// are not extracted in parallel to avoid unexpectedly using too many + /// resources. Reading a v1-format repository takes on the order of about 10 + /// seconds on a 2025-era CPU. + pub fn v1_compatibility(self, v1_compatibility: bool) -> Self { + Self { v1_compatibility, ..self } + } + + fn zip_base_urls(self) -> Self { + self.metadata_base_url( + "zip:///repo/metadata/" + .parse() + .expect("`zip:///repo/metadata/` is a valid URL"), + ) + .targets_base_url( + "zip:///repo/targets/" + .parse() + .expect("`zip:///repo/targets/` is a valid URL"), + ) + } + + /// Load a Tufaceous-generated ZIP archive from an owned buffer. + pub async fn load_zip_buffer( + self, + data: T, + log: &Logger, + ) -> Result + where + T: AsRef<[u8]> + Debug + Send + Sync + 'static, + { + let (data, sha256) = if self.compute_archive_sha256 { + tokio::task::spawn_blocking(move || { + let digest = Sha256::digest(&data).into(); + (data, Some(digest)) + }) + .await? + } else { + (data, None) + }; + let transport = ZipTransport::from_slice(data, log)?; + let mut repo = self.zip_base_urls().load(transport, log).await?; + repo.archive_sha256 = sha256; + Ok(repo) + } + + /// Load a Tufaceous-generated ZIP archive from a file path. + pub async fn load_zip_path( + self, + archive_path: Utf8PathBuf, + log: &Logger, + ) -> Result { + let file = try_path!( + tokio::fs::File::open(&archive_path).await, + OpenFile, + archive_path + ); + self.load_zip_file(file.into_std().await, Some(archive_path), log).await + } + + /// Load a Tufaceous-generated ZIP archive from an opened file. + /// + /// `archive_path` is used in errors, if available. + pub async fn load_zip_file( + self, + mut file: std::fs::File, + archive_path: Option, + log: &Logger, + ) -> Result { + let (sha256, transport) = { + let archive_path = archive_path.clone(); + let log = log.clone(); + let compute_archive_sha256 = self.compute_archive_sha256; + tokio::task::spawn_blocking(move || { + let sha256 = if compute_archive_sha256 { + let mut hasher = IoWrapper(Sha256::new()); + try_path!( + file.rewind().and_then(|()| std::io::copy( + &mut file, + &mut hasher + )), + ReadFile, + archive_path + ); + Some(hasher.0.finalize().0) + } else { + None + }; + let transport = + ZipTransport::from_file_blocking(file, archive_path, &log)?; + Ok::<_, Error>((sha256, transport)) + }) + .await?? + }; + + let mut repo = self.zip_base_urls().load(transport, log).await?; + repo.archive_path = archive_path; + repo.archive_sha256 = sha256; + Ok(repo) + } + + /// Load a Tufaceous-generated ZIP archive from a stream. + /// + /// This writes the contents of the stream to a temporary file. You can + /// optionally provide a maximum number of bytes that will be written. If + /// this is not provided, you should ensure something else is limiting the + /// size of this stream (such as an HTTP body length limit). + pub async fn load_zip_stream( + self, + stream: impl Stream>, + limit: Option, + log: &Logger, + ) -> Result + where + E: std::error::Error + Send + Sync + 'static, + { + pin_mut!(stream); + let file = + camino_tempfile::tempfile().map_err(ErrorKind::CreateTempFile)?; + let mut file = tokio::fs::File::from(file); + let mut hasher = self.compute_archive_sha256.then(Sha256::new); + let mut bytes_read = limit.map(|_| 0u64); + while let Some(chunk) = stream + .try_next() + .await + .map_err(|source| ErrorKind::ReadStream(Box::new(source)))? + { + if let Some(limit) = limit + && let Some(bytes_read) = bytes_read.as_mut() + { + *bytes_read = u64::try_from(chunk.len()) + .ok() + .and_then(|len| bytes_read.checked_add(len)) + .ok_or(ErrorKind::StreamLimit { limit })?; + if *bytes_read > limit { + return Err(ErrorKind::StreamLimit { limit }.into()); + } + } + if let Some(hasher) = hasher.as_mut() { + hasher.update(&chunk); + } + try_path!(file.write_all(&chunk).await, WriteFile, None); + } + let mut repo = + self.load_zip_file(file.into_std().await, None, log).await?; + repo.archive_sha256 = hasher.map(|hasher| hasher.finalize().into()); + Ok(repo) + } + + /// Load a repository from the configured metadata and targets base URLs + /// using the given transport. + pub async fn load( + self, + transport: impl tough::Transport + Clone + 'static, + log: &Logger, + ) -> Result { + let Some(metadata_base_url) = self.metadata_base_url else { + return Err(ErrorKind::MetadataBaseUrlUnset.into()); + }; + let Some(targets_base_url) = self.targets_base_url else { + return Err(ErrorKind::TargetsBaseUrlUnset.into()); + }; + + let trust_roots = match self.trust_store { + TrustStore::Store(trust_roots) => trust_roots, + TrustStore::UnsafeBlindTrust => { + let root_path = "1.root.json"; + let root_url = + metadata_base_url.join(root_path).map_err(|source| { + ErrorKind::UrlJoin { + source, + url: root_path, + base: metadata_base_url.to_string(), + } + })?; + let mut stream = transport + .fetch(root_url.clone()) + .await + .map_err(ErrorKind::Fetch)?; + let mut root = Vec::new(); + while let Some(bytes) = + stream.try_next().await.map_err(ErrorKind::Fetch)? + { + root.extend_from_slice(&bytes); + if usize64!(root.len()) > self.limits.max_root_size { + return Err(ErrorKind::MaxRootSizeExceeded { + max_size: self.limits.max_root_size, + } + .into()); + } + } + vec![root] + } + }; + + let mut last_error = ErrorKind::NoTrustRoots; + for trust_root in trust_roots { + match tough::RepositoryLoader::new( + &trust_root, + metadata_base_url.clone(), + targets_base_url.clone(), + ) + .expiration_enforcement(self.expiration_enforcement) + .limits(self.limits) + .transport(transport.clone()) + .load() + .await + { + Ok(repo) => { + return Repository::from_loaded( + repo, + log, + trust_root, + self.v1_compatibility, + ) + .await; + } + Err(error) => { + if matches!( + error, + tough::error::Error::VerifyMetadata { .. } + | tough::error::Error::VerifyTrustedMetadata { .. } + ) { + // failed to verify, try the next trust root + last_error = ErrorKind::from(error); + } else { + // other errors are fatal + return Err(Error::from(error)); + } + } + } + } + Err(last_error.into()) + } +} diff --git a/lib/src/mpsc_stream.rs b/lib/src/mpsc_stream.rs new file mode 100644 index 0000000..c5bdbe8 --- /dev/null +++ b/lib/src/mpsc_stream.rs @@ -0,0 +1,64 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use futures_util::FutureExt; +use futures_util::Stream; +use futures_util::StreamExt; +use futures_util::stream; +use slog::Logger; +use slog::error; +use tokio::sync::mpsc::Sender; +use tokio::sync::mpsc::error::SendError; +use tokio::task::JoinError; + +pub(crate) fn mpsc_stream( + log: Option, + f: F, +) -> impl Stream> + 'static +where + T: Send + 'static, + E: std::error::Error + From + Send + 'static, + F: FnOnce(Sender>) -> Result<(), SendError>> + + Send + + 'static, +{ + let (tx, mut rx) = tokio::sync::mpsc::channel(1); + let task = tokio::task::spawn_blocking(move || f(tx)); + stream::poll_fn(move |cx| rx.poll_recv(cx)).chain( + async move { + stream::iter(match task.await { + Ok(Ok(())) => None, + + // It is presumed a `SendError` is returned from an attempt to + // send a value across the channel created by `mpsc_stream`. + // This would indicate that either the receiver was explicitly + // closed (which this function does not do), or that the + // receiver was dropped (which would not be possible if we are + // in this match statement). Nonetheless, we will log such an + // oddity, and if the value that failed to send itself contained + // an error, we will return it as this stream's final value. + Ok(Err(SendError(Ok(_)))) => { + log.inspect(|log| { + error!(log, "stream reader unexpectedly hung up"); + }); + None + } + Ok(Err(SendError(Err(error)))) => { + log.inspect(|log| { + error!( + log, + "stream reader unexpectedly hung up \ + before receiving error"; + "err" => &crate::util::error_chain(&error), + ); + }); + Some(Err(error)) + } + + Err(join_error) => Some(Err(E::from(join_error))), + }) + } + .flatten_stream(), + ) +} diff --git a/lib/src/repo.rs b/lib/src/repo.rs new file mode 100644 index 0000000..9ad6675 --- /dev/null +++ b/lib/src/repo.rs @@ -0,0 +1,484 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +mod check; +mod v1; + +use std::collections::BTreeMap; +use std::collections::HashMap; +use std::pin::Pin; +use std::sync::Arc; + +use bytes::Bytes; +use camino::Utf8Path; +use camino::Utf8PathBuf; +use futures_util::Stream; +use futures_util::TryStreamExt; +use rawzip::FileReader; +use semver::Version; +use serde::de::DeserializeOwned; +use slog::Logger; +use tokio::sync::Semaphore; +use tokio::sync::TryAcquireError; +use tokio::task::JoinSet; +use tough::TargetName; +use tough::schema::Target; +use tufaceous_artifact::Artifact; +use tufaceous_artifact::ArtifactHash; +use tufaceous_artifact::ArtifactSet; +use tufaceous_artifact::KnownArtifactTags; +use tufaceous_artifact::Metadata; +use tufaceous_artifact::artifact_set::GetError; + +use crate::RepositoryLoader; +use crate::error::Error; +use crate::error::ErrorKind; +pub use crate::repo::check::CheckProblem; +use crate::schema::ArtifactSchema; +use crate::schema::ArtifactSetSchema; + +pub type TargetStream = + Pin> + Send + Sync + 'static>>; + +/// A loaded TUF repository. +#[derive(Debug, Clone)] +pub struct Repository { + log: Logger, + tuf_repo: tough::Repository, + system_version: Version, + trust_root: Vec, + artifacts: ArtifactSet, + artifact_data: BTreeMap, + metadata: BTreeMap, + + // These are set directly by the ZIP archive convenience methods in the + // loader module. + // TODO after v2 merge: Move the loader module under this repo module so + // that this doesn't need to be pub(crate). + pub(crate) archive_path: Option, + pub(crate) archive_sha256: Option<[u8; 32]>, +} + +impl Repository { + pub fn loader() -> RepositoryLoader { + RepositoryLoader::new() + } + + /// Generate and load a fake repository. + /// + /// This is shorthand for: + /// + /// ```rust + /// # tokio_test::block_on(async { + /// # let system_version = const { semver::Version::new(1, 0, 0) }; + /// # let log = slog::Logger::root(slog::Discard, slog::o!()); + /// # let log = &log; + /// let signed = tufaceous::edit::RepositoryEditor::fake(system_version)? + /// .finish() + /// .await? + /// .generate_root() + /// .sign() + /// .await?; + /// let zip = signed.write_zip(Vec::new(), chrono::Utc::now()).await?; + /// # Ok::<_, tufaceous::error::Error>( + /// tufaceous::RepositoryLoader::new() + /// .trust_root(signed.root()) + /// .load_zip_buffer(zip, log) + /// .await? + /// # ) + /// # }).unwrap(); + /// ``` + pub async fn fake( + system_version: Version, + log: &Logger, + ) -> Result { + let signed = crate::edit::RepositoryEditor::fake(system_version)? + .finish() + .await? + .generate_root() + .sign() + .await?; + let zip = signed.write_zip(Vec::new(), chrono::Utc::now()).await?; + RepositoryLoader::new() + .trust_root(signed.root()) + .load_zip_buffer(zip, log) + .await + } + + pub(crate) async fn from_loaded( + tuf_repo: tough::Repository, + log: &Logger, + trust_root: Vec, + v1_compatibility: bool, + ) -> Result { + let Some(ArtifactSetSchema { system_version, artifacts, metadata }) = + read_target_json(&tuf_repo, ArtifactSetSchema::TARGET_NAME).await? + else { + if v1_compatibility + && let Some(partial) = v1::from_loaded(&tuf_repo, log).await? + { + return Ok(Repository { + log: log.clone(), + tuf_repo, + trust_root, + system_version: partial.system_version, + artifacts: partial.artifacts, + artifact_data: partial.artifact_data, + metadata: BTreeMap::new(), + archive_path: None, + archive_sha256: None, + }); + } + + return Err(ErrorKind::TargetNotFound { + target_name: ArtifactSetSchema::TARGET_NAME.to_owned(), + } + .into()); + }; + + let (artifacts, artifact_data) = artifacts + .into_iter() + .map(|ArtifactSchema { target_name, version, tags }| { + let (hash, length) = target_meta(&tuf_repo, &target_name)?; + let artifact = Artifact { version, tags, hash, length }; + Ok::<_, Error>(( + artifact.clone(), + (artifact, ArtifactData::Target { target_name }), + )) + }) + .collect::>()?; + Ok(Repository { + log: log.clone(), + tuf_repo, + trust_root, + system_version, + artifacts, + artifact_data, + metadata, + archive_path: None, + archive_sha256: None, + }) + } + + pub fn system_version(&self) -> &Version { + &self.system_version + } + + pub fn trust_root(&self) -> &[u8] { + &self.trust_root + } + + /// Returns the path to the loaded archive, if there is one. + /// + /// Set when [`RepositoryLoader::load_zip_file`] or + /// [`RepositoryLoader::load_zip_path`] are used. + pub fn archive_path(&self) -> Option<&Utf8Path> { + self.archive_path.as_deref() + } + + /// Returns the SHA256 digest of the loaded archive, if there is one. + /// + /// Set when [`RepositoryLoader::compute_archive_sha256`] is set to `true` + /// and one of the archive loading methods is used. + pub fn archive_sha256(&self) -> Option<&[u8; 32]> { + self.archive_sha256.as_ref() + } + + pub fn targets(&self) -> &HashMap { + &self.tuf_repo.targets().signed.targets + } + + pub fn artifacts(&self) -> &ArtifactSet { + &self.artifacts + } + + /// Returns an [`ArtifactSchema`] for the artifacts in the repository. + /// + /// This is used by [`crate::edit::RepositoryEditor::import_repo`]. + /// + /// # Errors + /// + /// If this was a v1 repository, returns an error; we don't have a valid + /// target name for unpacked artifacts. + pub(crate) fn to_artifact_schema( + &self, + ) -> Result, Error> { + self.artifact_data + .iter() + .map(|(artifact, data)| match data { + ArtifactData::Target { target_name } => Ok(ArtifactSchema { + target_name: target_name.clone(), + version: artifact.version.clone(), + tags: artifact.tags.clone(), + }), + ArtifactData::V1Unpacked { .. } => { + Err(ErrorKind::ImportV1Repo.into()) + } + }) + .collect() + } + + pub fn metadata(&self) -> &BTreeMap { + &self.metadata + } + + pub fn structured_metadata(&self) -> Option { + Metadata::from_map(self.metadata.clone()).ok() + } + + pub async fn read_artifact( + &self, + artifact: &Artifact, + ) -> Result { + let data = self + .artifact_data + .get(artifact) + .ok_or_else(|| ErrorKind::ArtifactNotFound(artifact.clone()))?; + match data { + ArtifactData::Target { target_name } => { + self.read_target(target_name).await + } + ArtifactData::V1Unpacked { + file, + original_target_name, + inner_path, + } => { + let unpacked = v1::UnpackedArtifact { + file: file.clone(), + hash: artifact.hash, + length: artifact.length, + }; + Ok(Box::pin(unpacked.stream( + &self.log, + original_target_name, + inner_path, + ))) + } + } + } + + /// Read a target from the underlying TUF repository by its target name. + /// + /// If you have an [`Artifact`], use [`Self::read_artifact`]. + pub async fn read_target( + &self, + target_name: &str, + ) -> Result { + if let Some(stream) = read_target(&self.tuf_repo, target_name).await? { + Ok(Box::pin(stream)) + } else { + Err(ErrorKind::TargetNotFound { + target_name: target_name.to_owned(), + } + .into()) + } + } + + /// Returns an [`ArtifactHandle`] for the one (and only one) artifact + /// matching `tags` that can be used to stream the artifact. + /// + /// An artifact handle simply clones `artifact` and `self` and keeps them + /// together in a struct as a convenience. The repository must be wrapped in + /// [`Arc`] to call this method. + /// + /// # Errors + /// + /// Returns an error if there is not exactly one artifact matching `tags`. + pub fn get_handle( + self: &Arc, + tags: &KnownArtifactTags, + ) -> Result { + let artifact = self.artifacts.get_only(tags)?.clone(); + Ok(ArtifactHandle { artifact, repo: Arc::clone(self) }) + } + + /// Returns an iterator of [`ArtifactHandle`]s for every artifact + /// in the repository. + /// + /// An artifact handle simply clones `artifact` and `self` and keeps them + /// together in a struct as a convenience. The repository must be wrapped in + /// [`Arc`] to call this method. + pub fn handles(self: &Arc) -> impl Iterator { + self.artifacts + .iter() + .cloned() + .map(|artifact| ArtifactHandle { artifact, repo: Arc::clone(self) }) + } + + /// Reads all targets in the repository and verifies they have the correct + /// length and checksum. + /// + /// The repository must be wrapped in [`Arc`] to call this method. + /// + /// This method is *not* necessary to safely read the repository. All + /// streams returned from [`Repository::read_target`] and read completely + /// are verified to have the same correct length and checksum. + /// + /// However, this verification is useful to complete before using contents + /// for any operations if you have the entire repository available locally. + /// In the past, Tufaceous archives were used by completely unpacking the + /// archive, which verified the contents against the CRC-32 checksums in the + /// ZIP archive; this caught binaries that were corrupted in transit between + /// CI and destination hardware. + /// + /// `parallelism` controls how many targets are read at a time. + #[expect(clippy::missing_panics_doc)] + pub async fn verify_targets( + &self, + parallelism: usize, + ) -> Result<(), Error> { + // This is a reimplementation of `parallel-task-set` from Omicron, + // and could potentially be replaced with that if it's published to + // crates.io. + let semaphore = Arc::new(Semaphore::new(parallelism.max(1))); + let mut set: JoinSet> = JoinSet::new(); + + for target_name in self.targets().keys() { + let target = target_name.raw().to_owned(); + let permit = match Arc::clone(&semaphore).try_acquire_owned() { + Ok(permit) => permit, + Err(TryAcquireError::Closed) => { + unreachable!("we never close the semaphore") + } + Err(TryAcquireError::NoPermits) => { + if let Some(result) = set.join_next().await { + () = result??; + } + Arc::clone(&semaphore) + .acquire_owned() + .await + .expect("we never close the semaphore") + } + }; + let mut stream = self.read_target(&target).await?; + set.spawn(async move { + let _permit = permit; + // Read the stream to the end. There's no need to do anything + // with the data; the underlying stream performs verification. + while stream.try_next().await?.is_some() {} + Ok(()) + }); + } + + while let Some(result) = set.join_next().await { + () = result??; + } + + Ok(()) + } +} + +/// While we have v1 compatibility, artifacts might come from the underlying +/// `tough::Repository` or from an unpacked file on disk. When we drop all v1 +/// compatibility code we can remove this indirection. +#[derive(Debug, Clone)] +enum ArtifactData { + /// The artifact is read from the underlying `tough::Repository`. + Target { target_name: String }, + /// The artifact was unpacked from a composite artifact in a v1 repository, + /// and is read from a temporary file on disk. + V1Unpacked { + file: Arc, + original_target_name: String, + inner_path: Utf8PathBuf, + }, +} + +impl ArtifactData { + fn original_target_name(&self) -> &str { + match self { + ArtifactData::Target { target_name } => target_name, + ArtifactData::V1Unpacked { original_target_name, .. } => { + original_target_name + } + } + } +} + +/// An [`Artifact`] and the [`Repository`] it belongs to, for convenience to +/// code that works at the artifact level but needs to read the artifact data +/// from the repository. +/// +/// Created by [`Repository::get_handle`] or [`Repository::handles`]. +#[derive(Debug, Clone)] +pub struct ArtifactHandle { + artifact: Artifact, + repo: Arc, +} + +impl ArtifactHandle { + /// Return a reference to this handle's [`Artifact`]. + pub fn artifact(&self) -> &Artifact { + &self.artifact + } + + /// Convert this handle into its owned [`Artifact`], dropping its + /// association with its [`Repository`]. + pub fn into_artifact(self) -> Artifact { + self.artifact + } + + /// Read this artifact from its repository. + pub async fn stream(&self) -> Result { + self.repo.read_artifact(&self.artifact).await + } +} + +async fn read_target( + tuf_repo: &tough::Repository, + target: &str, +) -> Result> + 'static>, Error> { + let target_name = target.parse()?; + // Ensure the target is in the top-level targets.json role and not a + // delegated target; we don't permit the use of delegated targets in + // Tufaceous currently. + if !tuf_repo.targets().signed.targets.contains_key(&target_name) { + return Ok(None); + } + Ok(tuf_repo + .read_target(&target_name) + .await? + .map(TryStreamExt::err_into::)) +} + +async fn read_target_vec( + tuf_repo: &tough::Repository, + target: &str, +) -> Result>, Error> { + let Some(stream) = read_target(tuf_repo, target).await? else { + return Ok(None); + }; + stream.map_ok(Vec::from).try_concat().await.map(Some) +} + +async fn read_target_json( + tuf_repo: &tough::Repository, + target: &str, +) -> Result, Error> { + let Some(vec) = read_target_vec(tuf_repo, target).await? else { + return Ok(None); + }; + serde_json::from_slice(&vec).map_err(|source| { + ErrorKind::ParseTargetJson { source, target: target.into() }.into() + }) +} + +fn target_meta( + tuf_repo: &tough::Repository, + target_name: &str, +) -> Result<(ArtifactHash, u64), Error> { + let name = TargetName::new(target_name).map_err(|_| { + ErrorKind::UnsafeTargetName { target_name: target_name.to_owned() } + })?; + let target = + tuf_repo.targets().signed.targets.get(&name).ok_or_else(|| { + ErrorKind::TargetNotFound { target_name: target_name.to_owned() } + })?; + let hash = ArtifactHash(target.hashes.sha256.as_ref().try_into().map_err( + |_| ErrorKind::InvalidHashLength { + target_name: target_name.to_owned(), + }, + )?); + Ok((hash, target.length)) +} diff --git a/lib/src/repo/check.rs b/lib/src/repo/check.rs new file mode 100644 index 0000000..b373249 --- /dev/null +++ b/lib/src/repo/check.rs @@ -0,0 +1,198 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::BTreeMap; + +use daft::Diffable; +use futures_util::TryStreamExt; +use tufaceous_artifact::ArtifactVersion; +use tufaceous_artifact::DisplayTags; +use tufaceous_artifact::InstallinatorDocument; +use tufaceous_artifact::KnownArtifactTags; +use tufaceous_artifact::OsBoard; +use tufaceous_artifact::OsPhase1Tags; +use tufaceous_artifact::OsPhase2Tags; +use tufaceous_artifact::OsVariant; +use tufaceous_artifact::artifact_set::GetError; + +use crate::Repository; +use crate::error::Error; + +impl Repository { + /// Check the repository for consistency and other problems. + /// + /// # ⚠️ Causality Hazard ⚠️ + /// + /// The definition of a well-formed repository is subject to change. This + /// must **not** be used for repositories that may have been created by + /// another version of Tufaceous, such as when the control plane decides to + /// accept a repository. + #[allow(clippy::too_many_lines)] + pub async fn check_problems(&self) -> Result, Error> { + let mut problems = Vec::new(); + + for (artifact, data) in &self.artifact_data { + // Check that no artifact contains unknown tags. + if artifact.known_tags().is_none() { + problems.push(CheckProblem::UnknownTags { + target_name: data.original_target_name().to_owned(), + tags: artifact.tags.clone(), + }); + } + } + + match self.structured_metadata().and_then(|m| match m.to_map() { + Ok(m) => Some(m), + Err(source) => { + problems.push(CheckProblem::ConvertMetadataMapping(source)); + None + } + }) { + Some(metadata) => { + if self.metadata() != &metadata { + let diff = self.metadata().diff(&metadata); + for (key, leaf) in diff.modified() { + problems.push(CheckProblem::UnexpectedMetadataDiff { + key: key.clone(), + old_value: Some(leaf.before.clone()), + new_value: leaf.after.clone(), + }); + } + for (key, value) in diff.added { + problems.push(CheckProblem::UnexpectedMetadataDiff { + key: key.clone(), + old_value: None, + new_value: value.clone(), + }); + } + for (key, _) in diff.removed { + problems.push(CheckProblem::UnknownMetadataKey( + key.clone(), + )); + } + } + } + None => { + problems.push(CheckProblem::UnknownMetadata( + self.metadata().clone(), + )); + } + } + + // For all tags we expect to see a single artifact, except for + // MeasurementCorpus. + for (tags, iter) in self.artifacts().known() { + if iter.len() > 1 && !tags.allow_multiple_artifacts() { + problems.push(CheckProblem::MultipleArtifacts(tags.clone())); + } + } + + // We expect to see artifacts matching these tags for Installinator + // to work. + let mut expected = vec![KnownArtifactTags::InstallinatorDocument]; + for os_variant in [OsVariant::Host, OsVariant::Recovery] { + for os_board in [OsBoard::COSMO, OsBoard::GIMLET] { + expected.push(KnownArtifactTags::OsPhase1(OsPhase1Tags { + os_board, + os_variant, + })); + } + expected + .push(KnownArtifactTags::OsPhase2(OsPhase2Tags { os_variant })); + } + for tags in expected { + if matches!( + self.artifacts().get_only(&tags), + Err(GetError::NotFound) + ) { + problems.push(CheckProblem::MissingArtifact(tags)); + } + } + + if let Ok(artifact) = + self.artifacts().get_only(&KnownArtifactTags::InstallinatorDocument) + { + let stream = self.read_artifact(artifact).await?; + let bytes = stream.map_ok(Vec::from).try_concat().await?; + match serde_json::from_slice::(&bytes) { + Ok(doc) => { + if doc.system_version.as_str() + != self.system_version().to_string() + { + problems.push(CheckProblem::InstallinatorVersion { + doc_version: doc.system_version.clone(), + system_version: self.system_version().clone(), + }); + } + } + Err(source) => problems + .push(CheckProblem::DeserializeInstallinator(source)), + } + } + + Ok(problems) + } +} + +/// Possible problems found by [`Repository::check_problems`]. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum CheckProblem { + /// The Installinator document could not be parsed. + #[error("failed to deserialize Installinator document")] + DeserializeInstallinator(#[source] serde_json::Error), + + /// The Installinator document has the wrong system version. + #[error( + "Installinator document has version {doc_version} \ + but the system version is {system_version}" + )] + InstallinatorVersion { + doc_version: ArtifactVersion, + system_version: semver::Version, + }, + + /// An artifact matching these tags was not found. + #[error("no artifact matching {0}")] + MissingArtifact(KnownArtifactTags), + + /// Multiple artifacts for these tags were not expected. + #[error("multiple artifacts found matching {0}")] + MultipleArtifacts(KnownArtifactTags), + + /// The repository metadata is unexpectedly different from the parsed + /// structured metadata. + #[error( + "unexpected diff in structured metadata: \ + {key}={old_value:?}->{new_value:?}" + )] + UnexpectedMetadataDiff { + key: String, + old_value: Option, + new_value: String, + }, + + /// The repository metadata cannot be parsed. + #[error("couldn't parse metadata {0:?}")] + UnknownMetadata(BTreeMap), + + /// The repository metadata cannot be converted into a string mapping, which + /// is a bug. + #[error( + "couldn't convert metadata to string mapping; \ + this is a bug in Tufaceous" + )] + ConvertMetadataMapping(#[source] serde_json::Error), + + /// The repository metadata contains an unknown key. + #[error("unknown metadata key {0:?}")] + UnknownMetadataKey(String), + + /// An artifact has unknown tags. + #[error( + "artifact {target_name} has unknown tags {}", + DisplayTags::from(.tags) + )] + UnknownTags { target_name: String, tags: BTreeMap }, +} diff --git a/lib/src/repo/v1.rs b/lib/src/repo/v1.rs new file mode 100644 index 0000000..c950a44 --- /dev/null +++ b/lib/src/repo/v1.rs @@ -0,0 +1,769 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::BTreeMap; +use std::collections::HashMap; +use std::io::BufRead; +use std::io::BufReader; +use std::io::Read; +use std::io::Write; +use std::sync::Arc; + +use bytes::Buf; +use bytes::Bytes; +use bytes::BytesMut; +use camino::FromPathBufError; +use camino::Utf8Path; +use camino::Utf8PathBuf; +use flate2::bufread::GzDecoder; +use futures_util::Stream; +use futures_util::TryStreamExt; +use futures_util::pin_mut; +use hubtools::Caboose; +use hubtools::RawHubrisArchive; +use rawzip::FileReader; +use rawzip::RangeReader; +use rawzip::ReaderAt; +use semver::Version; +use serde::Deserialize; +use sha2::Digest; +use sha2::Sha256; +use slog::Logger; +use slog::info; +use slog::o; +use slog::warn; +use tokio::sync::mpsc; +use tufaceous_artifact::Artifact; +use tufaceous_artifact::ArtifactHash; +use tufaceous_artifact::ArtifactSet; +use tufaceous_artifact::ArtifactVersion; +use tufaceous_artifact::InstallinatorDocument; +use tufaceous_artifact::KnownArtifactTags; +use tufaceous_artifact::OsBoard; +use tufaceous_artifact::OsPhase1Tags; +use tufaceous_artifact::OsPhase2Tags; +use tufaceous_artifact::OsVariant; +use tufaceous_artifact::ReadCabooseError; +use tufaceous_artifact::RotSlot; +use tufaceous_artifact::ZoneTags; + +use crate::COSMO_PHASE_1_PATH; +use crate::GIMLET_PHASE_1_PATH; +use crate::PHASE_2_PATH; +use crate::error::DebugByteString; +use crate::error::Error; +use crate::error::ErrorKind; +use crate::error::try_path; +use crate::repo::ArtifactData; +use crate::repo::read_target; +use crate::repo::read_target_json; +use crate::repo::read_target_vec; +use crate::repo::target_meta; +use crate::util::ArtifactExt; + +pub(super) struct PartialRepository { + pub(super) system_version: Version, + pub(super) artifacts: ArtifactSet, + pub(super) artifact_data: BTreeMap, +} + +impl PartialRepository { + fn insert(&mut self, artifact: Artifact, data: ArtifactData) { + self.artifacts.insert(artifact.clone()); + self.artifact_data.insert(artifact, data); + } + + fn original_target_name(&self, artifact: &Artifact) -> Option<&str> { + self.artifact_data.get(artifact).map(ArtifactData::original_target_name) + } +} + +/// Attempt to load a `tough::Repository` as if it is a Tufaceous v1 repository, +/// converting artifacts into the v2 memory representation. +/// +/// Returns `None` if the v1 `artifacts.json` wasn't found. +#[expect(clippy::too_many_lines)] +pub(crate) async fn from_loaded( + tuf_repo: &tough::Repository, + log: &Logger, +) -> Result, Error> { + let Some(V1ArtifactSetSchema { system_version, artifacts: v1_artifacts }) = + read_target_json(tuf_repo, V1ArtifactSetSchema::TARGET_NAME).await? + else { + return Ok(None); + }; + + let mut partial = PartialRepository { + system_version, + artifacts: ArtifactSet::default(), + artifact_data: BTreeMap::new(), + }; + let mut installinator_document = None; + for V1Artifact { version, kind, target } in v1_artifacts { + let (hash, length) = target_meta(tuf_repo, &target)?; + let kind = match kind { + V1ArtifactKind::Known(kind) => kind, + V1ArtifactKind::Unknown(kind) => { + warn!( + log, + "skipping artifact"; + "target_name" => &target, + "error" => "unknown v1 kind", + "kind" => kind, + ); + continue; + } + }; + let tags = match kind { + // These arms represent a single artifact, and return their tags + // from the match arm. + V1KnownArtifactKind::GimletSp + | V1KnownArtifactKind::PscSp + | V1KnownArtifactKind::SwitchSp => { + let image = read_target_vec(tuf_repo, &target).await?; + let Some(image) = image else { continue }; + let mut is_lab_image = false; + let tags = caboose_tags(image, &target, |caboose| { + if let Ok(board) = caboose.board() + && let Ok(name) = caboose.name() + && board != name + { + // This is a lab image. These are stored in the TUF repo + // for manufacturing but are not used in the control + // plane, as they can never be used in an actual rack. + info!( + log, + "skipping lab SP image"; + "board" => ?DebugByteString(board), + "name" => ?DebugByteString(name), + ); + is_lab_image = true; + } + KnownArtifactTags::from_sp_caboose(caboose) + })?; + if is_lab_image { + continue; + } + tags + } + V1KnownArtifactKind::GimletRotBootloader + | V1KnownArtifactKind::PscRotBootloader + | V1KnownArtifactKind::SwitchRotBootloader => { + let image = read_target_vec(tuf_repo, &target).await?; + let Some(image) = image else { continue }; + let tags = caboose_tags( + image, + &target, + KnownArtifactTags::from_rot_bootloader_caboose, + )?; + if partial.artifacts.get_all(&tags).iter().any(|artifact| { + if hash == artifact.hash && length == artifact.length { + let existing = partial + .original_target_name(artifact) + .unwrap_or("???"); + info!( + log, + "skipping duplicate RoT bootloader image"; + "existing" => &existing, + "skipped" => &target, + ); + true + } else { + false + } + }) { + continue; + } + tags + } + + V1KnownArtifactKind::MeasurementCorpus => { + KnownArtifactTags::MeasurementCorpus + } + + // These arms represent composite artifacts, and all diverge + // with `continue`. They use methods on `CompositeArtifact` to + // unpack and add multiple artifacts to `partial` instead of the + // single-artifact logic at the end of this match statement. + V1KnownArtifactKind::GimletRot + | V1KnownArtifactKind::PscRot + | V1KnownArtifactKind::SwitchRot => { + CompositeArtifact::unpack(tuf_repo, target) + .await? + .read_rot(log, &mut partial, version) + .await?; + continue; + } + + V1KnownArtifactKind::Host => { + CompositeArtifact::unpack(tuf_repo, target) + .await? + .read_os_image(&mut partial, OsVariant::Host, &version)?; + continue; + } + V1KnownArtifactKind::Trampoline => { + CompositeArtifact::unpack(tuf_repo, target) + .await? + .read_os_image( + &mut partial, + OsVariant::Recovery, + &version, + )?; + continue; + } + + V1KnownArtifactKind::ControlPlane => { + CompositeArtifact::unpack(tuf_repo, target) + .await? + .read_control_plane(&mut partial) + .await?; + continue; + } + + // Do not directly use any Installinator document, because it is + // written for the v1 artifacts. We need to generate a new one + // for the v2 artifacts once all of the potential artifacts are + // extracted. + V1KnownArtifactKind::InstallinatorDocument => { + installinator_document = Some((version, target)); + continue; + } + }; + + let tags = tags.to_tags().map_err(ErrorKind::ConvertKnownTagsToMap)?; + partial.insert( + Artifact { version, tags, hash, length }, + ArtifactData::Target { target_name: target }, + ); + } + + // If we found an Installinator document, generate a new one with the v2 + // artifacts. + if let Some((version, target)) = installinator_document { + generate_installinator_document(&mut partial, version, target).await?; + } + Ok(Some(partial)) +} + +#[derive(Debug, Clone)] +pub(super) struct UnpackedArtifact { + pub(super) file: Arc, + pub(super) hash: ArtifactHash, + pub(super) length: u64, +} + +impl UnpackedArtifact { + fn new_blocking( + reader: &mut dyn BufRead, + map_read_err: impl FnOnce(std::io::Error) -> ErrorKind, + ) -> Result { + let mut file = + camino_tempfile::tempfile().map_err(ErrorKind::CreateTempFile)?; + let mut hasher = Sha256::new(); + let mut length = 0u64; + loop { + let buf = match reader.fill_buf() { + Ok(buf) => buf, + Err(error) => return Err(map_read_err(error).into()), + }; + if buf.is_empty() { + break; + } + let len = buf.len(); + file.write_all(buf).map_err(|source| ErrorKind::WriteFile { + source, + path: None, + })?; + hasher.update(buf); + reader.consume(len); + length += usize64!(len); + } + let file = Arc::new(file.into()); + let hash = ArtifactHash(hasher.finalize().0); + Ok(Self { file, hash, length }) + } + + pub(crate) fn stream( + self, + log: &Logger, + original_target_name: &str, + inner_path: &Utf8Path, + ) -> impl Stream> + 'static { + let log = log.new(o!( + "stream" => format!("{}::stream", std::any::type_name::()), + "original_target_name" => original_target_name.to_owned(), + "inner_path" => inner_path.to_string(), + )); + crate::mpsc_stream::mpsc_stream(Some(log), move |tx| { + type SendError = mpsc::error::SendError>; + + let mut buf = BytesMut::with_capacity(8192); + let mut hasher = Sha256::new(); + let mut bytes_read = 0; + loop { + if buf.capacity() == 0 { + buf.reserve(8192); + } + buf.resize(buf.capacity(), 0); + match self.file.read_at(&mut buf, bytes_read) { + Ok(n) => { + buf.truncate(n); + } + Err(source) => { + let err = ErrorKind::ReadFile { source, path: None }; + return tx.blocking_send(Err(err.into())); + } + } + + let bytes = buf.split().freeze(); + if bytes.is_empty() { + break; + } + hasher.update(&bytes); + bytes_read += usize64!(bytes.len()); + tx.blocking_send(Ok(bytes))?; + } + + let msg = if self.hash != ArtifactHash(hasher.finalize().into()) { + "invalid checksum" + } else if self.length != bytes_read { + "invalid length" + } else { + // correct checksum and length + return Ok::<_, SendError>(()); + }; + let source = + std::io::Error::new(std::io::ErrorKind::InvalidData, msg); + tx.blocking_send(Err( + ErrorKind::ReadFile { source, path: None }.into() + )) + }) + } +} + +fn caboose_tags( + image: Vec, + target_name: &str, + f: impl FnOnce(&Caboose) -> Result, +) -> Result { + let caboose = try_path!( + RawHubrisArchive::from_vec(image) + .and_then(|image| image.read_caboose()), + ReadHubrisArchive, + target_name + ); + Ok(try_path!(f(&caboose), ReadCaboose, target_name)) +} + +#[derive(Debug)] +struct CompositeArtifact { + entries: HashMap, + original_target_name: String, +} + +impl CompositeArtifact { + async fn unpack( + tuf_repo: &tough::Repository, + target_name: String, + ) -> Result { + let stream = + read_target(tuf_repo, &target_name).await?.ok_or_else(|| { + ErrorKind::TargetNotFound { target_name: target_name.clone() } + })?; + pin_mut!(stream); + + let (tx, rx) = mpsc::channel(1); + let target_name_clone = target_name.clone(); + let task = tokio::task::spawn_blocking(move || { + let mut archive = + tar::Archive::new(GzDecoder::new(MpscReader::new(rx))); + let mut entries = HashMap::new(); + for entry in archive.entries().map_err(|source| { + ErrorKind::ReadCompositeArtifact { + source, + target: target_name.clone(), + } + })? { + let (entry, path) = entry + .and_then(|entry| { + let path = entry.header().path()?.into_owned(); + let path = Utf8PathBuf::try_from(path) + .map_err(FromPathBufError::into_io_error)?; + Ok((entry, path)) + }) + .map_err(|source| ErrorKind::ReadCompositeArtifact { + source, + target: target_name.clone(), + })?; + let mut entry = BufReader::new(entry); + let unpacked_artifact = + UnpackedArtifact::new_blocking(&mut entry, |source| { + ErrorKind::ReadCompositeArtifact { + source, + target: target_name.clone(), + } + })?; + entries.insert(path, unpacked_artifact); + } + Ok(Self { entries, original_target_name: target_name }) + }); + + let mut stream_interrupted = false; + while let Some(item) = stream.try_next().await? { + let Ok(()) = tx.send(item).await else { + // The receiver hung up early. We are not allowed to return `Ok` + // from this function, otherwise we have not actually verified + // any of the data we just read against its hash. + stream_interrupted = true; + break; + }; + } + drop(tx); + let result = task.await?; + if stream_interrupted && result.is_ok() { + // No, it isn't ok. + Err(ErrorKind::ReadCompositeArtifact { + source: std::io::Error::new( + std::io::ErrorKind::Interrupted, + "stream unexpectedly interrupted", + ), + target: target_name_clone, + } + .into()) + } else { + result + } + } + + async fn read_rot( + mut self, + log: &Logger, + partial: &mut PartialRepository, + version: ArtifactVersion, + ) -> Result<(), Error> { + for slot in [RotSlot::A, RotSlot::B] { + let path = Utf8PathBuf::from(match slot { + RotSlot::A => "archive-a.zip", + RotSlot::B => "archive-b.zip", + }); + let Some(UnpackedArtifact { file, hash, length }) = + self.entries.remove(&path) + else { + continue; + }; + + let mut reader = RangeReader::new(file.clone(), 0..length); + let image = tokio::task::spawn_blocking(move || { + let capacity = usize::try_from(length).unwrap_or_default(); + let mut vec = Vec::with_capacity(capacity); + reader.read_to_end(&mut vec).map(|_| vec).map_err(|source| { + Error::from(ErrorKind::ReadFile { source, path: None }) + }) + }) + .await??; + + let tags = caboose_tags( + image, + &format!("{}/{path}", self.original_target_name), + |caboose| KnownArtifactTags::from_rot_caboose(caboose, slot), + )?; + if partial.artifacts.get_all(&tags).iter().any(|artifact| { + if hash == artifact.hash && length == artifact.length { + let existing = + partial.original_target_name(artifact).unwrap_or("???"); + info!( + log, + "skipping duplicate RoT image"; + "existing_target" => &existing, + "skipped_target" => &self.original_target_name, + "skipped_inner_file" => &path.as_str(), + ); + true + } else { + false + } + }) { + continue; + } + partial.insert( + Artifact { + version: version.clone(), + tags: tags + .to_tags() + .map_err(ErrorKind::ConvertKnownTagsToMap)?, + hash, + length, + }, + ArtifactData::V1Unpacked { + file, + original_target_name: self.original_target_name.clone(), + inner_path: path, + }, + ); + } + Ok(()) + } + + fn read_os_image( + mut self, + partial: &mut PartialRepository, + os_variant: OsVariant, + version: &ArtifactVersion, + ) -> Result<(), Error> { + for (file_name, tags) in [ + ( + COSMO_PHASE_1_PATH, + KnownArtifactTags::OsPhase1(OsPhase1Tags { + os_variant, + os_board: OsBoard::COSMO, + }), + ), + ( + GIMLET_PHASE_1_PATH, + KnownArtifactTags::OsPhase1(OsPhase1Tags { + os_variant, + os_board: OsBoard::GIMLET, + }), + ), + ( + PHASE_2_PATH, + KnownArtifactTags::OsPhase2(OsPhase2Tags { os_variant }), + ), + ] { + let path = Utf8PathBuf::from(format!("image/{file_name}")); + let Some(entry) = self.entries.remove(&path) else { + continue; + }; + partial.insert( + Artifact { + version: version.clone(), + tags: tags + .to_tags() + .map_err(ErrorKind::ConvertKnownTagsToMap)?, + hash: entry.hash, + length: entry.length, + }, + ArtifactData::V1Unpacked { + file: entry.file, + original_target_name: self.original_target_name.clone(), + inner_path: path, + }, + ); + } + Ok(()) + } + + async fn read_control_plane( + self, + partial: &mut PartialRepository, + ) -> Result<(), Error> { + for (tar_path, UnpackedArtifact { file, hash, length }) in self.entries + { + if !tar_path.starts_with("zones/") { + continue; + } + let (file, layer_info) = crate::util::read_zone_layer_info( + RangeReader::new(file, 0..length), + Utf8Path::new(&self.original_target_name).join(&tar_path), + ) + .await?; + let file = file.into_inner(); + let tags = + KnownArtifactTags::Zone(ZoneTags { zone_name: layer_info.pkg }); + partial.insert( + Artifact { + version: layer_info.version, + tags: tags + .to_tags() + .map_err(ErrorKind::ConvertKnownTagsToMap)?, + hash, + length, + }, + ArtifactData::V1Unpacked { + file, + original_target_name: self.original_target_name.clone(), + inner_path: tar_path, + }, + ); + } + Ok(()) + } +} + +struct MpscReader { + rx: mpsc::Receiver, + buf: Bytes, +} + +impl MpscReader { + fn new(rx: mpsc::Receiver) -> Self { + Self { rx, buf: Bytes::new() } + } +} + +impl Read for MpscReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.fill_buf()?; + let len = self.buf.len().min(buf.len()); + self.buf.copy_to_slice(&mut buf[..len]); + Ok(len) + } +} + +impl BufRead for MpscReader { + fn fill_buf(&mut self) -> std::io::Result<&[u8]> { + if self.buf.is_empty() + && let Some(next) = self.rx.blocking_recv() + { + self.buf = next; + } + Ok(&self.buf) + } + + fn consume(&mut self, amount: usize) { + self.buf.advance(amount); + } +} + +async fn generate_installinator_document( + partial: &mut PartialRepository, + version: ArtifactVersion, + original_target_name: String, +) -> Result<(), Error> { + let mut document = InstallinatorDocument::empty(version.clone()); + for (artifact, data) in &partial.artifact_data { + let target_name = match data { + ArtifactData::Target { target_name } => target_name, + ArtifactData::V1Unpacked { inner_path, .. } => inner_path.as_str(), + }; + if let Some(installinator) = artifact.to_installinator(target_name) { + document.artifacts.insert(installinator); + } + } + + let mut json = serde_json::to_string_pretty(&document) + .map_err(ErrorKind::SerializeInstallinator)?; + json.push('\n'); + let unpacked_artifact = tokio::task::spawn_blocking(move || { + UnpackedArtifact::new_blocking(&mut json.as_bytes(), |_error| { + unreachable!("Read::read for &[u8] does not return an error") + }) + }) + .await??; + + partial.insert( + Artifact { + version, + tags: KnownArtifactTags::InstallinatorDocument + .to_tags() + .map_err(ErrorKind::ConvertKnownTagsToMap)?, + hash: unpacked_artifact.hash, + length: unpacked_artifact.length, + }, + ArtifactData::V1Unpacked { + file: unpacked_artifact.file, + original_target_name, + inner_path: "v2.json".into(), + }, + ); + Ok(()) +} + +#[derive(Debug, Deserialize)] +pub(crate) struct V1ArtifactSetSchema { + system_version: Version, + artifacts: Vec, +} + +impl V1ArtifactSetSchema { + pub(crate) const TARGET_NAME: &str = "artifacts.json"; +} + +#[derive(Debug, Deserialize)] +struct V1Artifact { + version: ArtifactVersion, + kind: V1ArtifactKind, + target: String, +} + +#[derive(Debug, Deserialize)] +#[serde(untagged)] +enum V1ArtifactKind { + Known(V1KnownArtifactKind), + Unknown(String), +} + +#[derive(Debug, Deserialize, Clone, Copy)] +#[serde(rename_all = "snake_case")] +enum V1KnownArtifactKind { + GimletSp, + GimletRot, + GimletRotBootloader, + Host, + Trampoline, + InstallinatorDocument, + ControlPlane, + MeasurementCorpus, + PscSp, + PscRot, + PscRotBootloader, + SwitchSp, + SwitchRot, + SwitchRotBootloader, +} + +#[cfg(test)] +mod tests { + use std::io::Read; + + use bytes::Bytes; + use tokio::sync::mpsc; + + use crate::repo::v1::MpscReader; + + #[tokio::test] + async fn mpsc_reader() { + static CHUNKS: [Bytes; 5] = [ + Bytes::from_static(b"hello world"), + Bytes::from_static(&[0x5a; 512]), + Bytes::from_static(b"meow meow meow meow\0"), + Bytes::from_static(&[0; 12345]), + Bytes::from_static(&[0x5a; 256]), + ]; + + let expected = CHUNKS.concat(); + + let (tx, rx) = mpsc::channel(1); + let task = tokio::task::spawn_blocking(move || { + let mut bytes_read = Vec::new(); + let mut reader = MpscReader::new(rx); + let mut buf = [0; 2048]; + while let n = reader.read(&mut buf).unwrap() + && n > 0 + { + bytes_read.extend_from_slice(&buf[..n]); + } + bytes_read + }); + for chunk in &CHUNKS { + tx.send(chunk.clone()).await.unwrap(); + } + drop(tx); + let bytes_read = task.await.unwrap(); + assert_eq!(bytes_read, expected); + } + + #[tokio::test] + async fn mpsc_reader_empty() { + let (tx, rx) = mpsc::channel(1); + let task = tokio::task::spawn_blocking(move || { + let mut bytes_read = Vec::new(); + MpscReader::new(rx).read_to_end(&mut bytes_read).unwrap(); + bytes_read + }); + drop(tx); + let bytes_read = task.await.unwrap(); + assert!(bytes_read.is_empty()); + } +} diff --git a/lib/src/repository.rs b/lib/src/repository.rs deleted file mode 100644 index f9c99fd..0000000 --- a/lib/src/repository.rs +++ /dev/null @@ -1,670 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use std::collections::BTreeSet; -use std::num::NonZeroU64; - -use anyhow::{Context, Result, anyhow}; -use buf_list::BufList; -use camino::{Utf8Path, Utf8PathBuf}; -use chrono::{DateTime, Utc}; -use fs_err as fs; -use futures::TryStreamExt; -use semver::Version; -use tough::editor::RepositoryEditor; -use tough::editor::signed::SignedRole; -use tough::error::Error; -use tough::schema::{Root, Target}; -use tough::{ExpirationEnforcement, Repository, RepositoryLoader, TargetName}; -use tufaceous_artifact::{ - Artifact, ArtifactHash, ArtifactVersion, ArtifactsDocument, - InstallinatorDocument, KnownArtifactKind, -}; -use url::Url; - -use crate::assemble::{ - ArtifactDeploymentUnits, DeploymentUnitData, DeploymentUnitMapBuilder, - DeploymentUnitScope, -}; -use crate::key::Key; -use crate::target::TargetWriter; -use crate::utils::merge_anyhow_list; -use crate::{AddArtifact, ArchiveBuilder}; - -/// A TUF repository describing Omicron. -#[derive(Debug)] -pub struct OmicronRepo { - log: slog::Logger, - repo: Repository, - repo_path: Utf8PathBuf, -} - -impl OmicronRepo { - /// Loads a repository from the given path. - /// - #[cfg(test)] - /// This method enforces expirations. To load without expiration enforcement, use - /// [`Self::load_ignore_expiration`]. - pub async fn load( - log: &slog::Logger, - repo_path: &Utf8Path, - trusted_roots: impl IntoIterator>, - ) -> Result { - Self::load_impl( - log, - repo_path, - trusted_roots, - ExpirationEnforcement::Safe, - ) - .await - } - - /// Loads a repository from the given path, ignoring expiration. - /// - /// Use cases for this include: - /// - /// 1. When you're editing an existing repository and will re-sign it afterwards. - /// 2. When you're reading a repository that was uploaded out-of-band, - /// instead of fetched from a network-accessible repository - /// 3. In an environment in which time isn't available. - pub async fn load_ignore_expiration( - log: &slog::Logger, - repo_path: &Utf8Path, - trusted_roots: impl IntoIterator>, - ) -> Result { - Self::load_impl( - log, - repo_path, - trusted_roots, - ExpirationEnforcement::Unsafe, - ) - .await - } - - async fn load_impl( - log: &slog::Logger, - repo_path: &Utf8Path, - trusted_roots: impl IntoIterator>, - exp: ExpirationEnforcement, - ) -> Result { - let repo_path = repo_path.canonicalize_utf8()?; - let mut verify_error = None; - for root in trusted_roots { - match RepositoryLoader::new( - &root, - Url::from_file_path(repo_path.join("metadata")) - .expect("the canonical path is not absolute?"), - Url::from_file_path(repo_path.join("targets")) - .expect("the canonical path is not absolute?"), - ) - .expiration_enforcement(exp) - .load() - .await - { - Ok(repo) => { - return Ok(Self { - log: log.new(slog::o!("component" => "OmicronRepo")), - repo, - repo_path, - }); - } - Err( - err @ (Error::VerifyMetadata { .. } - | Error::VerifyTrustedMetadata { .. }), - ) => { - verify_error = Some(err.into()); - continue; - } - Err(err) => return Err(err.into()), - } - } - Err(verify_error.unwrap_or_else(|| anyhow!("trust store is empty"))) - } - - /// Loads a repository from the given path. - /// - /// This method enforces expirations. To load without expiration enforcement, use - /// [`Self::load_untrusted_ignore_expiration`]. - pub async fn load_untrusted( - log: &slog::Logger, - repo_path: &Utf8Path, - ) -> Result { - Self::load_untrusted_impl(log, repo_path, ExpirationEnforcement::Safe) - .await - } - - /// Loads a repository from the given path, ignoring expiration. - /// - /// Use cases for this include: - /// - /// 1. When you're editing an existing repository and will re-sign it afterwards. - /// 2. When you're reading a repository that was uploaded out-of-band, - /// instead of fetched from a network-accessible repository - /// 3. In an environment in which time isn't available. - pub async fn load_untrusted_ignore_expiration( - log: &slog::Logger, - repo_path: &Utf8Path, - ) -> Result { - Self::load_untrusted_impl(log, repo_path, ExpirationEnforcement::Unsafe) - .await - } - - async fn load_untrusted_impl( - log: &slog::Logger, - repo_path: &Utf8Path, - exp: ExpirationEnforcement, - ) -> Result { - let repo_path = repo_path.canonicalize_utf8()?; - let root_json = repo_path.join("metadata").join("1.root.json"); - let root = tokio::fs::read(&root_json) - .await - .with_context(|| format!("error reading from {root_json}"))?; - Self::load_impl(log, &repo_path, &[root], exp).await - } - - /// Returns a canonicalized form of the repository path. - pub fn repo_path(&self) -> &Utf8Path { - &self.repo_path - } - - /// Returns the repository. - pub fn repo(&self) -> &Repository { - &self.repo - } - - /// Reads the artifacts document from the repo. - pub async fn read_artifacts(&self) -> Result { - self.read_json(ArtifactsDocument::FILE_NAME).await - } - - /// Reads the installinator document from the repo. - pub async fn read_installinator_document( - &self, - file_name: &str, - ) -> Result { - self.read_json(file_name).await - } - - /// Reads a JSON document from the repo by target name. - async fn read_json(&self, file_name: &str) -> Result - where - T: serde::de::DeserializeOwned, - { - let reader = self - .repo - .read_target(&file_name.try_into()?) - .await? - .ok_or_else(|| anyhow!("{} should be present", file_name))?; - let buf_list = reader - .try_collect::() - .await - .with_context(|| format!("error reading from {}", file_name))?; - serde_json::from_reader(buf_list::Cursor::new(&buf_list)) - .with_context(|| format!("error deserializing {}", file_name)) - } - - /// Archives the repository to the given path as a zip file. - /// - /// ## Why zip and not tar? - /// - /// The main reason is that zip supports random access to files and tar does - /// not. - /// - /// In principle it should be possible to read the repository out of a zip - /// file from memory, but we ran into [this - /// issue](https://github.com/awslabs/tough/pull/563) while implementing it. - /// Once that is resolved (or we write our own TUF crate) it should be - /// possible to do that. - /// - /// Regardless of this roadblock, we don't want to foreclose that option - /// forever, so this code uses zip rather than having to deal with a - /// migration in the future. - pub(crate) fn archive(&self, output_path: &Utf8Path) -> Result<()> { - let mut builder = ArchiveBuilder::new(output_path.to_owned())?; - - let metadata_dir = self.repo_path.join("metadata"); - - // Gather metadata files. - for entry in metadata_dir.read_dir_utf8().with_context(|| { - format!("error reading entries from {metadata_dir}") - })? { - let entry = - entry.context("error reading entry from {metadata_dir}")?; - let file_name = entry.file_name(); - if file_name.ends_with(".root.json") - || file_name == "timestamp.json" - || file_name.ends_with(".snapshot.json") - || file_name.ends_with(".targets.json") - { - // This is a valid metadata file. - builder.write_file( - entry.path(), - &Utf8Path::new("metadata").join(file_name), - )?; - } - } - - let targets_dir = self.repo_path.join("targets"); - - // Gather all targets. - for (name, target) in self.repo.targets().signed.targets_iter() { - let target_filename = self.target_filename(target, name); - let target_path = targets_dir.join(&target_filename); - slog::trace!(self.log, "adding {} to archive", name.resolved()); - builder.write_file( - &target_path, - &Utf8Path::new("targets").join(&target_filename), - )?; - } - - builder.finish()?; - - Ok(()) - } - - /// Prepends the target digest to the name if using consistent snapshots. Returns both the - /// digest and the filename. - /// - /// Adapted from tough's source. - fn target_filename(&self, target: &Target, name: &TargetName) -> String { - let sha256 = &target.hashes.sha256.clone().into_vec(); - if self.repo.root().signed.consistent_snapshot { - format!("{}.{}", hex::encode(sha256), name.resolved()) - } else { - name.resolved().to_owned() - } - } -} - -/// An editable TUF repository, used to construct new ones. -pub(crate) struct OmicronRepoEditor { - editor: RepositoryEditor, - repo_path: Utf8PathBuf, - artifacts: ArtifactsDocument, - installinator_document: InstallinatorDocument, - - // Set of `TargetName::resolved()` names for every target that existed when - // the repo was opened. We use this to ensure we don't overwrite an existing - // target when adding new artifacts. - existing_target_names: BTreeSet, - // Set of (kind, hash) pairs for every artifact and deployment unit known to - // the repo. Used to ensure (kind, hash) pairs are unique within this repo. - existing_deployment_units: DeploymentUnitMapBuilder, -} - -impl OmicronRepoEditor { - pub(crate) async fn initialize( - repo_path: Utf8PathBuf, - root: SignedRole, - system_version: Version, - ) -> Result { - let metadata_dir = repo_path.join("metadata"); - let targets_dir = repo_path.join("targets"); - let root_path = metadata_dir - .join(format!("{}.root.json", root.signed().signed.version)); - - fs::create_dir_all(&metadata_dir)?; - fs::create_dir_all(&targets_dir)?; - fs::write(&root_path, root.buffer())?; - - let editor = RepositoryEditor::new(&root_path).await?; - - Ok(Self { - editor, - repo_path, - artifacts: ArtifactsDocument::empty(system_version.clone()), - installinator_document: InstallinatorDocument::empty( - system_version, - ), - existing_target_names: BTreeSet::new(), - existing_deployment_units: DeploymentUnitMapBuilder::new( - DeploymentUnitScope::Repository, - ), - }) - } - - /// Adds an artifact to the repository. - pub(crate) fn add_artifact( - &mut self, - new_artifact: &AddArtifact, - ) -> Result { - let target_name = format!( - "{}-{}-{}.tar.gz", - new_artifact.kind(), - new_artifact.name(), - new_artifact.version(), - ); - - let mut errors = Vec::new(); - - // Make sure we're not overwriting an existing target (either one that - // existed when we opened the repo, or one that's been added via this - // method) - if self.existing_target_names.contains(&target_name) { - errors.push(anyhow!( - "a target named {target_name} already exists in the repository", - )); - } - - // Start writing the target out to a temporary path, catching errors - // that might happen. - let targets_dir = self.repo_path.join("targets"); - let new_artifact = new_artifact.write_temp(&targets_dir)?; - - // Make sure we're not adding a new deployment unit with the same - // kind/hash as an existing one. - let res = match new_artifact.deployment_units() { - ArtifactDeploymentUnits::SingleUnit - | ArtifactDeploymentUnits::Unknown => { - // For single-unit artifacts, the artifact itself is the - // deployment unit. For unknown artifacts, we don't know, but - // treat them as single-unit. - self.existing_deployment_units.start_insert( - DeploymentUnitData { - name: new_artifact.name().to_owned(), - version: new_artifact.version().clone(), - kind: new_artifact.kind().clone(), - hash: new_artifact.digest(), - }, - ) - } - ArtifactDeploymentUnits::Composite { deployment_units } => { - // For composite artifacts, merge the deployment units. - self.existing_deployment_units - .start_bulk_insert(deployment_units.clone()) - } - }; - let new_units = match res { - Ok(units) => Some(units), - Err(error) => { - errors.push(anyhow!(error)); - None - } - }; - - let version = - match ArtifactVersion::new(new_artifact.version().to_string()) { - Ok(version) => Some(version), - Err(error) => { - errors.push( - anyhow!(error).context("invalid artifact version"), - ); - None - } - }; - - if !errors.is_empty() { - return Err(merge_anyhow_list(errors)); - } - - // --- - // No errors past this point. - // --- - - self.existing_target_names.insert(target_name.clone()); - self.artifacts.artifacts.push(Artifact { - name: new_artifact.name().to_owned(), - version: version.expect("version is None => errors handled above"), - kind: new_artifact.kind().clone(), - target: target_name, - }); - self.installinator_document - .artifacts - .extend(new_artifact.installinator_artifacts()); - // The host phase 2 image is part of the host image. - new_units.expect("new_units is None => errors handled above").commit(); - - new_artifact.finalize(&mut self.editor) - } - - /// Consumes self, signing the repository and writing out this repository to - /// disk. - pub(crate) async fn sign_and_finish( - mut self, - keys: Vec, - expiry: DateTime, - include_installinator_doc: bool, - ) -> Result<()> { - let targets_dir = self.repo_path.join("targets"); - - if include_installinator_doc { - let mut installinator_doc_writer = TargetWriter::new( - &targets_dir, - self.installinator_document.file_name(), - )?; - serde_json::to_writer_pretty( - &mut installinator_doc_writer, - &self.installinator_document, - )?; - installinator_doc_writer - .finish_write() - .finalize(&mut self.editor)?; - - // Add the installinator document in the artifacts.json if it's missing. - // - // TODO: our production users don't add artifacts incrementally to a TUF - // repo -- rather, they generate a manifest and create the TUF repo in a - // one-shot fashion. We should clean up any code we have to handle - // incremental updates of TUF repos, and remove this scan as part of - // that. - let system_version = self.artifacts.system_version.clone(); - let artifact_version = - ArtifactVersion::new(system_version.to_string()) - .expect("system versions are usable as artifact versions"); - if !self.artifacts.artifacts.iter().any(|artifact| { - artifact.kind.to_known() - == Some(KnownArtifactKind::InstallinatorDocument) - }) { - self.artifacts.artifacts.push(Artifact { - name: KnownArtifactKind::InstallinatorDocument.to_string(), - version: artifact_version, - kind: KnownArtifactKind::InstallinatorDocument.into(), - target: self.installinator_document.file_name(), - }); - } - } - - let mut artifacts_doc_writer = - TargetWriter::new(&targets_dir, ArtifactsDocument::FILE_NAME)?; - serde_json::to_writer_pretty( - &mut artifacts_doc_writer, - &self.artifacts, - )?; - artifacts_doc_writer.finish_write().finalize(&mut self.editor)?; - - update_versions(&mut self.editor, expiry)?; - - let signed = self - .editor - .sign(&crate::key::boxed_keys(keys)) - .await - .context("error signing keys")?; - signed - .write(self.repo_path.join("metadata")) - .await - .context("error writing repository")?; - Ok(()) - } -} - -fn update_versions( - editor: &mut RepositoryEditor, - expiry: DateTime, -) -> Result<()> { - let version = u64::try_from(Utc::now().timestamp()) - .and_then(NonZeroU64::try_from) - .expect("bad epoch"); - editor.snapshot_version(version); - editor.targets_version(version)?; - editor.timestamp_version(version); - editor.snapshot_expires(expiry); - editor.targets_expires(expiry)?; - editor.timestamp_expires(expiry); - Ok(()) -} - -#[cfg(test)] -mod tests { - use buf_list::BufList; - use camino_tempfile::Utf8TempDir; - use chrono::Days; - use dropshot::test_util::LogContext; - use dropshot::{ConfigLogging, ConfigLoggingIfExists, ConfigLoggingLevel}; - - use crate::assemble::{ - ArtifactDeploymentUnits, ArtifactManifest, OmicronRepoAssembler, - }; - use crate::{ArchiveExtractor, ArtifactSource}; - - use super::*; - - #[tokio::test] - async fn load_trusted() { - let log_config = ConfigLogging::File { - level: ConfigLoggingLevel::Trace, - path: "UNUSED".into(), - if_exists: ConfigLoggingIfExists::Fail, - }; - let logctx = LogContext::new( - "reject_artifacts_with_the_same_filename", - &log_config, - ); - - // Generate a "trusted" root and an "untrusted" root. - let expiry = Utc::now() + Days::new(1); - let trusted_key = Key::generate_ed25519().unwrap(); - let trusted_root = - crate::root::new_root(vec![trusted_key.clone()], expiry) - .await - .unwrap(); - let untrusted_key = Key::generate_ed25519().unwrap(); - let untrusted_root = - crate::root::new_root(vec![untrusted_key], expiry).await.unwrap(); - - // Generate a repository using the trusted root. - let tempdir = Utf8TempDir::new().unwrap(); - let archive_path = tempdir.path().join("repo.zip"); - let mut assembler = OmicronRepoAssembler::new( - &logctx.log, - ArtifactManifest::new_fake(), - vec![trusted_key], - expiry, - true, - archive_path.clone(), - ); - assembler.set_root_role(trusted_root.clone()); - assembler.build().await.unwrap(); - // And now that we've created an archive and cleaned up the build - // directory, immediately unarchive it... this is a bit silly, huh? - let repo_dir = tempdir.path().join("repo"); - ArchiveExtractor::from_path(&archive_path) - .unwrap() - .extract(&repo_dir) - .unwrap(); - - // If the trust store contains the root we generated the repo from, we - // should successfully load it. - for trust_store in [ - vec![trusted_root.buffer()], - vec![trusted_root.buffer(), untrusted_root.buffer()], - vec![untrusted_root.buffer(), trusted_root.buffer()], - vec![trusted_root.buffer(), trusted_root.buffer()], - vec![ - untrusted_root.buffer(), - untrusted_root.buffer(), - trusted_root.buffer(), - ], - ] { - OmicronRepo::load(&logctx.log, &repo_dir, trust_store) - .await - .unwrap(); - } - // If the trust store is empty, we should fail. - assert_eq!( - OmicronRepo::load(&logctx.log, &repo_dir, [] as [Vec; 0]) - .await - .unwrap_err() - .to_string(), - "trust store is empty" - ); - // If the trust store otherwise does not contain the root we generated - // the repo from, we should also fail. - for trust_store in [ - vec![untrusted_root.buffer()], - vec![untrusted_root.buffer(), untrusted_root.buffer()], - ] { - assert_eq!( - OmicronRepo::load(&logctx.log, &repo_dir, trust_store) - .await - .unwrap_err() - .to_string(), - "Failed to verify timestamp metadata: \ - Signature threshold of 1 not met for role timestamp \ - (0 valid signatures)" - ) - } - - logctx.cleanup_successful(); - } - - #[tokio::test] - async fn reject_artifacts_with_the_same_filename() { - let log_config = ConfigLogging::File { - level: ConfigLoggingLevel::Trace, - path: "UNUSED".into(), - if_exists: ConfigLoggingIfExists::Fail, - }; - let logctx = LogContext::new( - "reject_artifacts_with_the_same_filename", - &log_config, - ); - let tempdir = Utf8TempDir::new().unwrap(); - let keys = vec![Key::generate_ed25519().unwrap()]; - let expiry = Utc::now() + Days::new(1); - let root = crate::root::new_root(keys.clone(), expiry).await.unwrap(); - let mut repo = OmicronRepoEditor::initialize( - tempdir.path().to_owned(), - root, - "0.0.0".parse().unwrap(), - ) - .await - .unwrap(); - - // Targets are uniquely identified by their kind/name/version triple; - // trying to add two artifacts with identical triples should fail. - let kind = "test-kind"; - let name = "test-artifact-name"; - let version = "1.0.0"; - - repo.add_artifact(&AddArtifact::new( - kind.parse().unwrap(), - name.to_string(), - version.parse().unwrap(), - ArtifactSource::Memory(BufList::from("first")), - ArtifactDeploymentUnits::Unknown, - )) - .unwrap(); - - let err = repo - .add_artifact(&AddArtifact::new( - kind.parse().unwrap(), - name.to_string(), - version.parse().unwrap(), - ArtifactSource::Memory(BufList::from("second")), - ArtifactDeploymentUnits::Unknown, - )) - .unwrap_err(); - - println!("error: {:?}", err); - let err = err.to_string(); - - assert!(err.contains("a target named")); - assert!(err.contains(kind)); - assert!(err.contains(name)); - assert!(err.contains(version)); - assert!(err.contains("already exists")); - - logctx.cleanup_successful(); - } -} diff --git a/lib/src/root.rs b/lib/src/root.rs deleted file mode 100644 index dc64d37..0000000 --- a/lib/src/root.rs +++ /dev/null @@ -1,57 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use std::collections::HashMap; -use std::num::NonZeroU64; - -use anyhow::Result; -use aws_lc_rs::rand::SystemRandom; -use chrono::{DateTime, Utc}; -use tough::editor::signed::SignedRole; -use tough::schema::{KeyHolder, RoleKeys, RoleType, Root}; - -use crate::key::Key; - -pub async fn new_root( - keys: Vec, - expires: DateTime, -) -> Result> { - let mut root = Root { - spec_version: "1.0.0".to_string(), - consistent_snapshot: true, - version: NonZeroU64::new(1).unwrap(), - expires, - keys: HashMap::new(), - roles: HashMap::new(), - _extra: HashMap::new(), - }; - for key in &keys { - let key = key.as_tuf_key()?; - root.keys.insert(key.key_id()?, key); - } - for kind in [ - RoleType::Root, - RoleType::Snapshot, - RoleType::Targets, - RoleType::Timestamp, - ] { - root.roles.insert( - kind, - RoleKeys { - keyids: root.keys.keys().cloned().collect(), - threshold: NonZeroU64::new(1).unwrap(), - _extra: HashMap::new(), - }, - ); - } - - let keys = crate::key::boxed_keys(keys); - Ok(SignedRole::new( - root.clone(), - &KeyHolder::Root(root), - &keys, - &SystemRandom::new(), - ) - .await?) -} diff --git a/lib/src/schema.rs b/lib/src/schema.rs new file mode 100644 index 0000000..b40dd67 --- /dev/null +++ b/lib/src/schema.rs @@ -0,0 +1,43 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::BTreeMap; + +use semver::Version; +use serde::Deserialize; +use serde::Serialize; +use tufaceous_artifact::ArtifactVersion; +use tufaceous_artifact::KnownArtifactTags; + +#[derive(Debug, Deserialize, Serialize)] +pub(crate) struct ArtifactSetSchema { + pub(crate) system_version: Version, + #[serde(default)] + pub(crate) artifacts: Vec, + #[serde(default)] + pub(crate) metadata: BTreeMap, +} + +impl ArtifactSetSchema { + pub(crate) const TARGET_NAME: &str = "artifacts-v2.json"; +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, Hash)] +pub(crate) struct ArtifactSchema { + pub(crate) target_name: String, + pub(crate) version: ArtifactVersion, + pub(crate) tags: BTreeMap, +} + +impl ArtifactSchema { + pub(crate) fn known_tags(&self) -> Option { + KnownArtifactTags::from_tags(self.tags.clone()).ok() + } +} + +impl AsRef for ArtifactSchema { + fn as_ref(&self) -> &ArtifactSchema { + self + } +} diff --git a/lib/src/target.rs b/lib/src/target.rs deleted file mode 100644 index f6e4054..0000000 --- a/lib/src/target.rs +++ /dev/null @@ -1,106 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use std::collections::HashMap; -use std::io::Write; - -use anyhow::Result; -use camino::Utf8PathBuf; -use camino_tempfile::NamedUtf8TempFile; -use sha2::{Digest, Sha256}; -use tough::editor::RepositoryEditor; -use tough::schema::{Hashes, Target}; -use tufaceous_artifact::ArtifactHash; - -pub(crate) struct TargetWriter { - file: NamedUtf8TempFile, - targets_dir: Utf8PathBuf, - name: String, - length: u64, - hasher: Sha256, -} - -impl TargetWriter { - pub(crate) fn new( - targets_dir: impl Into, - name: impl Into, - ) -> Result { - let targets_dir = targets_dir.into(); - Ok(TargetWriter { - file: NamedUtf8TempFile::new_in(&targets_dir)?, - targets_dir, - name: name.into(), - length: 0, - hasher: Sha256::default(), - }) - } - - /// Marks that writing has been completed, though the file is still - /// temporary and not persisted yet. - /// - /// The main goal is to provide a way to obtain the hash of the file without - /// persisting it. - pub(crate) fn finish_write(self) -> TargetFinishWrite { - let digest = self.hasher.finalize(); - TargetFinishWrite { - file: self.file, - targets_dir: self.targets_dir, - name: self.name, - length: self.length, - digest: ArtifactHash(digest.into()), - } - } -} - -#[must_use = "the file is still temporary and must be finalized"] -pub(crate) struct TargetFinishWrite { - file: NamedUtf8TempFile, - targets_dir: Utf8PathBuf, - name: String, - length: u64, - digest: ArtifactHash, -} - -impl TargetFinishWrite { - pub(crate) fn digest(&self) -> ArtifactHash { - self.digest - } - - pub(crate) fn finalize( - self, - editor: &mut RepositoryEditor, - ) -> Result { - self.file.persist(self.targets_dir.join(format!( - "{}.{}", - hex::encode(self.digest.0), - self.name - )))?; - editor.add_target( - self.name, - Target { - length: self.length, - hashes: Hashes { - sha256: Vec::from(self.digest.0).into(), - _extra: HashMap::new(), - }, - custom: HashMap::new(), - _extra: HashMap::new(), - }, - )?; - Ok(self.digest) - } -} - -impl Write for TargetWriter { - fn write(&mut self, buf: &[u8]) -> std::io::Result { - let n = self.file.write(buf)?; - self.length += u64::try_from(n).unwrap(); - self.hasher.update(&buf[..n]); - Ok(n) - } - - fn flush(&mut self) -> std::io::Result<()> { - self.file.flush() - } -} diff --git a/lib/src/util.rs b/lib/src/util.rs new file mode 100644 index 0000000..97a7146 --- /dev/null +++ b/lib/src/util.rs @@ -0,0 +1,133 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::BTreeMap; +use std::io::Read; +use std::pin::Pin; +use std::task::Context; +use std::task::Poll; + +use camino::FromPathBufError; +use camino::Utf8Path; +use camino::Utf8PathBuf; +use flate2::read::GzDecoder; +use futures_util::Stream; +use tufaceous_artifact::Artifact; +use tufaceous_artifact::ArtifactHash; +use tufaceous_artifact::InstallinatorArtifact; +use tufaceous_artifact::KnownArtifactTags; +use tufaceous_brand_metadata::LayerInfo; +use tufaceous_brand_metadata::Metadata; + +use crate::error::Error; +use crate::error::ErrorKind; +use crate::error::try_path; + +pub(crate) trait ArtifactExt { + fn to_installinator( + &self, + target_name: &str, + ) -> Option; +} + +impl ArtifactExt for Artifact { + fn to_installinator( + &self, + target_name: &str, + ) -> Option { + installinator_artifact(self.tags.clone(), self.hash, target_name) + } +} + +pub(crate) fn installinator_artifact( + tags: BTreeMap, + hash: ArtifactHash, + target_name: &str, +) -> Option { + Some(InstallinatorArtifact { + kind: KnownArtifactTags::from_tags(tags).ok()?.to_installinator()?, + hash, + file_name: Utf8Path::new(target_name).file_name()?.to_string(), + }) +} + +pub(crate) async fn read_zone_layer_info( + reader: R, + path: Utf8PathBuf, +) -> Result<(R, LayerInfo), Error> { + tokio::task::spawn_blocking(move || { + let mut archive = tar::Archive::new(GzDecoder::new(reader)); + let layer_info = Metadata::read_from_tar(&mut archive) + .and_then(|metadata| metadata.layer_info().cloned()) + .map_err(|source| ErrorKind::ReadZoneOxideJson { source, path })?; + Ok((archive.into_inner().into_inner(), layer_info)) + }) + .await? +} + +pub(crate) async fn read_dir(path: Utf8PathBuf) -> Result { + let inner = try_path!(tokio::fs::read_dir(&path).await, ReadDir, path); + Ok(ReadDir { inner, path }) +} + +pub(crate) struct ReadDir { + inner: tokio::fs::ReadDir, + path: Utf8PathBuf, +} + +impl Stream for ReadDir { + type Item = Result; + + fn poll_next( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + self.inner.poll_next_entry(cx).map(|result| { + result + .and_then(|option| option.map(DirEntry::new).transpose()) + .map_err(|source| { + ErrorKind::ReadDir { source, path: Some(self.path.clone()) } + .into() + }) + .transpose() + }) + } +} + +pub(crate) struct DirEntry { + path: Utf8PathBuf, +} + +impl DirEntry { + #[expect(clippy::needless_pass_by_value)] + fn new(inner: tokio::fs::DirEntry) -> Result { + let path = + inner.path().try_into().map_err(FromPathBufError::into_io_error)?; + Ok(Self { path }) + } + + pub(crate) fn into_path(self) -> Utf8PathBuf { + self.path + } + + pub(crate) fn path(&self) -> &Utf8Path { + &self.path + } + + pub(crate) fn file_name(&self) -> &str { + self.path + .file_name() + .expect("path created through DirEntry must have a file name") + } +} + +pub(crate) fn error_chain(mut error: &dyn std::error::Error) -> String { + let mut s = error.to_string(); + while let Some(source) = error.source() { + s.push_str(": "); + s.push_str(&source.to_string()); + error = source; + } + s +} diff --git a/lib/src/utils.rs b/lib/src/utils.rs deleted file mode 100644 index c2ae66b..0000000 --- a/lib/src/utils.rs +++ /dev/null @@ -1,60 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use std::fmt::Write; - -use anyhow::anyhow; -use indent_write::fmt::IndentWriter; - -/// A temporary hack to convert a list of anyhow errors into a single -/// `anyhow::Error`. If no errors are provided, panic (this should be handled -/// at a higher level). -/// -/// Eventually we should gain first-class support for representing errors as -/// trees, but this will do for now. -pub(crate) fn merge_anyhow_list(errors: I) -> anyhow::Error -where - I: IntoIterator, -{ - let mut iter = errors.into_iter().peekable(); - // How many errors are there? - let Some(first_error) = iter.next() else { - // No errors: panic. - panic!("error_list_to_anyhow called with no errors"); - }; - - if iter.peek().is_none() { - // One error. - return first_error; - } - - // Multiple errors. - let mut out = String::new(); - let mut nerrors = 0; - for error in std::iter::once(first_error).chain(iter) { - if nerrors > 0 { - // Separate errors with a newline (we want there to not be a - // trailing newline to match anyhow generally). - writeln!(&mut out).unwrap(); - } - nerrors += 1; - let mut current: &dyn std::error::Error = error.as_ref(); - - let mut writer = IndentWriter::new_skip_initial(" ", &mut out); - write!(writer, "Error: {current}").unwrap(); - - while let Some(cause) = current.source() { - // This newline is not part of the `IndentWriter`'s output so that - // it is unaffected by the indent logic. - writeln!(&mut out).unwrap(); - - // The spaces align the causes with the "Error: " above. - let mut writer = - IndentWriter::new_skip_initial(" ", &mut out); - write!(writer, " - {cause}").unwrap(); - current = cause; - } - } - anyhow!(out).context(format!("{nerrors} errors encountered")) -} diff --git a/lib/src/zip_transport.rs b/lib/src/zip_transport.rs new file mode 100644 index 0000000..efd00d3 --- /dev/null +++ b/lib/src/zip_transport.rs @@ -0,0 +1,478 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +// ZIP is a somewhat dreadful archive format. We use it as the repository +// format of choice for Tufaceous because it has two qualities: +// +// 1. It is well-supported on most operating systems without any additional +// software, so anybody who finds themselves needing to deal with the +// repository can inspect it in a reasonable way. +// 2. Files can be accessed in random order. +// +// We use the `rawzip` crate to deal with parsing the structures in a ZIP +// archive; any higher-level use cases are an exercise left to the consumer. +// This allows us to create a very strict ZIP archive reader. A list of common +// problems with using ZIP files, and the defenses we take: +// +// - Directory traversal, symlinks, and similar naughty file operations related +// to extracting ZIP archives to a directory (e.g. "Zip Slip"). This module's +// defense is to never extract any files to disk, avoiding recreating any +// problems related to extraction. (The available API is to read specific +// files in the archive as a stream.) Symlinks in ZIP files largely do not +// work in the real world so they are ignored here (attempts to read a file +// which is a symlink result in an error). +// - Central directory confusion due to differences in readers. APPNOTE.TXT is +// full of doublespeak: one section says that archives must be read "from the +// back" (starting from searching for the end-of-central-directory record), +// and another suggests that archives can be streamed, or read "from the +// front". `rawzip` always reads "from the back". Additionally, APPNOTE.TXT +// arguably underspecifies comments, making it possible for the comment to +// contain something that is misparsed as an end-of-central-directory record. +// This module always expects the end-of-central-directory record to come at +// the very end of the file, rejecting any archives that contain comments or +// any extra content at the end. +// https://web.archive.org/web/20250131021721/https://games.greggman.com/game/zip-rant/ +// - Multiple entries with the same file name, leading to confusion due to +// differences in readers. These are detected and those files return an error +// if reading is attempted. +// - Quines and zip bombs which abuse overlapping file headers leading to a +// denial of service. This module takes the defense suggested by `rawzip`, +// which is to reject archives that contain files with overlapping file +// data. We additionally reject archives where file data overlaps the central +// directory. +// https://www.bamsoftware.com/hacks/zipbomb/ +// - Zip bombs which use extreme compression ratios leading to a denial of +// service. This module takes no defense against this; instead this defense +// is implemented in `tough`. The library places default size limits on +// repository metadata, and all targets in the repository have a file size in +// the signed metadata. A malicious archive designed to run a system out of +// memory or temporary disk space would therefore also need to be signed and +// trusted by the user (in the context of the Oxide control plane, this user +// is either an administrator over the entire control plane, or has physical +// access to the technician port). +// https://docs.rs/tough/0.21.0/tough/struct.Limits.html + +use std::collections::HashMap; +use std::fmt::Debug; +use std::fs::File; +use std::io::Cursor; +use std::io::Read; +use std::sync::Arc; +use std::sync::LazyLock; + +use bytes::Bytes; +use bytes::BytesMut; +use camino::Utf8PathBuf; +use flate2::read::DeflateDecoder; +use futures_util::Stream; +use futures_util::TryStreamExt; +use rawzip::CompressionMethod; +use rawzip::FileReader; +use rawzip::ReaderAt; +use rawzip::ZipArchive; +use rawzip::ZipArchiveEntryWayfinder; +use rawzip::ZipFileHeaderRecord; +use slog::Logger; +use slog::o; +use slog::warn; +use tough::Transport; +use tough::TransportError; +use tough::TransportErrorKind; +use tough::TransportStream; +use tough::async_trait; +use url::Url; + +use crate::error::Error; +use crate::error::ErrorKind; + +macro_rules! try_archive_path { + ($result:expr, $kind:ident, $path:expr) => { + $crate::error::try_path!($result, $kind, archive_path: $path) + }; +} + +// The length of the end-of-central-directory record if the comment is zero +// bytes. Archives with a "comment" (data after the EOCD) are rejected. +const EOCD_MAX_SEARCH_SPACE: u64 = 22; + +/// Implementation of [`tough::Transport`] that operates on a Zip archive. +/// +/// URLs used with this transport must use the `zip:///` protocol. For example, +/// if your metadata is found inside `repo/metadata` within the archive, use +/// `zip:///repo/metadata/` as the metadata base URL. +/// +/// Convenience methods for setting the correct transport +/// and base URLs for loading Tufaceous-generated +/// repositories are [`RepositoryLoader::load_zip_buffer`] and +/// [`RepositoryLoader::load_zip_file`]. +/// +/// [`RepositoryLoader::load_zip_buffer`]: crate::RepositoryLoader::load_zip_buffer +/// [`RepositoryLoader::load_zip_file`]: crate::RepositoryLoader::load_zip_file +#[derive(Debug)] +pub struct ZipTransport { + inner: Arc>, + log: Logger, +} + +#[derive(Debug)] +struct Inner { + archive: ZipArchive, + entries: HashMap, +} + +// Manually implemented, as the derive macro adds an unnecessary `T: Clone`. +impl Clone for ZipTransport { + fn clone(&self) -> Self { + Self { inner: self.inner.clone(), log: self.log.clone() } + } +} + +impl + Debug + Send + Sync + 'static> ZipTransport> { + pub fn from_slice(data: T, log: &Logger) -> Result { + let archive = ZipArchive::with_max_search_space(EOCD_MAX_SEARCH_SPACE) + .locate_in_slice(data) + .map_err(|(_, source)| ErrorKind::ReadZipEocd { + source, + archive_path: None, + })?; + Self::from_impl_blocking(archive.into_zip_archive(), None, None, log) + } +} + +impl ZipTransport { + pub fn from_file_blocking( + file: File, + archive_path: Option, + log: &Logger, + ) -> Result { + let mut buffer = vec![0; rawzip::RECOMMENDED_BUFFER_SIZE]; + let archive = try_archive_path!( + ZipArchive::with_max_search_space(EOCD_MAX_SEARCH_SPACE) + .locate_in_file(file, &mut buffer) + .map_err(|(_, error)| error), + ReadZipEocd, + archive_path + ); + Self::from_impl_blocking(archive, archive_path, Some(buffer), log) + } +} + +impl ZipTransport { + fn from_impl_blocking( + archive: ZipArchive, + archive_path: Option, + buffer: Option>, + log: &Logger, + ) -> Result { + let mut buffer = + buffer.unwrap_or_else(|| vec![0; rawzip::RECOMMENDED_BUFFER_SIZE]); + + let mut entries = HashMap::new(); + // Keep a list of all central directory entries (regardless of whether + // the path is valid) to check for an unexpected number of entries, or + // overlapping file ranges. + let mut all_entries = Vec::new(); + + let mut records = archive.entries(&mut buffer); + while let Some(record) = + try_archive_path!(records.next_entry(), ReadZipCdfh, archive_path) + { + if usize64!(all_entries.len()) >= archive.entries_hint() { + return Err(ErrorKind::ZipEntryCount { + expected: archive.entries_hint(), + actual: None, + archive_path, + } + .into()); + } + + all_entries.push(RawEntry::from(&record)); + if let Some((url, entry)) = Entry::new(&record, log) { + entries + .entry(url) + .and_modify(|e| *e = Entry::Duplicate) + .or_insert(entry); + } + } + + // First check: the number of entries in the central directory matches + // the value in the EOCD record. + let actual = usize64!(all_entries.len()); + if archive.entries_hint() != actual { + return Err(ErrorKind::ZipEntryCount { + expected: archive.entries_hint(), + actual: Some(actual), + archive_path, + } + .into()); + } + + let mut ranges = Vec::new(); + for RawEntry { raw_path, wayfinder } in all_entries { + let entry = try_archive_path!( + archive.get_entry(wayfinder), + ReadZipLocal, + archive_path + ); + let header = try_archive_path!( + entry.local_header(&mut buffer), + ReadZipLocal, + archive_path + ); + // Check that the path in the central directory matches the header + // that comes before the file data range. + if raw_path != header.file_path().as_bytes() { + return Err(ErrorKind::ZipPathMismatch { + central: raw_path, + local: header.file_path().as_bytes().to_vec(), + archive_path, + } + .into()); + } + + let (start, end) = entry.compressed_data_range(); + ranges.push((start..end, raw_path)); + // Check that no file ranges overlap with or come after the central + // directory. + if end > archive.directory_offset() { + return Err(ErrorKind::ZipRangeOverrun { + file_path: header.file_path().as_bytes().to_vec(), + data_range: start..end, + archive_path, + } + .into()); + } + } + // Check that no file ranges overlap with each other. + ranges.sort_by_key(|(range, _)| range.start); + for [(earlier, earlier_path), (later, later_path)] in + ranges.array_windows() + { + if earlier.end > later.start { + return Err(ErrorKind::ZipOverlappingRanges { + earlier_path: earlier_path.clone(), + earlier: earlier.clone(), + later_path: later_path.clone(), + later: later.clone(), + archive_path, + } + .into()); + } + } + + Ok(Self { + inner: Arc::new(Inner { archive, entries }), + log: log.clone(), + }) + } + + fn stream( + self, + entry_data: EntryData, + url: Url, + ) -> impl Stream> { + // This could be a lot nicer but the ZIP entry reader can't cross in/out + // of `spawn_blocking` because the type has an associated lifetime. + let log = self.log.new(o!( + "stream" => format!("{}::stream", std::any::type_name::()), + "url" => url.to_string(), + )); + crate::mpsc_stream::mpsc_stream(Some(log), move |tx| { + let mut reader = match self + .inner + .archive + .get_entry(entry_data.wayfinder) + .map_err(ZipTransportError::from) + .and_then(|entry| match entry_data.compression_method { + CompressionMethod::Store => Ok(entry.verifying_reader( + Box::new(entry.reader()) as Box, + )), + CompressionMethod::Deflate => Ok(entry.verifying_reader( + Box::new(DeflateDecoder::new(entry.reader())), + )), + other => Err(ZipTransportError::CompressionMethod(other)), + }) { + Ok(reader) => reader, + Err(error) => { + return tx.blocking_send(Err(error)); + } + }; + + let mut buf = BytesMut::with_capacity(8192); + loop { + if buf.capacity() == 0 { + buf.reserve(8192); + } + buf.resize(buf.capacity(), 0); + match reader.read(&mut buf) { + Ok(0) => return Ok(()), + Ok(n) => { + buf.truncate(n); + tx.blocking_send(Ok(buf.split().freeze()))?; + } + Err(error) => { + return tx.blocking_send(Err(ZipTransportError::from( + error, + ))); + } + } + } + }) + .map_err(move |e: ZipTransportError| e.into_tough_error(url.clone())) + } +} + +#[async_trait] +impl Transport + for ZipTransport +{ + async fn fetch(&self, url: Url) -> Result { + if url.scheme() != "zip" { + return Err(TransportError::new( + TransportErrorKind::UnsupportedUrlScheme, + url, + )); + } + match self.inner.entries.get(&url).copied() { + Some(Entry::File(entry_data)) => { + Ok(Box::pin(self.clone().stream(entry_data, url))) + } + Some(Entry::Dir) => { + Err(ZipTransportError::IsADirectory.into_tough_error(url)) + } + Some(Entry::Symlink) => { + Err(ZipTransportError::IsASymlink.into_tough_error(url)) + } + Some(Entry::Duplicate) => { + Err(ZipTransportError::Duplicate.into_tough_error(url)) + } + None => Err(ZipTransportError::FileNotFound.into_tough_error(url)), + } + } +} + +#[derive(Debug, Clone, Copy)] +enum Entry { + File(EntryData), + Dir, + Symlink, + Duplicate, +} + +#[derive(Debug, Clone, Copy)] +struct EntryData { + compression_method: CompressionMethod, + wayfinder: ZipArchiveEntryWayfinder, +} + +impl Entry { + fn new( + record: &ZipFileHeaderRecord<'_>, + log: &Logger, + ) -> Option<(Url, Self)> { + static BASE_URL: LazyLock = LazyLock::new(|| { + Url::parse("zip:///").expect("`zip:///` is a valid URL") + }); + + let path = record.file_path(); + let path = path + .try_normalize() + .inspect_err(|err| { + warn!( + log, + "ignoring invalid path in zip archive"; + "path" => path.as_bytes().escape_ascii().to_string(), + "error" => err.to_string(), + ); + }) + .ok()?; + let url = BASE_URL + .join(path.as_str()) + .inspect_err(|err| { + warn!( + log, + "ignoring invalid path in zip archive"; + "path" => path.as_str(), + "error" => err.to_string(), + ); + }) + .ok()?; + + let entry = if record.is_dir() { + Entry::Dir + } else if record.mode().is_symlink() { + Entry::Symlink + } else { + Entry::File(EntryData { + compression_method: record.compression_method(), + wayfinder: record.wayfinder(), + }) + }; + + Some((url, entry)) + } +} + +#[derive(Debug)] +struct RawEntry { + raw_path: Vec, + wayfinder: ZipArchiveEntryWayfinder, +} + +impl<'a> From<&'a ZipFileHeaderRecord<'a>> for RawEntry { + fn from(record: &'a ZipFileHeaderRecord<'a>) -> Self { + Self { + raw_path: record.file_path().as_bytes().to_vec(), + wayfinder: record.wayfinder(), + } + } +} + +/// Possible source errors of [`ZipTransport::fetch`]. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum ZipTransportError { + #[error(transparent)] + Io(#[from] std::io::Error), + #[error(transparent)] + Join(#[from] tokio::task::JoinError), + #[error("failed to join {url} onto {base}")] + UrlJoin { + #[source] + source: url::ParseError, + url: String, + base: String, + }, + #[error(transparent)] + Zip(#[from] rawzip::Error), + + #[error("unsupported compression method {0:?}")] + CompressionMethod(CompressionMethod), + #[error("multiple entries found for file")] + Duplicate, + #[error("file not found")] + FileNotFound, + #[error("is a directory")] + IsADirectory, + #[error("is a symlink")] + IsASymlink, +} + +impl ZipTransportError { + fn into_tough_error(self, url: Url) -> TransportError { + let kind = match &self { + ZipTransportError::FileNotFound => TransportErrorKind::FileNotFound, + + ZipTransportError::Io(_) + | ZipTransportError::Join(_) + | ZipTransportError::UrlJoin { .. } + | ZipTransportError::Zip(_) + | ZipTransportError::CompressionMethod(_) + | ZipTransportError::Duplicate + | ZipTransportError::IsADirectory + | ZipTransportError::IsASymlink => TransportErrorKind::Other, + }; + TransportError::new_with_cause(kind, url, self) + } +} diff --git a/lib/tests/check.rs b/lib/tests/check.rs new file mode 100644 index 0000000..1967a69 --- /dev/null +++ b/lib/tests/check.rs @@ -0,0 +1,51 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use chrono::Utc; +use semver::Version; +use tufaceous::CheckProblem; +use tufaceous::Repository; +use tufaceous::RepositoryLoader; +use tufaceous::edit::RepositoryEditor; +use tufaceous::error::Error; +use tufaceous_artifact::KnownArtifactTags; + +const V1: Version = Version::new(1, 0, 0); + +#[tokio::test] +async fn fake_checks_out() -> Result<(), Error> { + let log = slog::Logger::root(slog::Discard, slog::o!()); + let repo = Repository::fake(V1, &log).await?; + let problems = repo.check_problems().await?; + assert!(problems.is_empty(), "repo has unexpected problems: {problems:?}"); + Ok(()) +} + +#[tokio::test] +async fn missing_installinator() -> Result<(), Error> { + let log = slog::Logger::root(slog::Discard, slog::o!()); + let signed = RepositoryEditor::fake(V1)? + .set_generate_installinator_document(false) + .finish() + .await? + .generate_root() + .sign() + .await?; + let zip = signed.write_zip(Vec::new(), Utc::now()).await?; + let repo = RepositoryLoader::new() + .trust_root(signed.root()) + .load_zip_buffer(zip, &log) + .await?; + let problems = repo.check_problems().await?; + assert!( + matches!( + problems.as_slice(), + [CheckProblem::MissingArtifact( + KnownArtifactTags::InstallinatorDocument + )] + ), + "repo has unexpected problems: {problems:?}" + ); + Ok(()) +} diff --git a/lib/tests/data/corim-all-sp-v1.0.55.cbor b/lib/tests/data/corim-all-sp-v1.0.55.cbor new file mode 100644 index 0000000..3a9c348 Binary files /dev/null and b/lib/tests/data/corim-all-sp-v1.0.55.cbor differ diff --git a/lib/tests/data/v1-fake.zip b/lib/tests/data/v1-fake.zip new file mode 100644 index 0000000..a0eb1a9 Binary files /dev/null and b/lib/tests/data/v1-fake.zip differ diff --git a/lib/tests/tests.rs b/lib/tests/tests.rs new file mode 100644 index 0000000..4f1f5e6 --- /dev/null +++ b/lib/tests/tests.rs @@ -0,0 +1,197 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::convert::Infallible; +use std::io::Write; +use std::sync::Arc; + +use bytes::BufMut; +use bytes::BytesMut; +use chrono::Utc; +use futures_util::TryStreamExt; +use futures_util::stream; +use semver::Version; +use tufaceous::RepositoryLoader; +use tufaceous::edit::RepositoryEditor; +use tufaceous::error::Error; +use tufaceous_artifact::KnownArtifactTags; + +const V1: Version = Version::new(1, 0, 0); +const V2: Version = Version::new(2, 0, 0); + +#[tokio::test] +async fn it_works() -> Result<(), Error> { + let log = slog::Logger::root(slog::Discard, slog::o!()); + let signed = RepositoryEditor::fake(V1)? + .finish() + .await? + .generate_root() + .sign() + .await?; + let zip = signed.write_zip(Vec::new(), Utc::now()).await?; + let repo = RepositoryLoader::new() + .trust_root(signed.root()) + .load_zip_buffer(zip, &log) + .await?; + for artifact in repo.artifacts() { + repo.read_artifact(artifact) + .await? + .map_ok(|bytes| bytes.to_vec()) + .try_concat() + .await?; + } + Ok(()) +} + +#[tokio::test] +async fn verify_targets() -> Result<(), Error> { + let log = slog::Logger::root(slog::Discard, slog::o!()); + let signed = RepositoryEditor::fake(V1)? + .finish() + .await? + .generate_root() + .sign() + .await?; + let mut zip = signed.write_zip(Vec::new(), Utc::now()).await?; + let repo = RepositoryLoader::new() + .trust_root(signed.root()) + .load_zip_buffer(zip.clone(), &log) + .await?; + let repo = Arc::new(repo); + let parallelism = std::thread::available_parallelism().unwrap().get(); + repo.verify_targets(parallelism).await?; + + // Now intentionally fuck up the archive, and ensure verification fails. + let pos = memchr::memmem::find(&zip, b"hubris") + .expect("b\"hubris\" not found in archive"); + zip[pos] = b'H'; // flip! that! bit! + let repo = RepositoryLoader::new() + .trust_root(signed.root()) + .load_zip_buffer(zip.clone(), &log) + .await?; + let repo = Arc::new(repo); + let err = repo.verify_targets(parallelism).await.unwrap_err(); + // This error ultimately comes from `rawzip`'s CRC-32 checking. + assert!(err.to_string().contains("Invalid checksum")); + + Ok(()) +} + +#[tokio::test] +async fn no_artifacts() -> Result<(), Error> { + let log = slog::Logger::root(slog::Discard, slog::o!()); + let signed = RepositoryEditor::new(V1)? + .set_generate_installinator_document(false) + .finish() + .await? + .generate_root() + .sign() + .await?; + let zip = signed.write_zip(Vec::new(), Utc::now()).await?; + let repo = RepositoryLoader::new() + .trust_root(signed.root()) + .load_zip_buffer(zip, &log) + .await?; + assert_eq!(repo.artifacts().len(), 0); + assert!(repo.artifacts().is_empty()); + Ok(()) +} + +#[tokio::test] +async fn inconsistent_fake() -> Result<(), Error> { + let log = slog::Logger::root(slog::Discard, slog::o!()); + let signed = RepositoryEditor::fake(V1)? + .finish() + .await? + .generate_root() + .sign() + .await?; + let zip = signed.write_zip(Vec::new(), Utc::now()).await?; + let repo1 = RepositoryLoader::new() + .trust_root(signed.root()) + .load_zip_buffer(zip, &log) + .await?; + + let artifact_v1 = V1.to_string().parse()?; + let artifact_v2 = V2.to_string().parse()?; + let signed = + RepositoryEditor::inconsistent_fake(V2, &artifact_v2, &artifact_v1)? + .finish() + .await? + .generate_root() + .sign() + .await?; + let zip = signed.write_zip(Vec::new(), Utc::now()).await?; + let repo2 = RepositoryLoader::new() + .trust_root(signed.root()) + .load_zip_buffer(zip, &log) + .await?; + + for (first, second) in + repo1.artifacts().iter().zip(repo2.artifacts().iter()) + { + // Each artifact should have the same tags. + assert_eq!(first.tags, second.tags); + // If this is the Installinator document, skip the rest of the checks. + // The Installinator document's version is always the system version. + if first.known_tags() == Some(KnownArtifactTags::InstallinatorDocument) + { + continue; + } + // The first artifact should be 1.0.0, and the second should be 2.0.0. + assert_eq!(first.version, artifact_v1); + assert_eq!(second.version, artifact_v2); + // The artifacts should have the same contents (hash and length), as they + // should both have an interior version of 1.0.0. + assert_eq!(first.hash, second.hash); + assert_eq!(first.length, second.length); + } + + Ok(()) +} + +#[tokio::test] +async fn compute_archive_sha256() -> Result<(), Error> { + let log = slog::Logger::root(slog::Discard, slog::o!()); + let signed = RepositoryEditor::fake(V1)? + .finish() + .await? + .generate_root() + .sign() + .await?; + let zip = signed + .write_zip(BytesMut::new().writer(), Utc::now()) + .await? + .into_inner() + .freeze(); + for should_compute in [false, true] { + let repo = RepositoryLoader::new() + .compute_archive_sha256(should_compute) + .trust_root(signed.root()) + .load_zip_buffer(zip.clone(), &log) + .await?; + assert_eq!(repo.archive_sha256().is_some(), should_compute); + + let mut file = camino_tempfile::tempfile().unwrap(); + file.write_all(&zip).unwrap(); + let repo = RepositoryLoader::new() + .compute_archive_sha256(should_compute) + .trust_root(signed.root()) + .load_zip_file(file, None, &log) + .await?; + assert_eq!(repo.archive_sha256().is_some(), should_compute); + + let repo = RepositoryLoader::new() + .compute_archive_sha256(should_compute) + .trust_root(signed.root()) + .load_zip_stream( + stream::once(async { Ok::<_, Infallible>(zip.clone()) }), + None, + &log, + ) + .await?; + assert_eq!(repo.archive_sha256().is_some(), should_compute); + } + Ok(()) +} diff --git a/lib/tests/v1_compatibility.rs b/lib/tests/v1_compatibility.rs new file mode 100644 index 0000000..7d612ee --- /dev/null +++ b/lib/tests/v1_compatibility.rs @@ -0,0 +1,98 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use camino::Utf8Path; +use tufaceous::ExpirationEnforcement; +use tufaceous::RepositoryLoader; +use tufaceous::error::Error; +use tufaceous_artifact::ArtifactSet; +use tufaceous_artifact::KnownArtifactTags; +use tufaceous_artifact::OsBoard; +use tufaceous_artifact::OsPhase1Tags; +use tufaceous_artifact::OsPhase2Tags; +use tufaceous_artifact::OsVariant; +use tufaceous_artifact::RotBootloaderTags; +use tufaceous_artifact::RotKeyTableHash; +use tufaceous_artifact::RotSlot; +use tufaceous_artifact::RotTags; +use tufaceous_artifact::SpTags; +use tufaceous_artifact::ZoneTags; + +#[tokio::test] +async fn v1_fake() -> Result<(), Error> { + let log = slog::Logger::root(slog::Discard, slog::o!()); + let path = Utf8Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/data/v1-fake.zip"); + + let repo = RepositoryLoader::new() + .expiration_enforcement(ExpirationEnforcement::Unsafe) + .unsafe_blindly_trust_repo() + .v1_compatibility(true) + .load_zip_path(path, &log) + .await?; + let parallelism = std::thread::available_parallelism().unwrap().get(); + repo.verify_targets(parallelism).await?; + + // We expect to see one of each of these artifacts: + let mut expected = vec![ + KnownArtifactTags::InstallinatorDocument, + KnownArtifactTags::MeasurementCorpus, + ]; + for os_variant in [OsVariant::Host, OsVariant::Recovery] { + for os_board in [OsBoard::COSMO, OsBoard::GIMLET] { + expected.push(OsPhase1Tags { os_board, os_variant }.into()); + } + expected.push(OsPhase2Tags { os_variant }.into()); + } + for rot_rkth in ["sign-gimlet", "sign-psc", "sign-switch"] { + for rot_slot in [RotSlot::A, RotSlot::B] { + expected.push( + RotTags { + rot_board: "SimRot".into(), + rot_rkth: Some(RotKeyTableHash::new(rot_rkth)), + rot_slot, + } + .into(), + ); + } + expected.push( + RotBootloaderTags { + rot_board: "SimRot".into(), + rot_rkth: Some(RotKeyTableHash::new(rot_rkth)), + } + .into(), + ); + } + for sp_board in ["SimGimletSp", "SimPscSp", "SimSidecarSp"] { + expected.push(SpTags { sp_board: sp_board.into() }.into()); + } + for zone_name in ["zone-1", "zone-2"] { + expected.push(ZoneTags { zone_name: zone_name.into() }.into()); + } + let seen = expected + .iter() + .map(|tags| repo.artifacts().get_only(tags).unwrap().clone()) + .collect::(); + // And there should be no unexpected artifacts: + assert_eq!(&seen, repo.artifacts()); + + Ok(()) +} + +#[tokio::test] +#[should_panic(expected = "target artifacts-v2.json not found")] +async fn v1_fake_without_compat() { + let log = slog::Logger::root(slog::Discard, slog::o!()); + let path = Utf8Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/data/v1-fake.zip"); + + let _ = RepositoryLoader::new() + .expiration_enforcement(ExpirationEnforcement::Unsafe) + .unsafe_blindly_trust_repo() + .v1_compatibility(false) + .load_zip_path(path, &log) + .await + .map_err(|error| error.to_string()) + .unwrap(); +}