Tufaceous v2: project rewrite / RFD 621 implementation

Cargo.toml

@@ -8,65 +8,65 @@ members = [
 ]
 
 [workspace.package]
-version = "0.1.0"
-edition = "2021"
+version = "0.2.0"
+edition = "2024"
 license = "MPL-2.0"
 publish = false
 
 [workspace.dependencies]
 anyhow = "1.0.96"
 assert_cmd = "2.0.16"
-async-trait = "0.1.86"
+atomicwrites = "0.4.4"
 aws-lc-rs = "1.16.2"
-base64 = "0.22.1"
-buf-list = "1.0.3"
+byte-wrapper = { version = "0.1.0", default-features = false, features = ["alloc", "hex", "serde"] }
 bytes = "1.11.1"
-camino = { version = "1.1.9", features = ["serde1"] }
+camino = "1.1.9"
 camino-tempfile = "1.1.1"
 chrono = { version = "0.4.40", default-features = false, features = ["std"] }
-clap = { version = "4.5.31", features = ["cargo", "derive", "env", "wrap_help"] }
-console = { version = "0.15.10", default-features = false }
+ciborium = "0.2.2"
+clap = { version = "4.5.31", features = ["cargo", "derive", "wrap_help"] }
 daft = { version = "0.1.1", features = ["derive"] }
-datatest-stable = "0.2.9"
-debug-ignore = "1.0.5"
-dropshot = "0.17.0"
+digest-io = "0.1.0"
 expectorate = "1.1.0"
 flate2 = "1.0.35"
-fs-err = "2.11.0"
-futures = "0.3.31"
+futures-util = "0.3.31"
 hex = "0.4.3"
-hubtools = { git = "https://github.com/oxidecomputer/hubtools.git", branch = "main" }
-humantime = "2.1.0"
-indent_write = "2.2.0"
-itertools = "0.13.0"
-parse-size = "1.1.0"
-predicates = "3.1.3"
+hubtools = { git = "https://github.com/oxidecomputer/hubtools.git", rev = "2b1ef9b38d75563ea800baa3b17327eec17b1b7a" }
+memchr = "2.7.4"
 proptest = "1.5.0"
-rand = "0.8.6"
+rats-corim = { git = "https://github.com/oxidecomputer/rats-corim.git", rev = "f0d5d5168d3d31487a56df32c676b0c6240bcc6b" }
+rawzip = "0.4.4"
 regex = "1.11.1"
-schemars = { version = "0.8.21", features = ["semver"] }
+schemars = "0.8.21"
 semver = { version = "1.0.25", features = ["serde"] }
 serde = { version = "1.0.218", features = ["derive"] }
-serde_human_bytes = { git = "https://github.com/oxidecomputer/serde_human_bytes", branch = "main" }
 serde_json = "1.0.139"
-serde_path_to_error = "0.1.16"
-sha2 = "0.10.8"
+sha2 = "0.11.0"
 slog = "2.7.0"
 slog-async = "2.8.0"
 slog-envlogger = "2.2.0"
-slog-term = "2.9.1"
-strum = { version = "0.26.3", features = ["derive"] }
+slog-term = "2.9.2"
 tar = "0.4.45"
-tempfile = "3.13.0"
 test-strategy = "0.4.0"
 thiserror = "2.0.12"
-tokio = "1.43.0"
-toml = "0.8.20"
-tough = { version = "0.22.0", features = [ "http" ] }
-tufaceous-artifact = { path = "artifact", default-features = false }
-tufaceous-brand-metadata = { path = "brand-metadata" }
-tufaceous-lib = { path = "lib" }
+tokio = "1.47.0"
+tokio-test = "0.4.5"
+tough = "0.22.0"
+tufaceous.path = "lib"
+tufaceous-artifact.path = "artifact"
+tufaceous-brand-metadata.path = "brand-metadata"
 url = "2.5.4"
-zip = { version = "4.2.0", default-features = false }
 
 [workspace.lints.clippy]
+pedantic = { level = "warn", priority = -1 }
+missing-errors-doc = "allow"
+must-use-candidate = "allow"
+
+[profile.dev.package.flate2]
+opt-level = 3
+[profile.dev.package.miniz_oxide]
+opt-level = 3
+[profile.dev.package.rawzip]
+opt-level = 3
+[profile.dev.package.sha2]
+opt-level = 3

README.md

@@ -0,0 +1,68 @@
+# Tufaceous
+
+Tufaceous is the library and tooling for Oxide update repositories. It is a layer of Oxide-specific tooling on top of [tough](https://github.com/awslabs/tough), a Rust library that implements [TUF](https://theupdateframework.io/).
+
+This README is intended to provide a high-level overview of the library and instructions for adding to it. More context can be found in [RFD 621](https://rfd.shared.oxide.computer/rfd/0621).
+
+## Repository format
+
+We selected ZIP archives as the primary format for our repositories, despite ZIP's many shortcomings. Because Oxide customers are receiving update repositories and uploading them to their systems, we wanted them to be readable for most users so that they could see what is inside.
+
+The ZIP archive has a top-level `repo` directory. Inside this directory is a `metadata` directory, containing the signed TUF metadata, and a `targets` directory, containing all the targets described by that metadata.
+
+## Artifacts
+
+"Artifacts" are our jargon for a file that the Oxide control plane needs in order to perform updates. (All artifacts are "targets", TUF jargon for a file described by the repository metadata, but not all targets are artifacts.)
+
+Artifacts in the repository are described by the `artifacts-v2.json`. Each artifact has a set of tags that is used by the control plane to decide what to do with the artifact. When creating and using artifacts, these tags are strongly-typed (see `KnownArtifactTags` in `artifact/src/tags.rs`), but they are encoded as an arbitrary key-value mapping. This allows us to potentially release a new type of artifact and use it in the same release, once Nexus has been updated.
+
+As a general rule it is not possible to use this tooling to create a repository with arbitrary blobs and artifacts in it. The library is very opinionated about how artifacts are named to keep things usable for the CLI and for users of these repositories.
+
+## Tooling
+
+Most of the time you're working with repositories generated by the [Omicron release tooling](https://github.com/oxidecomputer/omicron/blob/main/docs/releng.adoc) (`cargo xtask releng` in Omicron). There is also a binary target that is a thin CLI on top of the library, which you can build with `cargo build --release --bin tufaceous`.
+
+In all commands that read from a repository, use either `-r`/`--trust-roots` to provide a list of trust roots to verify signatures against, or `--blindly-trust` to use the `metadata/1.root.json` root in the repository as the trust root. If you want to load a repository with an expired signature, use `--allow-expired`. For convenience you can combine `--allow-expired` and `--blindly-trust` with `-f`/`--force-load`.
+
+### Verify a repository
+
+`tufaceous verify` will read a repository, print its SHA256 checksum, and verify the metadata and checksums of all the metadata in the repository. It will also run a series of compatibility checks on the repository and warn you of any detected issues.
+
+### Unpack a repository
+
+Scripts can use `tufaceous list-targets` to get a list of targets from the repository, and `tufaceous show-target` to write a target to stdout.
+
+To unpack an entire repository, use `unzip` or ZIP tooling provided by your OS.
+
+### Build a repository
+
+`tufaceous build` has two required flags:
+
+- `--output file.zip`, which provides the path to write to
+- `--system-version 1.0.0`, which provides the system version
+
+The arguments are the set of artifacts to add:
+
+- A measurement corpus is a CoRIM file
+- An OS image is a directory containing `cosmo.rom`, `gimlet.rom`, and `zfs.img`, as produced by `helios-build image`
+- An RoT, RoT bootloader, or SP image is a Hubris archive (ZIP format)
+- A zone image is a tarball that starts with the `oxide.json` Omicron brand metadata
+
+When building a repository, an Installinator document is automatically created based on the artifacts in the repository. You can avoid this with `--no-installinator-document`.
+
+### Add/remove artifacts from a repository
+
+Using `tufaceous edit` you can remove targets from a repository using `--remove-targets` and add new artifacts using `--add-artifacts`.
+
+The Installinator document is automatically regenerated based on the artifacts in the final repository. You can avoid this with `--no-installinator-document`, which will keep an existing Installinator document if there is one.
+
+## Adding new artifacts
+
+In order to add a new type of artifact, you will need to make a series of changes to the library:
+
+- Modify `KnownArtifactTags` to describe your new artifact
+- Add methods for adding an actual artifact and a fake artifact to `RepositoryEditor`
+- Add a fake artifact in `Input::fake` (`lib/src/edit/fake.rs`)
+- Add code for guessing this type of artifact to `Input::guess` (`lib/src/edit/guess.rs`)
+
+As a general rule, Tufaceous artifacts are self-describing. The "guess" code is used by the CLI and generates tags from the artifact to avoid needing an unwieldy set of command line options to describe tags. If your artifact is not self-describing, you can staple a header of some kind to the beginning of the artifact that control plane code knows to strip before using the artifact.

artifact/Cargo.toml

@@ -4,20 +4,21 @@ version.workspace = true
 edition.workspace = true
 license.workspace = true
 publish.workspace = true
+rust-version = "1.94"
 
 [features]
+hubtools = ["dep:hubtools"]
 proptest = ["dep:proptest", "dep:test-strategy"]
-schemars = ["dep:schemars"]
+schemars = ["dep:schemars", "byte-wrapper/schemars08"]
 
 [dependencies]
+byte-wrapper.workspace = true
 daft.workspace = true
-hex.workspace = true
+hubtools = { workspace = true, optional = true }
 proptest = { workspace = true, optional = true }
 schemars = { workspace = true, optional = true }
-semver.workspace = true
 serde.workspace = true
-serde_human_bytes.workspace = true
-strum.workspace = true
+serde_json.workspace = true
 test-strategy = { workspace = true, optional = true }
 thiserror.workspace = true