From 15795101eb55271c09a102c4821aa8e16fce5704 Mon Sep 17 00:00:00 2001 From: op-q <141008398+op-q@users.noreply.github.com> Date: Tue, 15 Sep 2026 12:37:00 +0200 Subject: [PATCH 1/2] Archive a folder on each OS and receive it on every OS, in CI A cross-OS transfer can differ by platform in only two places: how the sender reads a tree, and how the receiver writes one. The wire is the same everywhere. Two CI runners cannot swap a transfer code, so the transfer is split in two. `produce`, on each OS, builds a tree using every kind of name and file that OS can hold and archives it exactly as `drop send` does. `consume`, on each OS, receives every OS's archive, tar and gzipped, through the real receive path over an in-process relay. It checks contents, directories, links (kept on Unix, skipped on Windows), executable bits, and Windows name rewriting. That is nine pairings. The workflow runs on pull requests that touch the tree code, nightly, and on demand. With one byte of an archive flipped, consume fails naming the file. Cross-platform plan, phase 4. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01G7Fy45hUvna94cd79WKG8S --- .github/workflows/cross-os.yml | 92 +++++ cli/tests/cross_os.rs | 356 +++++++++++++++++++ docs/implementation-checklist.md | 2 +- docs/plans/cross-platform-plan-2026-09-14.md | 25 +- 4 files changed, 470 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/cross-os.yml create mode 100644 cli/tests/cross_os.rs diff --git a/.github/workflows/cross-os.yml b/.github/workflows/cross-os.yml new file mode 100644 index 0000000..48f5b9f --- /dev/null +++ b/.github/workflows/cross-os.yml @@ -0,0 +1,92 @@ +name: Cross-OS archives + +# A folder archived on each operating system, received on every operating +# system, through the real receive path. The wire is platform-neutral, so the +# part of a cross-OS transfer that can differ is how a sender reads a tree and +# how a receiver writes one; this is that part, nine pairings. +# +# Two runners cannot swap a transfer code, so the transfer is split: `produce` +# archives on each OS, `consume` receives every archive on each OS over an +# in-process relay. The logic lives in cli/tests/cross_os.rs. +# +# Not on every pull request: only those touching the code that reads or writes +# trees, plus nightly, plus by hand. See +# docs/plans/cross-platform-plan-2026-09-14.md, phase 4. + +on: + pull_request: + paths: + - cli/src/tar.rs + - cli/src/untar.rs + - cli/src/payload.rs + - cli/src/recv.rs + - cli/src/names.rs + - cli/tests/cross_os.rs + - .github/workflows/cross-os.yml + schedule: + - cron: "17 3 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: cross-os-${{ github.ref }} + cancel-in-progress: true + +jobs: + produce: + name: Archive on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-14, windows-2025] + steps: + - name: Check out source + uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + - name: Build the fixture archive + shell: bash + env: + DROP_FIXTURE_OUT: ${{ runner.temp }}/fixtures + run: cargo test --locked -p drop-cli --test cross_os -- --ignored produce + - name: Upload the archive + uses: actions/upload-artifact@v7 + with: + name: fixtures-${{ matrix.os }} + path: ${{ runner.temp }}/fixtures + if-no-files-found: error + retention-days: 3 + + consume: + name: Receive on ${{ matrix.os }} + needs: produce + runs-on: ${{ matrix.os }} + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-14, windows-2025] + steps: + - name: Check out source + uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + - name: Download every OS's archive + uses: actions/download-artifact@v8 + with: + pattern: fixtures-* + path: ${{ runner.temp }}/fixtures + merge-multiple: true + - name: Receive and check them all + shell: bash + env: + DROP_FIXTURE_IN: ${{ runner.temp }}/fixtures + run: cargo test --locked -p drop-cli --test cross_os -- --ignored consume --nocapture diff --git a/cli/tests/cross_os.rs b/cli/tests/cross_os.rs new file mode 100644 index 0000000..e27116c --- /dev/null +++ b/cli/tests/cross_os.rs @@ -0,0 +1,356 @@ +//! Folders archived on one operating system and received on another. +//! +//! The wire is the same everywhere: JSON, big-endian integers, sealed bytes. +//! What differs by platform is how a sender reads a tree and how a receiver +//! writes one, so that is where a cross-OS transfer can go wrong. Two runners +//! cannot swap a transfer code mid-run, so these tests split a transfer in +//! two. One test, on each OS, builds a tree using every kind of name and file +//! that OS can hold and archives it exactly as `drop send` would. The other, +//! on each OS, receives every OS's archive through the real receive path over +//! an in-process relay and checks what landed. +//! +//! Both are `#[ignore]`d. They run in `.github/workflows/cross-os.yml`, which +//! moves the archives between runners: +//! +//! ```text +//! DROP_FIXTURE_OUT=dir cargo test -p drop-cli --test cross_os -- --ignored produce +//! DROP_FIXTURE_IN=dir cargo test -p drop-cli --test cross_os -- --ignored consume +//! ``` +//! +//! See `docs/plans/cross-platform-plan-2026-09-14.md`, phase 4. + +use std::{ + fs, + net::SocketAddr, + path::{Path, PathBuf}, + time::Duration, +}; + +use drop_cli::{ + cancel::Cancel, + consent::Acceptance, + names::Naming, + recv::{self, ReceiveOptions}, + send::{self, SendOptions}, + tar::TarPlan, +}; +use serde_json::{Value, json}; +use tokio::sync::oneshot; + +/// The OS the tests are running on, as it names fixtures. +fn this_os() -> &'static str { + if cfg!(windows) { + "windows" + } else if cfg!(target_os = "macos") { + "macos" + } else { + "linux" + } +} + +/// Contents for a fixture file, derived from its path so the consumer can +/// regenerate them without the producer shipping a copy or a hash. +fn contents_for(path: &str, len: usize) -> Vec { + let mut state = path.bytes().fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| { + (hash ^ u64::from(byte)).wrapping_mul(0x0100_0000_01b3) + }) | 1; + + (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state as u8 + }) + .collect() +} + +struct Tree { + root: PathBuf, + entries: Vec, +} + +impl Tree { + fn file(&mut self, relative: &str, len: usize, executable: bool) { + let path = self.root.join(relative); + fs::create_dir_all(path.parent().expect("a parent")).expect("parent directories"); + fs::write(&path, contents_for(relative, len)).expect("fixture file"); + + #[cfg(unix)] + if executable { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).expect("mode"); + } + + self.entries.push(json!({ + "kind": "file", + "path": relative, + "len": len, + "executable": executable && cfg!(unix), + })); + } + + fn directory(&mut self, relative: &str) { + fs::create_dir_all(self.root.join(relative)).expect("fixture directory"); + self.entries + .push(json!({ "kind": "dir", "path": relative })); + } + + #[cfg(unix)] + fn symlink(&mut self, relative: &str, target: &str) { + let path = self.root.join(relative); + fs::create_dir_all(path.parent().expect("a parent")).expect("parent directories"); + std::os::unix::fs::symlink(target, &path).expect("fixture symlink"); + self.entries + .push(json!({ "kind": "symlink", "path": relative, "target": target })); + } +} + +fn scratch(name: &str) -> PathBuf { + let base = std::env::temp_dir().join(format!( + "drop-cross-os-{}-{}-{name}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_nanos() + )); + fs::create_dir_all(&base).expect("scratch directory"); + base +} + +/// Builds this OS's fixture tree and archives it the way `drop send` does. +#[test] +#[ignore = "run by .github/workflows/cross-os.yml, which moves archives between runners"] +fn produce() { + let out = PathBuf::from(std::env::var("DROP_FIXTURE_OUT").expect("DROP_FIXTURE_OUT")); + fs::create_dir_all(&out).expect("output directory"); + + let base = scratch("produce"); + let mut tree = Tree { + root: base.join("tree"), + entries: Vec::new(), + }; + + tree.file("readme.md", 200, false); + tree.file("empty.txt", 0, false); + tree.file("nested/deeper/data.bin", 3 * 1024 * 1024 + 7, false); + tree.directory("empty-dir"); + tree.file("Łódź 東京 🎉.txt", 64, false); + // Longer than a ustar header holds, so it travels in a long-name record. + tree.file(&format!("{}.txt", "long-name-".repeat(12)), 32, false); + // Deeper than Windows' historic 260-character path limit, once the + // runner's temporary directory is in front of it. + let deep: Vec = (0..12) + .map(|level| format!("level-{level:02}-padding")) + .collect(); + tree.file(&format!("{}/bottom.txt", deep.join("/")), 16, false); + tree.file("scripts/run.sh", 40, true); + + // What only a Unix sender can put in a folder. + #[cfg(unix)] + { + tree.symlink("links/to-readme", "../readme.md"); + tree.file("10:30 standup.md", 48, false); + tree.file("nul.txt", 8, false); + tree.file("trailing dot.", 8, false); + } + + let plan = TarPlan::scan(&tree.root).expect("scan"); + let name = format!("from-{}", this_os()); + + let mut archive = fs::File::create(out.join(format!("{name}.tar"))).expect("archive"); + plan.write_to(&mut archive, |warning| { + panic!("fixture changed while archiving: {warning}") + }) + .expect("archived"); + + let tar_bytes = fs::read(out.join(format!("{name}.tar"))).expect("read back"); + let mut gzip = flate2::write::GzEncoder::new( + fs::File::create(out.join(format!("{name}.tar.gz"))).expect("gzip archive"), + flate2::Compression::new(6), + ); + std::io::Write::write_all(&mut gzip, &tar_bytes).expect("compressed"); + gzip.finish().expect("gzip finished"); + + fs::write( + out.join(format!("{name}.json")), + serde_json::to_vec_pretty(&json!({ "producer": this_os(), "entries": tree.entries })) + .expect("manifest"), + ) + .expect("manifest written"); + + fs::remove_dir_all(&base).ok(); +} + +async fn spawn_relay() -> String { + let state = api::build_state(); + api::start_background_services(state.clone()); + let app = api::build_app(state); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("relay listener"); + let addr: SocketAddr = listener.local_addr().expect("relay address"); + tokio::spawn(async move { api::serve(listener, app).await.expect("relay") }); + tokio::time::sleep(Duration::from_millis(50)).await; + format!("http://{addr}") +} + +/// Sends `archive` and receives it, extracting, into `destination`. +async fn transfer(origin: &str, archive: &Path, destination: &Path) { + let (code_tx, code_rx) = oneshot::channel(); + let mut code_tx = Some(code_tx); + + let sender = tokio::spawn({ + let origin = origin.to_string(); + let archive = archive.to_path_buf(); + async move { + send::run_cancellable( + &archive, + SendOptions { + origin: Some(origin), + compress: None, + path: drop_cli::direct::Path::Relay, + status: false, + rendezvous: drop_cli::direct::Rendezvous::default(), + on_code: Box::new(move |code| { + if let Some(sender) = code_tx.take() { + let _ = sender.send(code.to_string()); + } + }), + }, + Cancel::default(), + ) + .await + .map_err(|error| error.to_string()) + } + }); + + let code = tokio::time::timeout(Duration::from_secs(30), code_rx) + .await + .expect("a code in time") + .expect("a code"); + + recv::run_deciding( + &code, + &ReceiveOptions { + path: drop_cli::direct::Path::Relay, + status: false, + rendezvous: drop_cli::direct::Rendezvous::default(), + origin: Some(origin.to_string()), + out_dir: destination.to_path_buf(), + extract: true, + force: false, + acceptance: Acceptance::Yes, + }, + &mut Acceptance::Yes, + Cancel::default(), + ) + .await + .unwrap_or_else(|error| panic!("receiving {} failed: {error}", archive.display())); + + sender + .await + .expect("sender task") + .unwrap_or_else(|error| panic!("sending {} failed: {error}", archive.display())); +} + +/// Where an entry from the archive is expected to land on this OS. +fn landed(destination: &Path, relative: &str) -> PathBuf { + let naming = Naming::for_this_platform(); + let mut path = destination.join("tree"); + for component in relative.split('/') { + path.push(naming.component(component).as_ref()); + } + path +} + +/// Receives every OS's archive, tar and gzipped tar, and checks what landed. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "run by .github/workflows/cross-os.yml, which moves archives between runners"] +async fn consume() { + let input = PathBuf::from(std::env::var("DROP_FIXTURE_IN").expect("DROP_FIXTURE_IN")); + let origin = spawn_relay().await; + + let mut manifests: Vec = fs::read_dir(&input) + .expect("fixture directory") + .map(|entry| entry.expect("entry").path()) + .filter(|path| { + path.extension() + .is_some_and(|extension| extension == "json") + }) + .collect(); + manifests.sort(); + assert!(!manifests.is_empty(), "no fixtures in {}", input.display()); + + let mut report = Vec::new(); + + for manifest_path in manifests { + let manifest: Value = + serde_json::from_slice(&fs::read(&manifest_path).expect("manifest")).expect("json"); + let producer = manifest["producer"].as_str().expect("producer").to_string(); + let stem = manifest_path + .file_stem() + .expect("stem") + .to_string_lossy() + .into_owned(); + + for suffix in ["tar", "tar.gz"] { + let archive = input.join(format!("{stem}.{suffix}")); + let destination = scratch(&format!("{stem}-{}", suffix.replace('.', "-"))); + transfer(&origin, &archive, &destination).await; + + for entry in manifest["entries"].as_array().expect("entries") { + let relative = entry["path"].as_str().expect("path"); + let path = landed(&destination, relative); + let what = format!("{producer} {suffix} -> {}: {relative}", this_os()); + + match entry["kind"].as_str().expect("kind") { + "file" => { + let len = entry["len"].as_u64().expect("len") as usize; + let arrived = fs::read(&path) + .unwrap_or_else(|error| panic!("{what}: missing ({error})")); + assert!( + arrived == contents_for(relative, len), + "{what}: contents differ" + ); + + #[cfg(unix)] + if entry["executable"].as_bool() == Some(true) { + use std::os::unix::fs::PermissionsExt; + let mode = fs::metadata(&path).expect("metadata").permissions().mode(); + assert!(mode & 0o111 != 0, "{what}: lost its executable bit"); + } + } + "dir" => assert!(path.is_dir(), "{what}: directory missing"), + "symlink" => { + if cfg!(unix) { + let target = fs::read_link(&path) + .unwrap_or_else(|error| panic!("{what}: link missing ({error})")); + assert_eq!( + target.to_string_lossy(), + entry["target"].as_str().expect("target"), + "{what}: link target changed" + ); + } else { + // Skipped with a warning on a platform that + // cannot create links, never half-made. + assert!( + fs::symlink_metadata(&path).is_err(), + "{what}: something was created where a link was skipped" + ); + } + } + other => panic!("{what}: unknown entry kind {other}"), + } + } + + report.push(format!("{producer} {suffix} -> {}: ok", this_os())); + fs::remove_dir_all(&destination).ok(); + } + } + + for line in report { + println!("{line}"); + } +} diff --git a/docs/implementation-checklist.md b/docs/implementation-checklist.md index a090f99..ead3ab2 100644 --- a/docs/implementation-checklist.md +++ b/docs/implementation-checklist.md @@ -501,7 +501,7 @@ compiled. the console is closed, VT processing for the progress line. - [x] Phase 3 — done 2026-09-15, short of a tagged release. `x86_64`/`aarch64-pc-windows-msvc` release zips with a static CRT, `install.ps1`, README install lines. -- [ ] Phase 4 — archives produced on each OS and extracted on the others in CI, +- [x] Phase 4 — written 2026-09-15; first run on its pull request. Archives produced on each OS and extracted on the others in CI, nine pairings. - [ ] Phase 5 — manual Windows ↔ Linux checklist on the user's machine, both carriers, the interface, Ctrl-C, closing the window, the firewall dialog. diff --git a/docs/plans/cross-platform-plan-2026-09-14.md b/docs/plans/cross-platform-plan-2026-09-14.md index 6783e46..ada2d21 100644 --- a/docs/plans/cross-platform-plan-2026-09-14.md +++ b/docs/plans/cross-platform-plan-2026-09-14.md @@ -1,6 +1,6 @@ # Cross-platform plan: Windows, macOS and Linux, and transfers between them -Status: **active** — phases 0–3 done (2 and 3 on 2026-09-15); 4 and 5 next +Status: **active** — phases 0–4 written (2–4 on 2026-09-15); phase 4's first CI run and phase 5, the manual Windows checklist, next Created: **2026-09-14** Last updated: **2026-09-15** @@ -389,7 +389,7 @@ The wire is platform-neutral (see above), so an archive written on one OS and extracted on another is the cross-OS risk that remains. It can be tested without networking two runners together. -- [ ] A separate workflow, `cross-os.yml`, triggered on pull requests that touch +- [x] A separate workflow, `cross-os.yml`, triggered on pull requests that touch `cli/src/tar.rs`, `cli/src/untar.rs`, `cli/src/payload.rs` or `cli/src/recv.rs`, and nightly. Job `produce`, a matrix over three OSes, builds a fixed tree using every name that platform can hold: nested @@ -398,12 +398,12 @@ without networking two runners together. allows one, an executable script, and colons where the platform allows them. It writes the archive bytes `TarPlan` produces, uncompressed and gzipped, and uploads them with a manifest of expected contents. -- [ ] Job `consume`, needing `produce`, is also a matrix over three OSes. It +- [x] Job `consume`, needing `produce`, is also a matrix over three OSes. It downloads all three archives and extracts each through the real receive path, not only through `TarExtractor`, so the gzip and naming code runs too. It then checks against the manifest: identical bytes for every representable file, and the expected warning for every file that is not. -- [ ] Driven by `#[ignore]`d tests reading `DROP_FIXTURE_OUT` / +- [x] Driven by `#[ignore]`d tests reading `DROP_FIXTURE_OUT` / `DROP_FIXTURE_IN`, so the logic lives in Rust with the rest of the tests and the workflow is only plumbing. @@ -414,6 +414,23 @@ have no channel to swap a code mid-run, and making one would mean a test-only way to fix a transfer code in advance. That is a knob that weakens the one secret the protocol has, and it would exist in shipped binaries. +Written 2026-09-15 on `ci/cross-os-archives`; the nine pairings have their first +run on that branch's pull request. Notes: + +- Fixture contents are derived from each file's path, so a consumer regenerates + what it expects instead of the producer shipping hashes or copies. +- The tree covers: nested folders, an empty folder, an empty file, a 3 MiB file, + Unicode names, a name too long for a ustar header, a path deeper than 260 + characters, and an executable script. Only a Unix producer adds a relative + symlink, `10:30 standup.md`, `nul.txt` and `trailing dot.`, which a Windows + receiver has to rewrite or skip. +- The consumer expects Windows names rewritten exactly as `names::Naming` says, + links skipped (and nothing left in their place) on Windows, and executable bits + kept on Unix. +- **Negative control:** with one byte flipped in the middle of the Linux tar, + `consume` fails with `nested/deeper/data.bin: contents differ`. +- Linux to Linux was run locally, for both the tar and the gzipped tar. + ### Phase 5 — a real Windows machine against a real Linux one CI runners are clean virtual machines without the firewall prompt, antivirus From 3a51028e543046b931a122aa2c80017dd6b7e037 Mon Sep 17 00:00:00 2001 From: op-q <141008398+op-q@users.noreply.github.com> Date: Tue, 15 Sep 2026 12:51:39 +0200 Subject: [PATCH 2/2] Record where the 2026-09-14 order stands Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01G7Fy45hUvna94cd79WKG8S --- docs/plans/README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/plans/README.md b/docs/plans/README.md index 78ad170..0301260 100644 --- a/docs/plans/README.md +++ b/docs/plans/README.md @@ -198,3 +198,15 @@ order that serves them, by dependency: 7. **Tag 0.4.0.** 8. Then the NAT proof, once self-hosted rendezvous phase 3 is decided, and cross-platform phases 4–5. + +**Progress, 2026-09-15.** Steps 1–5 and the cross-platform phases 1–4 are written, +as a stack of pull requests waiting for review: #65 (these plans), #66 (move the +installer), #67 (CI on three OSes), #68 (display sanitisation), #69 (browser +removal), #70 (Windows receiver), #71 (`meta_ok` confirmation), #72 (consent), +#73 (cancel and status), #74 (Windows sender), #75 (Windows release and +installers), #76 (cross-OS archives in CI). Separately, #77 fixes a shipped bug +found by the lab: a stray QUIC handshake ended a direct-path send, which was +the long-unexplained `authentication failed`. Not started: consent phase 5 (the +interface screens) and cross-platform phase 5 (the manual Windows checklist). +Tagging 0.4.0 waits on the merges. The NAT proof still waits on the decision in +the self-hosted rendezvous plan's phase 3.