diff --git a/.github/workflows/cross-os.yml b/.github/workflows/cross-os.yml new file mode 100644 index 0000000..48f5b9f --- /dev/null +++ b/.github/workflows/cross-os.yml @@ -0,0 +1,92 @@ +name: Cross-OS archives + +# A folder archived on each operating system, received on every operating +# system, through the real receive path. The wire is platform-neutral, so the +# part of a cross-OS transfer that can differ is how a sender reads a tree and +# how a receiver writes one; this is that part, nine pairings. +# +# Two runners cannot swap a transfer code, so the transfer is split: `produce` +# archives on each OS, `consume` receives every archive on each OS over an +# in-process relay. The logic lives in cli/tests/cross_os.rs. +# +# Not on every pull request: only those touching the code that reads or writes +# trees, plus nightly, plus by hand. See +# docs/plans/cross-platform-plan-2026-09-14.md, phase 4. + +on: + pull_request: + paths: + - cli/src/tar.rs + - cli/src/untar.rs + - cli/src/payload.rs + - cli/src/recv.rs + - cli/src/names.rs + - cli/tests/cross_os.rs + - .github/workflows/cross-os.yml + schedule: + - cron: "17 3 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: cross-os-${{ github.ref }} + cancel-in-progress: true + +jobs: + produce: + name: Archive on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-14, windows-2025] + steps: + - name: Check out source + uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + - name: Build the fixture archive + shell: bash + env: + DROP_FIXTURE_OUT: ${{ runner.temp }}/fixtures + run: cargo test --locked -p drop-cli --test cross_os -- --ignored produce + - name: Upload the archive + uses: actions/upload-artifact@v7 + with: + name: fixtures-${{ matrix.os }} + path: ${{ runner.temp }}/fixtures + if-no-files-found: error + retention-days: 3 + + consume: + name: Receive on ${{ matrix.os }} + needs: produce + runs-on: ${{ matrix.os }} + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-14, windows-2025] + steps: + - name: Check out source + uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + - name: Download every OS's archive + uses: actions/download-artifact@v8 + with: + pattern: fixtures-* + path: ${{ runner.temp }}/fixtures + merge-multiple: true + - name: Receive and check them all + shell: bash + env: + DROP_FIXTURE_IN: ${{ runner.temp }}/fixtures + run: cargo test --locked -p drop-cli --test cross_os -- --ignored consume --nocapture diff --git a/cli/tests/cross_os.rs b/cli/tests/cross_os.rs new file mode 100644 index 0000000..e27116c --- /dev/null +++ b/cli/tests/cross_os.rs @@ -0,0 +1,356 @@ +//! Folders archived on one operating system and received on another. +//! +//! The wire is the same everywhere: JSON, big-endian integers, sealed bytes. +//! What differs by platform is how a sender reads a tree and how a receiver +//! writes one, so that is where a cross-OS transfer can go wrong. Two runners +//! cannot swap a transfer code mid-run, so these tests split a transfer in +//! two. One test, on each OS, builds a tree using every kind of name and file +//! that OS can hold and archives it exactly as `drop send` would. The other, +//! on each OS, receives every OS's archive through the real receive path over +//! an in-process relay and checks what landed. +//! +//! Both are `#[ignore]`d. They run in `.github/workflows/cross-os.yml`, which +//! moves the archives between runners: +//! +//! ```text +//! DROP_FIXTURE_OUT=dir cargo test -p drop-cli --test cross_os -- --ignored produce +//! DROP_FIXTURE_IN=dir cargo test -p drop-cli --test cross_os -- --ignored consume +//! ``` +//! +//! See `docs/plans/cross-platform-plan-2026-09-14.md`, phase 4. + +use std::{ + fs, + net::SocketAddr, + path::{Path, PathBuf}, + time::Duration, +}; + +use drop_cli::{ + cancel::Cancel, + consent::Acceptance, + names::Naming, + recv::{self, ReceiveOptions}, + send::{self, SendOptions}, + tar::TarPlan, +}; +use serde_json::{Value, json}; +use tokio::sync::oneshot; + +/// The OS the tests are running on, as it names fixtures. +fn this_os() -> &'static str { + if cfg!(windows) { + "windows" + } else if cfg!(target_os = "macos") { + "macos" + } else { + "linux" + } +} + +/// Contents for a fixture file, derived from its path so the consumer can +/// regenerate them without the producer shipping a copy or a hash. +fn contents_for(path: &str, len: usize) -> Vec { + let mut state = path.bytes().fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| { + (hash ^ u64::from(byte)).wrapping_mul(0x0100_0000_01b3) + }) | 1; + + (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state as u8 + }) + .collect() +} + +struct Tree { + root: PathBuf, + entries: Vec, +} + +impl Tree { + fn file(&mut self, relative: &str, len: usize, executable: bool) { + let path = self.root.join(relative); + fs::create_dir_all(path.parent().expect("a parent")).expect("parent directories"); + fs::write(&path, contents_for(relative, len)).expect("fixture file"); + + #[cfg(unix)] + if executable { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).expect("mode"); + } + + self.entries.push(json!({ + "kind": "file", + "path": relative, + "len": len, + "executable": executable && cfg!(unix), + })); + } + + fn directory(&mut self, relative: &str) { + fs::create_dir_all(self.root.join(relative)).expect("fixture directory"); + self.entries + .push(json!({ "kind": "dir", "path": relative })); + } + + #[cfg(unix)] + fn symlink(&mut self, relative: &str, target: &str) { + let path = self.root.join(relative); + fs::create_dir_all(path.parent().expect("a parent")).expect("parent directories"); + std::os::unix::fs::symlink(target, &path).expect("fixture symlink"); + self.entries + .push(json!({ "kind": "symlink", "path": relative, "target": target })); + } +} + +fn scratch(name: &str) -> PathBuf { + let base = std::env::temp_dir().join(format!( + "drop-cross-os-{}-{}-{name}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_nanos() + )); + fs::create_dir_all(&base).expect("scratch directory"); + base +} + +/// Builds this OS's fixture tree and archives it the way `drop send` does. +#[test] +#[ignore = "run by .github/workflows/cross-os.yml, which moves archives between runners"] +fn produce() { + let out = PathBuf::from(std::env::var("DROP_FIXTURE_OUT").expect("DROP_FIXTURE_OUT")); + fs::create_dir_all(&out).expect("output directory"); + + let base = scratch("produce"); + let mut tree = Tree { + root: base.join("tree"), + entries: Vec::new(), + }; + + tree.file("readme.md", 200, false); + tree.file("empty.txt", 0, false); + tree.file("nested/deeper/data.bin", 3 * 1024 * 1024 + 7, false); + tree.directory("empty-dir"); + tree.file("Łódź 東京 🎉.txt", 64, false); + // Longer than a ustar header holds, so it travels in a long-name record. + tree.file(&format!("{}.txt", "long-name-".repeat(12)), 32, false); + // Deeper than Windows' historic 260-character path limit, once the + // runner's temporary directory is in front of it. + let deep: Vec = (0..12) + .map(|level| format!("level-{level:02}-padding")) + .collect(); + tree.file(&format!("{}/bottom.txt", deep.join("/")), 16, false); + tree.file("scripts/run.sh", 40, true); + + // What only a Unix sender can put in a folder. + #[cfg(unix)] + { + tree.symlink("links/to-readme", "../readme.md"); + tree.file("10:30 standup.md", 48, false); + tree.file("nul.txt", 8, false); + tree.file("trailing dot.", 8, false); + } + + let plan = TarPlan::scan(&tree.root).expect("scan"); + let name = format!("from-{}", this_os()); + + let mut archive = fs::File::create(out.join(format!("{name}.tar"))).expect("archive"); + plan.write_to(&mut archive, |warning| { + panic!("fixture changed while archiving: {warning}") + }) + .expect("archived"); + + let tar_bytes = fs::read(out.join(format!("{name}.tar"))).expect("read back"); + let mut gzip = flate2::write::GzEncoder::new( + fs::File::create(out.join(format!("{name}.tar.gz"))).expect("gzip archive"), + flate2::Compression::new(6), + ); + std::io::Write::write_all(&mut gzip, &tar_bytes).expect("compressed"); + gzip.finish().expect("gzip finished"); + + fs::write( + out.join(format!("{name}.json")), + serde_json::to_vec_pretty(&json!({ "producer": this_os(), "entries": tree.entries })) + .expect("manifest"), + ) + .expect("manifest written"); + + fs::remove_dir_all(&base).ok(); +} + +async fn spawn_relay() -> String { + let state = api::build_state(); + api::start_background_services(state.clone()); + let app = api::build_app(state); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("relay listener"); + let addr: SocketAddr = listener.local_addr().expect("relay address"); + tokio::spawn(async move { api::serve(listener, app).await.expect("relay") }); + tokio::time::sleep(Duration::from_millis(50)).await; + format!("http://{addr}") +} + +/// Sends `archive` and receives it, extracting, into `destination`. +async fn transfer(origin: &str, archive: &Path, destination: &Path) { + let (code_tx, code_rx) = oneshot::channel(); + let mut code_tx = Some(code_tx); + + let sender = tokio::spawn({ + let origin = origin.to_string(); + let archive = archive.to_path_buf(); + async move { + send::run_cancellable( + &archive, + SendOptions { + origin: Some(origin), + compress: None, + path: drop_cli::direct::Path::Relay, + status: false, + rendezvous: drop_cli::direct::Rendezvous::default(), + on_code: Box::new(move |code| { + if let Some(sender) = code_tx.take() { + let _ = sender.send(code.to_string()); + } + }), + }, + Cancel::default(), + ) + .await + .map_err(|error| error.to_string()) + } + }); + + let code = tokio::time::timeout(Duration::from_secs(30), code_rx) + .await + .expect("a code in time") + .expect("a code"); + + recv::run_deciding( + &code, + &ReceiveOptions { + path: drop_cli::direct::Path::Relay, + status: false, + rendezvous: drop_cli::direct::Rendezvous::default(), + origin: Some(origin.to_string()), + out_dir: destination.to_path_buf(), + extract: true, + force: false, + acceptance: Acceptance::Yes, + }, + &mut Acceptance::Yes, + Cancel::default(), + ) + .await + .unwrap_or_else(|error| panic!("receiving {} failed: {error}", archive.display())); + + sender + .await + .expect("sender task") + .unwrap_or_else(|error| panic!("sending {} failed: {error}", archive.display())); +} + +/// Where an entry from the archive is expected to land on this OS. +fn landed(destination: &Path, relative: &str) -> PathBuf { + let naming = Naming::for_this_platform(); + let mut path = destination.join("tree"); + for component in relative.split('/') { + path.push(naming.component(component).as_ref()); + } + path +} + +/// Receives every OS's archive, tar and gzipped tar, and checks what landed. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "run by .github/workflows/cross-os.yml, which moves archives between runners"] +async fn consume() { + let input = PathBuf::from(std::env::var("DROP_FIXTURE_IN").expect("DROP_FIXTURE_IN")); + let origin = spawn_relay().await; + + let mut manifests: Vec = fs::read_dir(&input) + .expect("fixture directory") + .map(|entry| entry.expect("entry").path()) + .filter(|path| { + path.extension() + .is_some_and(|extension| extension == "json") + }) + .collect(); + manifests.sort(); + assert!(!manifests.is_empty(), "no fixtures in {}", input.display()); + + let mut report = Vec::new(); + + for manifest_path in manifests { + let manifest: Value = + serde_json::from_slice(&fs::read(&manifest_path).expect("manifest")).expect("json"); + let producer = manifest["producer"].as_str().expect("producer").to_string(); + let stem = manifest_path + .file_stem() + .expect("stem") + .to_string_lossy() + .into_owned(); + + for suffix in ["tar", "tar.gz"] { + let archive = input.join(format!("{stem}.{suffix}")); + let destination = scratch(&format!("{stem}-{}", suffix.replace('.', "-"))); + transfer(&origin, &archive, &destination).await; + + for entry in manifest["entries"].as_array().expect("entries") { + let relative = entry["path"].as_str().expect("path"); + let path = landed(&destination, relative); + let what = format!("{producer} {suffix} -> {}: {relative}", this_os()); + + match entry["kind"].as_str().expect("kind") { + "file" => { + let len = entry["len"].as_u64().expect("len") as usize; + let arrived = fs::read(&path) + .unwrap_or_else(|error| panic!("{what}: missing ({error})")); + assert!( + arrived == contents_for(relative, len), + "{what}: contents differ" + ); + + #[cfg(unix)] + if entry["executable"].as_bool() == Some(true) { + use std::os::unix::fs::PermissionsExt; + let mode = fs::metadata(&path).expect("metadata").permissions().mode(); + assert!(mode & 0o111 != 0, "{what}: lost its executable bit"); + } + } + "dir" => assert!(path.is_dir(), "{what}: directory missing"), + "symlink" => { + if cfg!(unix) { + let target = fs::read_link(&path) + .unwrap_or_else(|error| panic!("{what}: link missing ({error})")); + assert_eq!( + target.to_string_lossy(), + entry["target"].as_str().expect("target"), + "{what}: link target changed" + ); + } else { + // Skipped with a warning on a platform that + // cannot create links, never half-made. + assert!( + fs::symlink_metadata(&path).is_err(), + "{what}: something was created where a link was skipped" + ); + } + } + other => panic!("{what}: unknown entry kind {other}"), + } + } + + report.push(format!("{producer} {suffix} -> {}: ok", this_os())); + fs::remove_dir_all(&destination).ok(); + } + } + + for line in report { + println!("{line}"); + } +} diff --git a/docs/implementation-checklist.md b/docs/implementation-checklist.md index a090f99..ead3ab2 100644 --- a/docs/implementation-checklist.md +++ b/docs/implementation-checklist.md @@ -501,7 +501,7 @@ compiled. the console is closed, VT processing for the progress line. - [x] Phase 3 — done 2026-09-15, short of a tagged release. `x86_64`/`aarch64-pc-windows-msvc` release zips with a static CRT, `install.ps1`, README install lines. -- [ ] Phase 4 — archives produced on each OS and extracted on the others in CI, +- [x] Phase 4 — written 2026-09-15; first run on its pull request. Archives produced on each OS and extracted on the others in CI, nine pairings. - [ ] Phase 5 — manual Windows ↔ Linux checklist on the user's machine, both carriers, the interface, Ctrl-C, closing the window, the firewall dialog. diff --git a/docs/plans/README.md b/docs/plans/README.md index 78ad170..0301260 100644 --- a/docs/plans/README.md +++ b/docs/plans/README.md @@ -198,3 +198,15 @@ order that serves them, by dependency: 7. **Tag 0.4.0.** 8. Then the NAT proof, once self-hosted rendezvous phase 3 is decided, and cross-platform phases 4–5. + +**Progress, 2026-09-15.** Steps 1–5 and the cross-platform phases 1–4 are written, +as a stack of pull requests waiting for review: #65 (these plans), #66 (move the +installer), #67 (CI on three OSes), #68 (display sanitisation), #69 (browser +removal), #70 (Windows receiver), #71 (`meta_ok` confirmation), #72 (consent), +#73 (cancel and status), #74 (Windows sender), #75 (Windows release and +installers), #76 (cross-OS archives in CI). Separately, #77 fixes a shipped bug +found by the lab: a stray QUIC handshake ended a direct-path send, which was +the long-unexplained `authentication failed`. Not started: consent phase 5 (the +interface screens) and cross-platform phase 5 (the manual Windows checklist). +Tagging 0.4.0 waits on the merges. The NAT proof still waits on the decision in +the self-hosted rendezvous plan's phase 3. diff --git a/docs/plans/cross-platform-plan-2026-09-14.md b/docs/plans/cross-platform-plan-2026-09-14.md index 6783e46..ada2d21 100644 --- a/docs/plans/cross-platform-plan-2026-09-14.md +++ b/docs/plans/cross-platform-plan-2026-09-14.md @@ -1,6 +1,6 @@ # Cross-platform plan: Windows, macOS and Linux, and transfers between them -Status: **active** — phases 0–3 done (2 and 3 on 2026-09-15); 4 and 5 next +Status: **active** — phases 0–4 written (2–4 on 2026-09-15); phase 4's first CI run and phase 5, the manual Windows checklist, next Created: **2026-09-14** Last updated: **2026-09-15** @@ -389,7 +389,7 @@ The wire is platform-neutral (see above), so an archive written on one OS and extracted on another is the cross-OS risk that remains. It can be tested without networking two runners together. -- [ ] A separate workflow, `cross-os.yml`, triggered on pull requests that touch +- [x] A separate workflow, `cross-os.yml`, triggered on pull requests that touch `cli/src/tar.rs`, `cli/src/untar.rs`, `cli/src/payload.rs` or `cli/src/recv.rs`, and nightly. Job `produce`, a matrix over three OSes, builds a fixed tree using every name that platform can hold: nested @@ -398,12 +398,12 @@ without networking two runners together. allows one, an executable script, and colons where the platform allows them. It writes the archive bytes `TarPlan` produces, uncompressed and gzipped, and uploads them with a manifest of expected contents. -- [ ] Job `consume`, needing `produce`, is also a matrix over three OSes. It +- [x] Job `consume`, needing `produce`, is also a matrix over three OSes. It downloads all three archives and extracts each through the real receive path, not only through `TarExtractor`, so the gzip and naming code runs too. It then checks against the manifest: identical bytes for every representable file, and the expected warning for every file that is not. -- [ ] Driven by `#[ignore]`d tests reading `DROP_FIXTURE_OUT` / +- [x] Driven by `#[ignore]`d tests reading `DROP_FIXTURE_OUT` / `DROP_FIXTURE_IN`, so the logic lives in Rust with the rest of the tests and the workflow is only plumbing. @@ -414,6 +414,23 @@ have no channel to swap a code mid-run, and making one would mean a test-only way to fix a transfer code in advance. That is a knob that weakens the one secret the protocol has, and it would exist in shipped binaries. +Written 2026-09-15 on `ci/cross-os-archives`; the nine pairings have their first +run on that branch's pull request. Notes: + +- Fixture contents are derived from each file's path, so a consumer regenerates + what it expects instead of the producer shipping hashes or copies. +- The tree covers: nested folders, an empty folder, an empty file, a 3 MiB file, + Unicode names, a name too long for a ustar header, a path deeper than 260 + characters, and an executable script. Only a Unix producer adds a relative + symlink, `10:30 standup.md`, `nul.txt` and `trailing dot.`, which a Windows + receiver has to rewrite or skip. +- The consumer expects Windows names rewritten exactly as `names::Naming` says, + links skipped (and nothing left in their place) on Windows, and executable bits + kept on Unix. +- **Negative control:** with one byte flipped in the middle of the Linux tar, + `consume` fails with `nested/deeper/data.bin: contents differ`. +- Linux to Linux was run locally, for both the tar and the gzipped tar. + ### Phase 5 — a real Windows machine against a real Linux one CI runners are clean virtual machines without the firewall prompt, antivirus