From a38d2161e8e71e6f65eb6a22579778525fd089b3 Mon Sep 17 00:00:00 2001 From: fylorn <249551762+fylorn@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:02:29 +0800 Subject: [PATCH] Sync: joining adopts the sync data's default key; CLI asks per item; iCloud placeholders download Default gateway key on join - Every installation generates a default gateway key. When the key that default_key points to has the same name on both sides and a different value, joining is not a conflict: the joining device takes the sync data's value, in both join modes (a pinned default key is no longer kept over it on use_remote), and GatewayKeysChanged follows so taken-over clients are updated. Decisions given for that item are ignored. - POST /sync/enable/preview lists it in the new SyncJoinPreview.auto ({ item, action: overwrite, reason: default_key }); the row itself stays in rows. auto is filled for initial configurations too. twcore sync enable - Merging asks about each item that differs; the default is "use the sync data", "keep this device's version" says it replaces the other devices' one, and "keep both" is offered where a new name is suggested. Before, pressing Enter skipped every conflict, which pushed this device's values (its default key among them) onto the other devices. - The device name defaults to the computer name: ComputerName (then LocalHostName) on macOS, the host name on Linux, COMPUTERNAME on Windows. iCloud placeholders (macOS 13 and earlier) - Reading the real name of a legacy .name.icloud placeholder does not download it. The folder location now asks iCloud to download it through NSFileManager startDownloadingUbiquitousItemAtURL:error:, called from the system osascript (JavaScript for Automation) with the path as an argument, at most every 5 minutes per file. - A placeholder (an empty read) is "arriving", not "unreadable": it does not count toward the 30 minutes after which unreadable files are skipped, and is skipped only after a day so it cannot hold uploads back forever. The keyring gets the same treatment. Co-Authored-By: Claude Opus 5.5 --- bin/twcore/src/sync_cmd.rs | 160 +++++++++++++++++--- crates/tw-api/src/lib.rs | 3 +- crates/tw-api/src/sync.rs | 25 ++- crates/tw-control/src/sync.rs | 62 ++++++-- crates/tw-control/tests/sync_devices.rs | 10 +- crates/tw-control/tests/sync_regressions.rs | 104 +++++++++++++ crates/tw-sync/src/engine.rs | 74 ++++++++- crates/tw-sync/src/local.rs | 4 + crates/tw-sync/src/store/folder.rs | 83 +++++++++- crates/tw-sync/tests/regressions.rs | 35 ++++- docs/config.md | 5 + docs/config.zh-CN.md | 2 + 12 files changed, 520 insertions(+), 47 deletions(-) diff --git a/bin/twcore/src/sync_cmd.rs b/bin/twcore/src/sync_cmd.rs index b9965132..a38cb948 100644 --- a/bin/twcore/src/sync_cmd.rs +++ b/bin/twcore/src/sync_cmd.rs @@ -179,19 +179,94 @@ pub fn disable(config: &Path, purge: bool) -> Result<()> { Ok(()) } +/// 一段机器名能不能当设备名：去掉首尾空白，不能有控制字符，太长的截短 +fn usable_name(raw: &str) -> Option { + let n = raw.trim(); + if n.is_empty() || n.chars().any(char::is_control) { + return None; + } + Some(n.chars().take(tw_config::sync::DEVICE_NAME_MAX).collect()) +} + +/// 跑一个系统自带的命令，取它输出的第一行 +#[cfg(unix)] +fn command_line(program: &str, args: &[&str]) -> Option { + let out = std::process::Command::new(program) + .args(args) + .stdin(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .output() + .ok()?; + if !out.status.success() { + return None; + } + usable_name(String::from_utf8_lossy(&out.stdout).lines().next()?) +} + +/// 这台计算机的名称，当设备名称的缺省值：macOS 是系统设置里的「电脑名称」（ComputerName， +/// 取不到用 LocalHostName），Linux 是主机名，Windows 是计算机名 fn host_name() -> String { - for var in ["COMPUTERNAME", "HOSTNAME"] { - if let Ok(v) = std::env::var(var) - && !v.trim().is_empty() - { - return v.trim().to_string(); + #[cfg(target_os = "macos")] + let found = command_line("/usr/sbin/scutil", &["--get", "ComputerName"]) + .or_else(|| command_line("/usr/sbin/scutil", &["--get", "LocalHostName"])) + .or_else(|| command_line("/bin/hostname", &[])); + #[cfg(windows)] + let found = std::env::var("COMPUTERNAME") + .ok() + .and_then(|v| usable_name(&v)); + #[cfg(all(unix, not(target_os = "macos")))] + let found = ["/etc/hostname", "/proc/sys/kernel/hostname"] + .iter() + .find_map(|p| { + std::fs::read_to_string(p) + .ok() + .and_then(|s| usable_name(&s)) + }) + .or_else(|| command_line("hostname", &[])) + .or_else(|| std::env::var("HOSTNAME").ok().and_then(|v| usable_name(&v))); + #[cfg(not(any(unix, windows)))] + let found: Option = None; + found.unwrap_or_else(|| "server".into()) +} + +/// 合并加入时，同步数据里和本机不一样的一项怎么办。**缺省是采用同步数据**：留本机的那一份 +/// 会随这次加入上传，把别的设备上的那一项也改成本机的 —— 那要明确选 +fn decide(row: &tw_api::ImportRow) -> Result { + let differs = if row.differs.is_empty() { + String::new() + } else { + format!(" ({})", row.differs.join(", ")) + }; + let what = match row.item.name.as_str() { + "" => row.item.kind.slug().to_string(), + name => format!("{} `{name}`", row.item.kind.slug()), + }; + eprintln!("{what}{differs} differs between this device and the sync data:"); + eprintln!(" u use the sync data (default)"); + eprintln!( + " k keep this device's version; it is uploaded and replaces the other devices' one" + ); + let rename = row.suggested_name.clone().filter(|_| !row.global); + if let Some(n) = &rename { + eprintln!(" r keep both: the sync data's one is added as {n}"); + } + loop { + let a = ask("Choice", Some("u"))?; + let action = match a.to_ascii_lowercase().as_str() { + "u" => "overwrite", + "k" => "skip", + "r" if rename.is_some() => "rename", + _ => { + eprintln!("type u, k{}", if rename.is_some() { " or r" } else { "" }); + continue; + } + }; + let mut d = serde_json::json!({ "item": row.item, "action": action }); + if action == "rename" { + d["new_name"] = rename.clone().into(); } + return Ok(d); } - std::fs::read_to_string("/etc/hostname") - .map(|s| s.trim().to_string()) - .ok() - .filter(|s| !s.is_empty()) - .unwrap_or_else(|| "server".into()) } pub fn enable(config: &Path) -> Result<()> { @@ -275,11 +350,23 @@ pub fn enable(config: &Path) -> Result<()> { "/sync/enable/preview", Some(serde_json::json!({ "target": target, "passphrase": body.get("passphrase") })), )?; + for a in &preview.auto { + match a.reason { + tw_api::SyncJoinAutoReason::DefaultKey => eprintln!( + "The default gateway key ({}) takes the sync data's value; clients set up \ + on this device are updated to it", + a.item.name + ), + } + } if !preview.initial { + let auto: Vec<&tw_api::ItemRef> = preview.auto.iter().map(|a| &a.item).collect(); let conflicts: Vec<&tw_api::ImportRow> = preview .rows .iter() - .filter(|r| r.status == tw_api::ImportStatus::Conflict) + .filter(|r| { + r.status == tw_api::ImportStatus::Conflict && !auto.contains(&&r.item) + }) .collect(); let new = preview .rows @@ -292,16 +379,17 @@ pub fn enable(config: &Path) -> Result<()> { conflicts.len() ); let replace = yes( - "Replace this device's configuration with the sync data (no keeps this \ - device's version of every item that differs)", + "Replace this device's configuration with the sync data (no merges them; \ + each item that differs is asked)", false, )?; if !replace { body["join"] = "merge".into(); - body["decisions"] = conflicts - .iter() - .map(|r| serde_json::json!({ "item": r.item, "action": "skip" })) - .collect(); + let mut decisions = Vec::new(); + for r in &conflicts { + decisions.push(decide(r)?); + } + body["decisions"] = decisions.into(); } } } @@ -311,3 +399,41 @@ pub fn enable(config: &Path) -> Result<()> { print_view(&v); Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_machine_name_is_trimmed_and_cut_to_a_device_name() { + assert_eq!(usable_name(" Mac mini \n").as_deref(), Some("Mac mini")); + assert_eq!(usable_name(" "), None); + assert_eq!(usable_name("a\u{7}b"), None); + let long = "名".repeat(100); + assert_eq!( + usable_name(&long).unwrap().chars().count(), + tw_config::sync::DEVICE_NAME_MAX + ); + } + + /// macOS 上缺省是「电脑名称」（ComputerName），不是 `server` + #[cfg(target_os = "macos")] + #[test] + fn on_macos_the_default_device_name_is_the_computer_name() { + let want = command_line("/usr/sbin/scutil", &["--get", "ComputerName"]) + .or_else(|| command_line("/usr/sbin/scutil", &["--get", "LocalHostName"])); + let got = host_name(); + assert_ne!(got, "server"); + if let Some(w) = want { + assert_eq!(got, w); + } + } + + /// 别的 Unix 上是主机名 + #[cfg(all(unix, not(target_os = "macos")))] + #[test] + fn elsewhere_the_default_device_name_is_the_host_name() { + let got = host_name(); + assert!(!got.is_empty()); + } +} diff --git a/crates/tw-api/src/lib.rs b/crates/tw-api/src/lib.rs index 16bd6778..cef7b79d 100644 --- a/crates/tw-api/src/lib.rs +++ b/crates/tw-api/src/lib.rs @@ -993,7 +993,8 @@ pub const MSG_CODES: &str = include_str!("../msg-codes.txt"); /// `POST /sync/disable`（[`SyncDisable`]）、`POST /sync/republish`（同步位置空了、keyring 不见了 /// 之后由人选的「重新上传本机配置」，→ [`SyncView`]）。[`SyncState`] 里要人处理的有 /// `keyring_missing`、`emptied`、`misconfigured`，[`SyncView`] 的 `skipped` 是同步位置上跳过了的 -/// 文件数，[`SyncConflict`] 的 `options` 是能选的处理。状态变了发 [`Event::SyncStatus`]；应用别的设备的 +/// 文件数，[`SyncConflict`] 的 `options` 是能选的处理，[`SyncJoinPreview`] 的 `auto` 是加入时 +/// 不用选的项（两边的默认网关密钥同名、值不同时采用同步数据的值）。状态变了发 [`Event::SyncStatus`]；应用别的设备的 /// 版本时配置的来源是 `sync`，改了网关密钥时发 `GatewayKeysChanged`。[`ProviderView`] 多了 /// `sync_excluded`（不同步的上游）和 `sync_url_changed`（地址经同步变更，改之前的地址）。 /// 同步的码是 `sync.*`。安全模式下不自动同步（`paused`） diff --git a/crates/tw-api/src/sync.rs b/crates/tw-api/src/sync.rs index c7dd837f..1e95e604 100644 --- a/crates/tw-api/src/sync.rs +++ b/crates/tw-api/src/sync.rs @@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize}; -use crate::{ImportDecision, ImportRow, ItemRef, Msg}; +use crate::{ImportAction, ImportDecision, ImportRow, ItemRef, Msg}; fn direct() -> String { "direct".to_string() @@ -179,6 +179,29 @@ pub struct SyncJoinPreview { pub initial: bool, /// 同步数据里的每个项目和本机比（[`ImportRow`]）。`initial` 时是空的 pub rows: Vec, + /// core 替人定了、不用选的那几项（两种加入方式都照它办，`decisions` 里给了也不看）。它们 + /// 在 `rows` 里照常列出，界面那一行显示成定下的处理和原因，不当作冲突 + pub auto: Vec, +} + +/// 加入时不用选的一项（[`SyncJoinPreview::auto`]） +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncJoinAuto { + pub item: ItemRef, + /// 怎么处理。现在只有 `overwrite`：采用同步数据 + pub action: ImportAction, + pub reason: SyncJoinAutoReason, +} + +slug_enum! { + /// 为什么不用选。 + pub enum SyncJoinAutoReason { + /// 两边的默认网关密钥（`default_key` 指着的那一把）同名、值不同：每个安装都会自动生成 + /// 一把，加入的设备采用同步数据的值。本机已接管的客户端随之自动更新 + /// （`GatewayKeysChanged`） + DefaultKey = "default_key", + } } slug_enum! { diff --git a/crates/tw-control/src/sync.rs b/crates/tw-control/src/sync.rs index 3a6883b2..e0db8a45 100644 --- a/crates/tw-control/src/sync.rs +++ b/crates/tw-control/src/sync.rs @@ -1158,9 +1158,10 @@ fn is_initial(snap: &Snapshot) -> bool { /// 加入时「合并」：和导入同一套规矩（`tw_pack::import`），本机打底，同步数据逐项并进来。 /// 新增的默认导入，冲突的必须选（覆盖、改名导入、跳过），没选是 `sync.join_undecided`。 /// -/// **两处和导入不一样**：导入进来的插件按导入的规矩是停用的，加入同步时照同步数据里的开关 +/// **三处和导入不一样**：导入进来的插件按导入的规矩是停用的，加入同步时照同步数据里的开关 /// （各设备一致，见方案 6.2；不加密时同步一轮里另有停用的规矩）；导入预览里只有本机有的 -/// 项目不出现，这里也一样 —— 它们会随这次加入上传 +/// 项目不出现，这里也一样 —— 它们会随这次加入上传；两边的默认网关密钥同名、值不同时不用选， +/// 采用同步数据的值（[`engine::adopted_default_key`]，给了别的选择也不看） fn merge_join( local: &Pack, remote: &Pack, @@ -1172,8 +1173,18 @@ fn merge_join( .iter() .map(|d| (crate::backups::item_id(&d.item), d)) .collect(); + let adopt = model::snapshot(local) + .ok() + .and_then(|l| adopted(&l, remote)); let mut out = Vec::new(); for r in &rows { + if adopt.as_ref() == Some(&r.item) { + out.push(tw_pack::import::Decision { + item: r.item.clone(), + action: tw_pack::import::Action::Overwrite, + }); + continue; + } let action = match decided.get(&r.item) { Some(d) => match d.action { tw_api::ImportAction::Import => tw_pack::import::Action::Import, @@ -1340,17 +1351,13 @@ async fn join_preview( let (target, store) = config_target(&s, &req.target)?; let cur = current_pack(&s.cfg).map_err(|m| fail(StatusCode::INTERNAL_SERVER_ERROR, m))?; let local = model::snapshot(&cur.pack).map_err(|e| fail(StatusCode::BAD_REQUEST, e.msg()))?; - if is_initial(&local) { - return Ok(Json(tw_api::SyncJoinPreview { - initial: true, - rows: Vec::new(), - })); - } + let initial = is_initial(&local); let probe = Eng::probe(&store, req.passphrase.as_deref()).await; if probe.existing.is_none() { return Ok(Json(tw_api::SyncJoinPreview { - initial: false, + initial, rows: Vec::new(), + auto: Vec::new(), })); } let encrypted = probe.existing.as_ref().is_some_and(|e| e.encrypted); @@ -1360,16 +1367,49 @@ async fn join_preview( store, &draft(target.clone(), encrypted, req.passphrase.clone(), "preview"), ); - let remote = e.latest().await.map_err(|e| round_fail(&e))?; + let remote = match e.latest().await { + Ok(r) => r, + // 初始配置不出现合并这一步：读不到同步数据（口令还没给）也照样答 + Err(_) if initial => { + return Ok(Json(tw_api::SyncJoinPreview { + initial, + rows: Vec::new(), + auto: Vec::new(), + })); + } + Err(e) => return Err(round_fail(&e)), + }; + let auto = adopted(&local, &remote) + .map(|id| tw_api::SyncJoinAuto { + item: item_ref(&id), + action: tw_api::ImportAction::Overwrite, + reason: tw_api::SyncJoinAutoReason::DefaultKey, + }) + .into_iter() + .collect(); + if initial { + return Ok(Json(tw_api::SyncJoinPreview { + initial, + rows: Vec::new(), + auto, + })); + } // 和导入预览同一套规矩 let rows = tw_pack::import::preview(&cur.pack, &remote, &crate::backups::import_ctx()) .map_err(|e| import_fail(&e))?; Ok(Json(tw_api::SyncJoinPreview { - initial: false, + initial, rows: rows.into_iter().map(crate::backups::row_view).collect(), + auto, })) } +/// 加入时不用选、采用同步数据的那一把默认网关密钥（[`engine::adopted_default_key`]） +fn adopted(local: &Snapshot, remote: &Pack) -> Option { + let remote = model::snapshot(remote).ok()?; + engine::adopted_default_key(local, &remote) +} + async fn enable( State(s): State, Json(req): Json, diff --git a/crates/tw-control/tests/sync_devices.rs b/crates/tw-control/tests/sync_devices.rs index 038dcf7e..de7bdb85 100644 --- a/crates/tw-control/tests/sync_devices.rs +++ b/crates/tw-control/tests/sync_devices.rs @@ -355,8 +355,6 @@ async fn joining_with_a_configuration_merges_by_the_import_rules() { "target": folder(loc.path()), "encrypt": false, "device_name": "B", "join": "merge", "decisions": [ { "item": { "kind": "provider", "name": "shared" }, "action": "rename", "new_name": "shared (A)" }, - // 两边各自生成的默认密钥值不同：留本机的 - { "item": { "kind": "key", "name": "default" }, "action": "skip" }, ], }), ) @@ -369,11 +367,9 @@ async fn joining_with_a_configuration_merges_by_the_import_rules() { assert_eq!(names(&b), ["mine", "shared", "shared (A)"]); a.ok("POST", "/sync/now", Value::Null).await; assert_eq!(names(&a), ["mine", "shared", "shared (A)"], "{}", a.text()); - assert_eq!( - a.config().clients[0].key, - "tw-default-b", - "选了留本机的那一把，A 跟着换" - ); + // 两边各自生成的默认密钥值不同：不用选，B 采用同步数据的值，A 的不变 + assert_eq!(b.config().clients[0].key, "tw-default-a"); + assert_eq!(a.config().clients[0].key, "tw-default-a"); } #[tokio::test] diff --git a/crates/tw-control/tests/sync_regressions.rs b/crates/tw-control/tests/sync_regressions.rs index 5473cf7f..7e8ca29e 100644 --- a/crates/tw-control/tests/sync_regressions.rs +++ b/crates/tw-control/tests/sync_regressions.rs @@ -345,3 +345,107 @@ async fn turning_encryption_off_keeps_the_passphrase() { assert_eq!(v["encrypted"], true, "{v}"); assert!(b.config().sync.unwrap().encrypt); } + +/// 收下 B 上的 `GatewayKeysChanged` 里的名字 +fn keys_changed(rx: &mut tokio::sync::broadcast::Receiver) -> Vec { + let mut out = Vec::new(); + while let Ok(ev) = rx.try_recv() { + if let tw_api::Event::GatewayKeysChanged { names, .. } = ev { + out.extend(names); + } + } + out +} + +/// 已有配置的设备以「合并」加入：两边的默认网关密钥同名、值不同 —— 不当作冲突（不用选，选了 +/// 「跳过」也不看），采用同步数据的值；A 上的那一把不变；B 发 `GatewayKeysChanged` +#[tokio::test] +async fn merging_in_adopts_the_default_key_of_the_sync_data() { + let loc = tempfile::tempdir().unwrap(); + let a = Dev::new('a'); + a.enable(loc.path(), false, "A", Value::Null).await; + let b = Dev::new('b'); + b.edit(|t| { + add_raw( + t, + " - name: mine\n base_url: https://mine.example\n key: sk-m\n", + ) + }) + .await; + b.ok("PUT", "/pinned-keys", json!({ "names": ["default"] })) + .await; + let preview = b + .ok( + "POST", + "/sync/enable/preview", + json!({ "target": folder(loc.path()) }), + ) + .await; + assert_eq!(preview["initial"], false); + assert_eq!( + preview["auto"], + json!([{ "item": { "kind": "key", "name": "default" }, "action": "overwrite", "reason": "default_key" }]), + "{preview}" + ); + assert!( + preview["rows"] + .as_array() + .unwrap() + .iter() + .any(|r| r["item"]["name"] == "default"), + "那一行照常列出：{preview}" + ); + let mut events = b.state.bus().subscribe(); + // 命令行以前的缺省：给默认密钥选「跳过」—— 不看 + b.enable( + loc.path(), + false, + "B", + json!({ + "join": "merge", + "decisions": [ { "item": { "kind": "key", "name": "default" }, "action": "skip" } ], + }), + ) + .await; + let key = |d: &Dev| { + d.config() + .clients + .into_iter() + .find(|c| c.name == "default") + .unwrap() + .key + }; + assert_eq!(key(&b), "tw-default-a", "B 采用同步数据的值"); + assert_eq!(keys_changed(&mut events), ["default"]); + let v = a.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["conflicts"], 0, "{v}"); + assert_eq!(key(&a), "tw-default-a", "A 的默认密钥没被改掉"); + assert!(a.provider("mine").is_some()); +} + +/// 新安装（初始配置）上已经接管了客户端：用同步数据加入时，默认密钥也采用同步数据的值，不把 +/// 本机那一把传上去改掉别的设备的 +#[tokio::test] +async fn a_fresh_device_with_clients_set_up_adopts_the_default_key() { + let loc = tempfile::tempdir().unwrap(); + let a = Dev::new('a'); + a.enable(loc.path(), false, "A", Value::Null).await; + let b = Dev::new('b'); + b.ok("PUT", "/pinned-keys", json!({ "names": ["default"] })) + .await; + let preview = b + .ok( + "POST", + "/sync/enable/preview", + json!({ "target": folder(loc.path()) }), + ) + .await; + assert_eq!(preview["initial"], true); + assert_eq!(preview["auto"][0]["reason"], "default_key", "{preview}"); + let mut events = b.state.bus().subscribe(); + b.enable(loc.path(), false, "B", Value::Null).await; + assert_eq!(b.config().clients[0].key, "tw-default-a"); + assert_eq!(keys_changed(&mut events), ["default"]); + a.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(a.config().clients[0].key, "tw-default-a"); +} diff --git a/crates/tw-sync/src/engine.rs b/crates/tw-sync/src/engine.rs index e92f7870..6ca71619 100644 --- a/crates/tw-sync/src/engine.rs +++ b/crates/tw-sync/src/engine.rs @@ -8,7 +8,8 @@ //! 写着已关闭的 keyring 才算关了加密。 //! 2. 列出 `versions/`，读出每个版本（加密的用版本密钥解开）。读不出来的分两种：**坏了**的（不是 //! 配置包、不是这把密钥加密的、加密的位置上没加密）马上跳过；**可能没传完**的当作尚未到达， -//! 最新那一层有没到的就等下一轮，等了 30 分钟还读不出来也跳过。跳过的计数报给界面。 +//! 最新那一层有没到的就等下一轮，等了 30 分钟还读不出来也跳过。云盘还没下载到本机的（iCloud +//! 的占位文件，文件夹位置请求下载、读成空的）不算读不出来，等一天。跳过的计数报给界面。 //! 3. 找出所有的**头**（没有别的版本依据它的版本）。不止一个时（两台设备同时上传、云盘晚到 //! 的版本）先以它们共同的上一版本为基准两两合并，得到「同步位置的最新」。 //! 4. 以本机基准（上次同步的那一版）为基准，和本机当前配置三方合并（[`crate::merge`]）。基准 @@ -58,6 +59,11 @@ const DEVICE_RECORD_EVERY_MS: u64 = 6 * 3600 * 1000; /// 一个可能没传完的版本（或者 keyring）等多久：之后跳过，不再让它挡着 pub const SKIP_AFTER_MS: u64 = 30 * 60 * 1000; +/// 云盘还没下载到本机的文件（iCloud 的占位文件，读出来是空的）等多久。**不算「读不出来」**： +/// 它是好的，只是还没到（文件夹位置已经请求了下载），所以等得比读不出来的久。一天还没到才 +/// 跳过，免得一直挡着这台设备上传 +pub const ARRIVING_SKIP_AFTER_MS: u64 = 24 * 3600 * 1000; + /// 同步位置看上去空了：连着这么多轮、过了这么久、自己的设备记录也不在了，才算真的清空了 const EMPTY_ROUNDS: u32 = 3; const EMPTY_AFTER_MS: u64 = 10 * 60 * 1000; @@ -254,6 +260,8 @@ enum Got { Broken, /// 可能还没传完 Pending, + /// 还没下载到本机（读出来是空的：云盘的占位文件） + Arriving, /// 列出来之后又没了 Gone, } @@ -602,6 +610,30 @@ fn key_value(it: &Item) -> Option<&Value> { it.units.get(unit::ALL)?.get("key") } +/// 一份配置的默认网关密钥（`default_key` 指着的那一把，不写就是叫 `default` 的那把）的名字 +fn default_key_name(s: &Snapshot) -> String { + s.items + .get(&ItemId::new(ItemKind::Global, model::global::DEFAULT_KEY)) + .and_then(|it| it.units.get(model::global::DEFAULT_KEY)) + .and_then(Value::as_str) + .unwrap_or(tw_config::DEFAULT_KEY) + .to_string() +} + +/// 加入时两边的默认网关密钥同名、值不同：**不当作冲突，采用同步数据的值**。每个安装都会 +/// 自动生成一把默认密钥，新加入的设备向已经在同步的那一方看齐 —— 反过来的话，加入的这一台 +/// 会把别的设备上的默认密钥改掉。本机已接管的客户端随之更新（`GatewayKeysChanged`）。是这样 +/// 时给出那一把的标识 +pub fn adopted_default_key(local: &Snapshot, remote: &Snapshot) -> Option { + let name = default_key_name(local); + if name != default_key_name(remote) { + return None; + } + let id = ItemId::new(ItemKind::Key, name); + let (l, r) = (local.items.get(&id)?, remote.items.get(&id)?); + (key_value(l) != key_value(r)).then_some(id) +} + // ─────────────────────────────────────────────── 等登录的上游 /// 本机配置里的上游（带着这台设备自己的登录） @@ -828,6 +860,18 @@ impl Engine { } return Ok(self.set_mode(None, None, None)); }; + // 还没下载到本机（云盘的占位文件）：等它到，不算读不出来 + if bytes.is_empty() { + let since = *st + .arriving + .entry(KEYRING.to_string()) + .or_insert_with(now_ms); + if now_ms().saturating_sub(since) >= ARRIVING_SKIP_AFTER_MS { + return Err(RoundError::KeyringMissing(KeyringError::Corrupt.msg())); + } + return Err(RoundError::NotArrived); + } + st.arriving.remove(KEYRING); let hash: [u8; 32] = Sha256::digest(&bytes).into(); if self.mode_of == Some(Some(hash)) { return Ok(self.sealed); @@ -902,7 +946,7 @@ impl Engine { }; // 空的：云盘的占位文件、正在写 if bytes.is_empty() { - return Ok(Got::Pending); + return Ok(Got::Arriving); } // 认得出是配置包（ZIP）或者加密过的：读不出来多半是没传完 let shaped = keyring::is_age(&bytes) || bytes.starts_with(b"PK\x03\x04"); @@ -1011,22 +1055,26 @@ impl Engine { match self.fetch(n).await? { Got::Ok(v) => { st.unreadable.remove(&n.id()); + st.arriving.remove(&n.id()); sc.names.push(n.clone()); sc.ok.push(v); } Got::TooNew(cv) => { st.unreadable.remove(&n.id()); + st.arriving.remove(&n.id()); sc.names.push(n.clone()); sc.too_new.push((n.clone(), cv)); } Got::Broken => { st.unreadable.remove(&n.id()); + st.arriving.remove(&n.id()); sc.skipped.push(n.clone()); if n.device == me { sc.own_broken.push(n.clone()); } } Got::Pending => { + st.arriving.remove(&n.id()); let since = *st.unreadable.entry(n.id()).or_insert(now); if now.saturating_sub(since) >= SKIP_AFTER_MS { sc.skipped.push(n.clone()); @@ -1035,11 +1083,22 @@ impl Engine { sc.pending.push(n.clone()); } } + Got::Arriving => { + st.unreadable.remove(&n.id()); + let since = *st.arriving.entry(n.id()).or_insert(now); + if now.saturating_sub(since) >= ARRIVING_SKIP_AFTER_MS { + sc.skipped.push(n.clone()); + } else { + sc.names.push(n.clone()); + sc.pending.push(n.clone()); + } + } Got::Gone => {} } } - st.unreadable - .retain(|k, _| k == KEYRING || all.iter().any(|n| &n.id() == k)); + let listed = |k: &String| k == KEYRING || all.iter().any(|n| &n.id() == k); + st.unreadable.retain(|k, _| listed(k)); + st.arriving.retain(|k, _| listed(k)); sc.all = all; Ok(sc) } @@ -2254,7 +2313,8 @@ impl Engine { /// 加入已有的同步。`merged` 是本机按导入的规则合并之后的完整配置包（「合并」）；`None` 是 /// 用同步数据替换本机配置（新安装、或者选了「使用同步数据」）—— 那时本机已接管的客户端 - /// 正在用的网关密钥并入同步数据，不同步的上游留在本机。 + /// 正在用的网关密钥并入同步数据（默认密钥除外：采用同步数据的值，见 + /// [`adopted_default_key`]），不同步的上游留在本机。 /// /// 本机先照结果改，再上传一个依据同步位置最新版本的新版本，作为这台设备的基准。两种方式 /// 下，别的设备上登录着、这台没登录的上游都在这台停用着，**上传的是同步的样子**（启用着）。 @@ -2297,10 +2357,12 @@ impl Engine { let mut target = match &merged { Some(p) => model::snapshot(p).map_err(|e| RoundError::Local(e.msg()))?, None => { + // 已接管的客户端在用的密钥并入同步数据；默认密钥例外，采用同步数据的值 let mut t = remote.snap.clone(); let pinned = self.host.pinned_keys(); + let adopted = adopted_default_key(&local, &remote.snap); for (id, it) in local.of_kind(ItemKind::Key) { - if pinned.contains(&id.name) { + if pinned.contains(&id.name) && adopted.as_ref() != Some(id) { t.items.insert(id.clone(), it.clone()); } } diff --git a/crates/tw-sync/src/local.rs b/crates/tw-sync/src/local.rs index 904d5c09..d1584626 100644 --- a/crates/tw-sync/src/local.rs +++ b/crates/tw-sync/src/local.rs @@ -60,6 +60,10 @@ pub struct State { /// 同步位置上读不出来的文件：名字 → 第一次读不出来的时刻。读出来了就去掉 #[serde(skip_serializing_if = "BTreeMap::is_empty")] pub unreadable: BTreeMap, + /// 同步位置上还没下载到本机的文件（云盘的占位文件）：名字 → 第一次看到的时刻。和读不出来 + /// 的分开算（等得更久） + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + pub arriving: BTreeMap, /// 同步位置看上去是空的：从什么时候起、连着几轮 pub empty_since_ms: Option, pub empty_rounds: u32, diff --git a/crates/tw-sync/src/store/folder.rs b/crates/tw-sync/src/store/folder.rs index a6ae4891..418bdcfc 100644 --- a/crates/tw-sync/src/store/folder.rs +++ b/crates/tw-sync/src/store/folder.rs @@ -18,9 +18,70 @@ use super::{ too_large, }; -/// iCloud Drive 还没下载到本机的文件，在目录里是一个 `.名字.icloud` 的占位文件 +/// iCloud Drive 还没下载到本机的文件，在目录里是一个 `.名字.icloud` 的占位文件（macOS 13 及 +/// 以前；之后的 iCloud Drive 文件照真名列出，读的时候系统自己去下载） const ICLOUD: &str = ".icloud"; +/// 同一个占位文件多久才再请求一次下载 +#[cfg(target_os = "macos")] +const DOWNLOAD_AGAIN_AFTER: std::time::Duration = std::time::Duration::from_secs(5 * 60); + +/// 请求下载的脚本（JavaScript for Automation）：路径是第一个参数，答 `true` / `false` +#[cfg(target_os = "macos")] +const DOWNLOAD_SCRIPT: &str = "function run(argv) { ObjC.import('Foundation'); \ + return String($.NSFileManager.defaultManager.startDownloadingUbiquitousItemAtURLError(\ + $.NSURL.fileURLWithPath(argv[0]), null)); }"; + +/// 让 iCloud 把一个还没下载的文件下载下来。**旧式占位文件读真名不会触发下载**，要明确请求： +/// 苹果的接口是 `-[NSFileManager startDownloadingUbiquitousItemAtURL:error:]`（macOS 10.7 起都有； +/// `brctl download` 在新系统上已经没有了）。经系统自带的 `osascript` 以 JavaScript for +/// Automation 调它，core 里不写原生调用；路径作为参数传进去，不拼进脚本。只请求、不等下载完： +/// 下载完了真名的文件就出现了，文件夹的监听随之再来一轮。别的系统什么都不做 +fn start_download(real: &Path) { + #[cfg(target_os = "macos")] + { + use std::collections::HashMap; + use std::sync::{Mutex, OnceLock}; + use std::time::Instant; + static ASKED: OnceLock>> = OnceLock::new(); + let asked = ASKED.get_or_init(Default::default); + if let Ok(mut m) = asked.lock() { + if m.get(real) + .is_some_and(|t| t.elapsed() < DOWNLOAD_AGAIN_AFTER) + { + return; + } + m.retain(|_, t| t.elapsed() < DOWNLOAD_AGAIN_AFTER); + m.insert(real.to_path_buf(), Instant::now()); + } + let child = std::process::Command::new("/usr/bin/osascript") + .args(["-l", "JavaScript", "-e", DOWNLOAD_SCRIPT]) + .arg(real) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn(); + match child { + // 不等它：另起一个线程收尸，最多等 10 秒 + Ok(mut c) => { + std::thread::spawn(move || { + for _ in 0..100 { + if !matches!(c.try_wait(), Ok(None)) { + return; + } + std::thread::sleep(std::time::Duration::from_millis(100)); + } + let _ = c.kill(); + let _ = c.wait(); + }); + } + Err(e) => tracing::debug!("iCloud could not be asked to download a file: {e}"), + } + } + #[cfg(not(target_os = "macos"))] + let _ = real; +} + /// 读一个文件，**边读边数**：超过上限就不读了 fn read_limited(p: &Path) -> std::io::Result, ()>> { use std::io::Read; @@ -206,11 +267,12 @@ impl Store for Folder { if !root.is_dir() { return Err(io(&root, e)); } - // iCloud 还没下载的：有，只是还没到 —— 读成空的（调用方当作尚未到达） + // iCloud 还没下载的：有，只是还没到 —— 请求下载，读成空的（调用方当作尚未到达） let placeholder = p .file_name() .map(|n| p.with_file_name(format!(".{}{ICLOUD}", n.to_string_lossy()))); if placeholder.is_some_and(|ph| ph.is_file()) { + start_download(&p); Ok(Some(Vec::new())) } else { Ok(None) @@ -471,6 +533,23 @@ mod tests { ); } + /// 请求下载的那段脚本在这台 Mac 上跑得通：对一个不在 iCloud 里的文件，接口答 `false`（不是 + /// 脚本出错、不是 osascript 崩了） + #[cfg(target_os = "macos")] + #[test] + fn the_download_request_script_runs() { + let d = tempfile::tempdir().unwrap(); + let f = d.path().join("x"); + std::fs::write(&f, b"x").unwrap(); + let out = std::process::Command::new("/usr/bin/osascript") + .args(["-l", "JavaScript", "-e", DOWNLOAD_SCRIPT]) + .arg(&f) + .output() + .unwrap(); + assert!(out.status.success(), "{out:?}"); + assert_eq!(String::from_utf8_lossy(&out.stdout).trim(), "false"); + } + #[tokio::test] async fn an_oversized_file_is_not_read() { let (d, f) = bed(); diff --git a/crates/tw-sync/tests/regressions.rs b/crates/tw-sync/tests/regressions.rs index 08440fb7..992b5bdf 100644 --- a/crates/tw-sync/tests/regressions.rs +++ b/crates/tw-sync/tests/regressions.rs @@ -215,7 +215,7 @@ fn forge_plain_version(loc: &Path, config_yaml: String) -> String { /// 把本机记着的「从什么时候起」往前拨，模拟时间过去了 fn age_state(e: &Engine, by_ms: u64) { let mut st = e.local().state(); - for t in st.unreadable.values_mut() { + for t in st.unreadable.values_mut().chain(st.arriving.values_mut()) { *t = t.saturating_sub(by_ms); } st.empty_since_ms = st.empty_since_ms.map(|t| t.saturating_sub(by_ms)); @@ -953,7 +953,8 @@ async fn a_version_that_never_finishes_arriving_is_skipped_after_a_while() { assert!(t.b.provider("one").is_some()); } -/// iCloud 还没下载的最新版本（只有占位文件）：当作尚未到达 +/// iCloud 还没下载的最新版本（只有旧式占位文件）：当作尚未到达，**不算读不出来**——过了 +/// 30 分钟照样等（文件夹位置请求了下载）；一天还没到才跳过 #[tokio::test] async fn an_icloud_placeholder_on_top_is_waited_for() { let mut t = bed(false).await; @@ -972,9 +973,39 @@ async fn an_icloud_placeholder_on_top_is_waited_for() { ) .unwrap(); assert!(matches!(t.eb.round().await, Err(RoundError::NotArrived))); + age_state(&t.eb, SKIP_AFTER_MS + 1000); + assert!( + matches!(t.eb.round().await, Err(RoundError::NotArrived)), + "占位文件不按读不出来的 30 分钟跳过" + ); std::fs::rename(parked.path().join(&file), &real).unwrap(); t.eb.round().await.unwrap(); assert!(t.b.provider("one").is_some()); + assert!(t.eb.state().arriving.is_empty()); +} + +/// 一直没下载下来的占位文件：一天之后跳过，不再挡着这台设备上传 +#[tokio::test] +async fn an_icloud_placeholder_that_never_downloads_is_skipped_after_a_day() { + let mut t = bed(false).await; + let up = t.ea.round().await.unwrap().uploaded; + assert!(up.is_none()); + let head = versions(t.loc.path()).last().unwrap().clone(); + let seq: u64 = head[..10].parse().unwrap(); + std::fs::write( + t.loc + .path() + .join("versions") + .join(format!(".{:010}-cccccccc.twconfig.icloud", seq + 1)), + b"placeholder", + ) + .unwrap(); + t.b.edit(|s| add_provider(s, "fromb", "https://b.example", "sk-b")); + assert!(matches!(t.eb.round().await, Err(RoundError::NotArrived))); + age_state(&t.eb, tw_sync::engine::ARRIVING_SKIP_AFTER_MS + 1000); + let r = t.eb.round().await.unwrap(); + assert_eq!(r.skipped, 1); + assert!(r.uploaded.is_some()); } // ─────────────────────────────────────────────── 空的同步位置 diff --git a/docs/config.md b/docs/config.md index 63b3b118..32abe538 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1522,6 +1522,11 @@ plugins and the general settings, keys included. `listen`, `retention` and OAuth tokens. An upstream signed in on another device stays turned off on this one until it is signed in here; that does not turn it off anywhere else. +A device that joins with a configuration of its own merges it item by item; +items that differ are chosen one by one, except the default gateway key: every +installation makes one, so the joining device takes the sync data's value and +the clients set up on it are updated. + With encryption on, versions that are not encrypted are never read. A keyring that went missing, or an older one put back, does not turn encryption off: the app asks whether to put back the copy this device keeps. A location that reads diff --git a/docs/config.zh-CN.md b/docs/config.zh-CN.md index 930fa9c0..b9b7ee4b 100644 --- a/docs/config.zh-CN.md +++ b/docs/config.zh-CN.md @@ -1107,6 +1107,8 @@ export const manifest = { 同步的内容即配置包带的内容：上游、出站代理、网关密钥、别名、分组、路由、价目表、安全规则、插件和全局设置，包括密钥。`listen`、`retention` 和 `sync` 本身留在本机，ChatGPT 账号的登录、OAuth 的令牌也一样。别的设备上登录着的上游在这台设备上停用着，在这台登录之后才启用；这不会让它在别的设备上停用。 +已有配置的设备加入时逐项合并，不一样的项目逐项选择；默认网关密钥例外：每个安装都会生成一把，加入的设备采用同步数据的值，本机已接管的客户端随之更新。 + 开着加密时，不加密的版本一概不读。keyring 不见了、被换成更早的那一份，都不会关掉加密：应用会问要不要放回这台设备记着的那一份。同步位置读起来是空的时，要再看几次才提示，停不停止同步由你决定。