diff --git a/bin/twcore/src/pack_cli.rs b/bin/twcore/src/pack_cli.rs index f02b860e..be68ffc8 100644 --- a/bin/twcore/src/pack_cli.rs +++ b/bin/twcore/src/pack_cli.rs @@ -381,9 +381,7 @@ pub fn restore(path: &Path, id: &str, dry_run: bool) -> Result<()> { let pinned = tw_pack::disk::read_pinned(path); let after = if tw_pack::model::snapshot(&local).is_err() { println!("the configuration here does not load; the whole version comes back"); - let last = tw_pack::backup::list(path) - .ok() - .and_then(|all| all.last().and_then(|b| tw_pack::backup::open(b).ok())); + let last = tw_pack::backup::last_readable(path, &cur.text); tw_pack::restore::whole(&backup, &cur.text, last.as_ref()) } else { let rows = tw_pack::restore::preview(&local, &backup, &pinned, true).map_err(model)?; diff --git a/crates/tw-api/msg-codes.txt b/crates/tw-api/msg-codes.txt index 8a1cda17..95d160d9 100644 --- a/crates/tw-api/msg-codes.txt +++ b/crates/tw-api/msg-codes.txt @@ -5,6 +5,9 @@ # test only produced by tests; never reaches a UI backup.io backup.not_found +backup.pinned.name_too_long +backup.pinned.source_too_long +backup.pinned.too_many backup.prefs_too_large backup.write_failed config.alias_blank_model @@ -516,6 +519,7 @@ pack.not_encrypted pack.passphrase_short pack.schema_too_new pack.too_large +pack.too_much_work pack.too_new pack.verify_failed pack.wrong_key diff --git a/crates/tw-api/src/ep.rs b/crates/tw-api/src/ep.rs index 34735dc5..fa991be1 100644 --- a/crates/tw-api/src/ep.rs +++ b/crates/tw-api/src/ep.rs @@ -53,7 +53,7 @@ endpoints! { /// 恢复到一个版本会改什么。**不写任何东西** RestorePreview: POST "/backups/preview", api::RestorePreviewRequest => api::RestorePreview; /// 恢复到一个版本(全部,或者选的几项)。恢复本身也留一个版本 - Restore: POST "/backups/restore", api::RestoreRequest => api::ConfigWritten; + Restore: POST "/backups/restore", api::RestoreRequest => api::RestoreWritten; /// Lite 的通用偏好 GetClientPrefs: GET "/client-prefs", () => api::ClientPrefs; PutClientPrefs: PUT "/client-prefs", api::ClientPrefs => (); diff --git a/crates/tw-api/src/lib.rs b/crates/tw-api/src/lib.rs index cef7b79d..762163e5 100644 --- a/crates/tw-api/src/lib.rs +++ b/crates/tw-api/src/lib.rs @@ -964,17 +964,20 @@ pub const MSG_CODES: &str = include_str!("../msg-codes.txt"); /// `POST /config/rollback`(`ConfigVersion`、`RollbackRequest`)删了,换成 `GET /backups` /// ([`BackupEntry`]:本机最近 10 个版本,开着同步时连同同步位置上的,同一个版本一行)、 /// `POST /backups/file`(另存为配置包)、`POST /backups/preview`([`RestorePreview`]:按项目 -/// 列出差异)、`POST /backups/restore`(可以只恢复几项,本机设置和 ChatGPT 的登录留着,接管的 -/// 客户端在用的密钥不删)。一版是一个完整的配置包,连同插件和 Lite 的通用偏好。新端点 +/// 列出差异;给了 `items` 是只恢复这几项时会动什么)、`POST /backups/restore`(可以只恢复几项, +/// 本机设置和 ChatGPT 的登录留着,接管的客户端在用的密钥不删;→ [`RestoreWritten`],`rows` 是 +/// 实际动了的项目,和同样条件下的预览一样)。一版是一个完整的配置包,连同插件和 Lite 的通用偏好。新端点 /// `GET /config/items`([`ItemNode`])、`POST /config/export`([`ExportRequest`] → /// [`ExportedFile`])、`POST /config/import/preview`([`ImportSource`] → [`ImportPreview`]:逐项 /// 新增、相同、冲突、错误)和 `POST /config/import`([`ImportRequest`]:逐项导入、覆盖、改名 /// 导入或跳过)。Lite 的通用偏好由 core 存:`GET` / `PUT /client-prefs`([`ClientPrefs`], /// 不解析,最大 64 KiB),变了发 [`Event::ClientPrefsChanged`]。`PUT /pinned-keys` -/// ([`PinnedKeys`]:`source` 是这个 Lite 的标识,几个 Lite 各报各的、core 取并集,30 天没更新 -/// 的来源不算)报告接管的客户端在用哪几把密钥。恢复时留下的项目还引用着的不删 -/// ([`KeptReason`] 的 `referenced`),所选项目引用的、本机已没有的一起恢复;不带密钥的包导入时 -/// 比较和覆盖都沿用本机的密钥。导入覆盖、恢复改了或删了网关密钥时发 +/// ([`PinnedKeys`]:`source` 是这个 Lite 的标识,几个 Lite 各报各的、core 取并集,来源不过期, +/// 超出上限是 `backup.pinned.*`)报告接管的客户端在用哪几把密钥。恢复时留下的项目还引用着的、 +/// 同步设置走的出站代理不删([`KeptReason`] 的 `referenced`),所选项目引用的、本机已没有的一起 +/// 恢复;恢复同步位置上的版本时本机不同步的上游不列成删除。不带密钥的包导入时比较和覆盖都沿用 +/// 本机的密钥,**只在目的地不变时**(协议、主机、端口、AWS 区域、OAuth 令牌端点、代理地址 +/// 一样);改名导入的不沿用,缺着的标 `missing_key`(代理的密码也算)。导入覆盖、恢复改了或删了网关密钥时发 /// [`Event::GatewayKeysChanged`]。[`ConfigOrigin`] 删了 `rollback`,多了 `sync`、`import`、 /// `restore`、`repair`(一键修复以前记成 `ui`)。备份相关的端点在安全模式下照样可用。配置包的 /// 码是 `pack.*`,导入的是 `import.*`,备份与恢复的是 `backup.*`。停用着的上游可以缺凭据 diff --git a/crates/tw-api/src/pack.rs b/crates/tw-api/src/pack.rs index ab10f724..a86c9119 100644 --- a/crates/tw-api/src/pack.rs +++ b/crates/tw-api/src/pack.rs @@ -311,6 +311,10 @@ pub struct BackupFileRequest { #[cfg_attr(feature = "ts", derive(ts_rs::TS))] pub struct RestorePreviewRequest { pub id: String, + /// 只恢复这几项时会动什么(连同一起恢复的、留着的)。不给是全部。和 + /// [`RestoreRequest::items`] 一样,预览和写入走同一个计划 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub items: Option>, } /// 恢复会改什么。**什么都不写**。 @@ -371,6 +375,17 @@ pub struct RestoreRequest { pub items: Option>, } +/// `POST /backups/restore` 的结果。 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct RestoreWritten { + /// 写下去之后的配置版本 + pub version: String, + /// 实际动了的项目:所选的,连同一起恢复的(`kept` 为空)和留着没改的(`kept` 写明原因)。 + /// 和同样条件下的 [`RestorePreview::rows`] 一样。安全模式下整份换回时每一项都是新增 + pub rows: Vec, +} + /// Lite 的通用偏好(语言、外观、菜单栏、通知、检查更新),**core 不解析**,最大 64 KiB。 #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[cfg_attr(feature = "ts", derive(ts_rs::TS))] @@ -386,7 +401,8 @@ pub const CLIENT_PREFS_MAX: usize = 64 * 1024; /// `PUT /pinned-keys`:这一个 Lite 接管的客户端正在用的网关密钥。恢复、导入、同步不删它们。 /// /// **按来源记**:几个 Lite 连着同一个 core(远程模式)时各报各的,core 取并集;一个来源报的 -/// 换掉它自己上次报的。30 天没更新的来源不再算 +/// 换掉它自己上次报的。来源不过期(Lite 每次连上 core 都重报一次);最多记 64 个来源,多出来 +/// 的去掉最久没报的。来源最长 128 字节,一次最多 1024 把、每个名字最长 256 字节,超出是 400 #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[cfg_attr(feature = "ts", derive(ts_rs::TS))] pub struct PinnedKeys { diff --git a/crates/tw-control/src/backups.rs b/crates/tw-control/src/backups.rs index 4286d3ad..6aa48518 100644 --- a/crates/tw-control/src/backups.rs +++ b/crates/tw-control/src/backups.rs @@ -349,10 +349,12 @@ async fn land( changed_keys: Vec, ) -> Result { let path = s.config_path().to_path_buf(); - // 偏好的上限导入、恢复也照样查:一份带着超大偏好的包不该绕过 `PUT /client-prefs` 的那道关 - if core_keeps_prefs() + // 偏好的上限导入、恢复也照样查:一份带着超大偏好的包不该绕过 `PUT /client-prefs` 的那道关。 + // **只查要写下去的新偏好**:已经存着的一份超大的(上限之前存的)不该让每一次导入、恢复都失败 + let prefs_changed = core_keeps_prefs() && after.client_prefs != before.client_prefs; + if prefs_changed && let Some(p) = &after.client_prefs - && p.to_string().len() > tw_api::CLIENT_PREFS_MAX + && tw_pack::disk::prefs_too_large(p) { return Err(prefs_too_large()); } @@ -364,7 +366,6 @@ async fn land( e, ) })?; - let prefs_changed = core_keeps_prefs() && after.client_prefs != before.client_prefs; if prefs_changed && let Err(e) = tw_pack::disk::write_prefs(&path, after.client_prefs.as_ref()) { undo.restore(); @@ -742,23 +743,10 @@ async fn restore_preview( None => 0, }; // 配置读不进来:整份换回那一版,每一项都是「新增」 - let rows = match tw_pack::restore::preview( - &local, - &backup, - &s.cfg.pinned_keys(), - core_keeps_prefs(), - ) { + let opts = restore_options(&s, &req.id, req.items.as_deref(), &local, &backup); + let rows = match tw_pack::restore::preview_opts(&local, &backup, &opts) { Ok(rows) => rows.into_iter().map(restore_row).collect(), - Err(_) if model::snapshot(&local).is_err() => model::snapshot(&backup) - .map_err(|e| model_fail(e, Some(&backup)))? - .items - .keys() - .map(|id| tw_api::RestoreRow { - item: item_ref(id), - change: tw_api::RestoreChange::Add, - kept: None, - }) - .collect(), + Err(_) if model::snapshot(&local).is_err() => whole_rows(&backup)?, Err(e) => return Err(model_fail(e, Some(&backup))), }; Ok(Json(tw_api::RestorePreview { @@ -769,10 +757,54 @@ async fn restore_preview( })) } +/// 安全模式下整份换回:那一版的每一项都是「新增」 +fn whole_rows(backup: &Pack) -> Result, Fail> { + Ok(model::snapshot(backup) + .map_err(|e| model_fail(e, Some(backup)))? + .items + .keys() + .map(|id| tw_api::RestoreRow { + item: item_ref(id), + change: tw_api::RestoreChange::Add, + kept: None, + }) + .collect()) +} + +/// 恢复的条件:预览和写入用同一份。恢复同步位置上的版本时,本机不同步的上游和随之不同步的 +/// 项目不参与比较(规格 7.2):那一版里本来就没有它们 +fn restore_options( + s: &ControlState, + id: &str, + items: Option<&[tw_api::ItemRef]>, + local: &Pack, + backup: &Pack, +) -> tw_pack::restore::Options { + let skip = if id.starts_with(SYNC_ID) { + let names: BTreeSet = tw_config::try_parse(&local.config_yaml) + .ok() + .and_then(|c| c.sync) + .map(|sync| sync.exclude.into_iter().collect()) + .unwrap_or_default(); + match (model::snapshot(local), model::snapshot(backup)) { + (Ok(l), Ok(b)) if !names.is_empty() => tw_sync::engine::excluded(&names, &[&l, &b]), + _ => BTreeSet::new(), + } + } else { + BTreeSet::new() + }; + tw_pack::restore::Options { + pinned: s.cfg.pinned_keys(), + with_prefs: core_keeps_prefs(), + items: items.map(|xs| xs.iter().map(item_id).collect()), + skip, + } +} + async fn restore( State(s): State, Json(req): Json, -) -> Result, Fail> { +) -> Result, Fail> { let raw = backup_pack(&s, &req.id).await?; let (backup, _) = tw_pack::import::prepare(raw.clone()).map_err(|e| import_fail(e, &raw))?; let (cur, local) = current(&s, PackOrigin::Restore)?; @@ -782,35 +814,26 @@ async fn restore( current: cur.version(), })); } - let picked: Option> = req - .items - .as_ref() - .map(|xs| xs.iter().map(item_id).collect()); - let (after, changed) = if model::snapshot(&local).is_err() { + let (after, changed, rows) = if model::snapshot(&local).is_err() { // 安全模式:现在这份读不进来,整份换回去 - // 本机那几节取不到时(原文坏到读不成 YAML)用最近一个本机备份里的 - let last = tw_pack::backup::list(s.config_path()) - .ok() - .and_then(|all| all.last().and_then(|b| tw_pack::backup::open(b).ok())); + // 本机那几节取不到时(原文坏到读不成 YAML)用最近一个读得成 YAML 的本机备份里的 + let last = tw_pack::backup::last_readable(s.config_path(), &cur.text); let mut whole = tw_pack::restore::whole(&backup, &cur.text, last.as_ref()); if !core_keeps_prefs() { whole.client_prefs = local.client_prefs.clone(); } - (whole, Vec::new()) + let rows = whole_rows(&backup)?; + (whole, Vec::new(), rows) } else { - let applied = tw_pack::restore::apply( - &local, - &backup, - picked.as_deref(), - &s.cfg.pinned_keys(), - core_keeps_prefs(), - ) - .map_err(|e| model_fail(e, Some(&backup)))?; - (applied.pack, applied.changed_keys) + let opts = restore_options(&s, &req.id, req.items.as_deref(), &local, &backup); + let applied = tw_pack::restore::apply_opts(&local, &backup, &opts) + .map_err(|e| model_fail(e, Some(&backup)))?; + let rows = applied.rows.into_iter().map(restore_row).collect(); + (applied.pack, applied.changed_keys, rows) }; crate::plugins::guard_raw_write(&s, &cur.text, &after.config_yaml).await?; let version = land(&s, &local, &after, &cur.version(), Origin::Restore, changed).await?; - Ok(Json(tw_api::ConfigWritten { version })) + Ok(Json(tw_api::RestoreWritten { version, rows })) } // ─────────────────────────────────────────────────────────── 通用偏好、固定密钥 @@ -826,7 +849,7 @@ async fn put_prefs( Json(req): Json, ) -> Result, Fail> { if let Some(p) = &req.prefs - && p.to_string().len() > tw_api::CLIENT_PREFS_MAX + && tw_pack::disk::prefs_too_large(p) { return Err(prefs_too_large()); } @@ -853,10 +876,31 @@ async fn pin_keys( State(s): State, Json(req): Json, ) -> Result, Fail> { + use crate::config::PinnedError; + use tw_pack::disk::{PINNED_NAME_MAX, PINNED_NAMES_MAX, PINNED_SOURCE_MAX, PinnedTooLarge}; let names: BTreeSet = req.names.into_iter().collect(); s.cfg .set_pinned_keys(&req.source, names) - .map_err(|e| io_fail(&tw_pack::disk::pinned_path(s.config_path()), e))?; + .map_err(|e| match e { + PinnedError::Io(e) => io_fail(&tw_pack::disk::pinned_path(s.config_path()), e), + PinnedError::TooLarge(t) => fail( + StatusCode::BAD_REQUEST, + match t { + PinnedTooLarge::Source => msg!( + "backup.pinned.source_too_long", max = PINNED_SOURCE_MAX => + "The source of the pinned keys is longer than {max} bytes." + ), + PinnedTooLarge::Names => msg!( + "backup.pinned.too_many", max = PINNED_NAMES_MAX => + "More than {max} pinned keys were reported at once." + ), + PinnedTooLarge::Name => msg!( + "backup.pinned.name_too_long", max = PINNED_NAME_MAX => + "A pinned key name is longer than {max} bytes." + ), + }, + ), + })?; Ok(Json(())) } diff --git a/crates/tw-control/src/config.rs b/crates/tw-control/src/config.rs index 9d988b59..d3a5b9c0 100644 --- a/crates/tw-control/src/config.rs +++ b/crates/tw-control/src/config.rs @@ -379,6 +379,8 @@ impl ConfigManager { plugins = ?healed, "plugin files that did not match the configuration were put back from a backup" ); + // 网关已经照找回来之前的文件读过插件(对不上的停用了):照找回来的重读一遍 + self.gateway.reload_plugins(); } if let Ok(cur) = self.current() { self.backup(tw_pack::backup::record( @@ -389,26 +391,31 @@ impl ConfigManager { } } - /// 本机接管的客户端在用的网关密钥(固定密钥):各个来源报上来的并集,30 天没更新的来源 - /// 不算。恢复、导入、同步不删它们 + /// 本机接管的客户端在用的网关密钥(固定密钥):各个来源报上来的并集。恢复、导入、同步 + /// 不删它们 pub fn pinned_keys(&self) -> std::collections::BTreeSet { self.pinned .lock() - .map(|g| tw_pack::disk::pinned_union(&g, now_ms())) + .map(|g| tw_pack::disk::pinned_union(&g)) .unwrap_or_default() } /// 一个来源(一个 Lite)报上来它的固定密钥:换掉它上次报的,存进数据目录 - /// (`pinned-keys.json`),重启之后还在 + /// (`pinned-keys.json`),重启之后还在。**读、改、写都在锁里**:两个 Lite 同时报, + /// 谁也不会盖掉谁的 pub fn set_pinned_keys( &self, source: &str, names: std::collections::BTreeSet, - ) -> std::io::Result<()> { - let all = tw_pack::disk::write_pinned(&self.path, source, &names)?; - if let Ok(mut g) = self.pinned.lock() { - *g = all; - } + ) -> Result<(), PinnedError> { + let mut g = self + .pinned + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let next = tw_pack::disk::with_pinned(&g, source, &names, now_ms()) + .map_err(PinnedError::TooLarge)?; + tw_pack::disk::write_pinned_sources(&self.path, &next).map_err(PinnedError::Io)?; + *g = next; Ok(()) } @@ -641,6 +648,14 @@ fn bad_path(e: tw_yaml::PatchError) -> ApplyError { ApplyError::BadPath(e.msg()) } +/// 存固定密钥没成。 +#[derive(Debug)] +pub enum PinnedError { + /// 报上来的超出了上限([`tw_pack::disk::PINNED_SOURCE_MAX`] 这几个) + TooLarge(tw_pack::disk::PinnedTooLarge), + Io(std::io::Error), +} + #[cfg(test)] mod patch_seq_tests { use super::*; diff --git a/crates/tw-control/tests/backups.rs b/crates/tw-control/tests/backups.rs index 3d016a43..837da99e 100644 --- a/crates/tw-control/tests/backups.rs +++ b/crates/tw-control/tests/backups.rs @@ -788,3 +788,283 @@ async fn in_safe_mode_a_synced_version_comes_back_whole() { assert_eq!(cfg.listen.control.key.as_deref(), Some(KEY)); assert!(cfg.providers.iter().any(|p| p.name == "relay")); } + +fn rows_of(v: &serde_json::Value) -> Vec { + v["rows"] + .as_array() + .unwrap() + .iter() + .map(|r| { + format!( + "{} {}/{}{}", + r["change"].as_str().unwrap(), + r["item"]["kind"].as_str().unwrap(), + r["item"]["name"].as_str().unwrap(), + r["kept"] + .as_str() + .map(|k| format!(" ({k})")) + .unwrap_or_default() + ) + }) + .collect() +} + +/// 只恢复几项:预览接收所选的项目,和写入走同一个计划;写入的结果说实际动了哪几项 +#[tokio::test] +async fn a_partial_restore_writes_what_its_preview_listed() { + let with = format!( + "{}proxies:\n - name: px\n type: http\n addr: 10.0.0.1:8080\n", + mine() + ) + .replace( + "routes:", + " - name: gone\n base_url: https://gone.example\n key: sk-g\n proxy: px\nroutes:", + ); + tw_config::try_parse(&with).unwrap(); + let b = bed(&with); + b.state.cfg.record_startup(); + b.state + .cfg + .write(&mine(), None, tw_config::history::Origin::Ui) + .await + .unwrap(); + let (_, list) = call(&b.app, "GET", "/backups", serde_json::json!(null)).await; + let oldest = list.as_array().unwrap().last().unwrap()["id"].clone(); + let items = serde_json::json!([{ "kind": "provider", "name": "gone" }]); + let (st, preview) = call( + &b.app, + "POST", + "/backups/preview", + serde_json::json!({ "id": oldest, "items": items }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{preview}"); + assert_eq!(rows_of(&preview), ["add provider/gone", "add proxy/px"]); + let (st, v) = call( + &b.app, + "POST", + "/backups/restore", + serde_json::json!({ "id": oldest, "base_version": preview["base_version"], "items": items }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{v}"); + assert_eq!(rows_of(&v), rows_of(&preview)); + assert_eq!(v["version"], serde_json::json!(b.version())); + let cfg = tw_config::try_parse(&b.file()).unwrap(); + assert_eq!(cfg.proxies.len(), 1); +} + +/// 恢复同步位置上的版本:本机不同步的上游(和指向它的路由)那一版里本来就没有,不列成删除 +#[tokio::test] +async fn restoring_a_synced_version_leaves_what_this_machine_does_not_sync() { + let d = tempfile::tempdir().unwrap(); + let here = format!( + "{}sync:\n device_id: 3f9a2c1e0b1d4e2f\n device_name: srv\n encrypt: false\n exclude: [ollama]\n target:\n type: folder\n path: {}\n", + mine().replace( + "routes:", + " - name: ollama\n base_url: http://127.0.0.1:11434\nroutes:\n - name: local\n rules:\n - name: all\n to: ollama\n" + ), + d.path().join("sync").display() + ); + tw_config::try_parse(&here).unwrap_or_else(|e| panic!("{e}\n{here}")); + // 同步位置上的那一版:没有 ollama 和 local,官方的地址不一样 + let synced = { + let mut m = tw_pack::Manifest::new(tw_pack::PackKind::Full, tw_pack::PackOrigin::Sync); + m.secrets = true; + tw_pack::Pack { + manifest: m, + config_yaml: mine() + .replace("https://api.anthropic.com\n", "https://synced.example\n") + .replace(&format!("listen:\n control:\n key: {KEY}\n"), ""), + plugins: Default::default(), + client_prefs: None, + } + }; + let b = bed(&here); + b.state + .cfg + .set_remote_backups(Some(Arc::new(OneRemote(synced)))); + let (_, list) = call(&b.app, "GET", "/backups", serde_json::json!(null)).await; + let id = list + .as_array() + .unwrap() + .iter() + .find(|e| e["id"].as_str().unwrap().starts_with("sync-")) + .unwrap()["id"] + .clone(); + let (st, preview) = call( + &b.app, + "POST", + "/backups/preview", + serde_json::json!({ "id": id }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{preview}"); + assert_eq!(rows_of(&preview), ["modify provider/官方"]); + let (st, v) = call( + &b.app, + "POST", + "/backups/restore", + serde_json::json!({ "id": id, "base_version": preview["base_version"] }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{v}"); + let cfg = tw_config::try_parse(&b.file()).unwrap(); + let names: Vec<&str> = cfg.providers.iter().map(|p| p.name.as_str()).collect(); + assert_eq!(names, ["官方", "ollama"]); + assert_eq!(cfg.providers[0].base_url, "https://synced.example"); + assert!(cfg.routes.iter().any(|r| r.name == "local")); +} + +/// 固定密钥:来源不过期,有上限;早先的文件照样算;同时报的几个来源谁也不盖掉谁 +#[tokio::test] +async fn pinned_keys_have_limits_and_an_older_file_still_counts() { + let d = tempfile::tempdir().unwrap(); + let p = d.path().join("config.yaml"); + std::fs::write(&p, mine()).unwrap(); + std::fs::write( + tw_pack::disk::pinned_path(&p), + br#"{"names":["claude-code"]}"#, + ) + .unwrap(); + let gw = tw_gateway::AppState::new(tw_config::try_parse(&mine()).unwrap()).unwrap(); + let bus = gw.bus.clone(); + let mgr = Arc::new(ConfigManager::new(p.clone(), gw.clone(), bus)); + assert!(mgr.pinned_keys().contains("claude-code")); + let app = tw_control::router(state_for(p.clone(), mgr.clone(), gw)); + for (body, code) in [ + ( + serde_json::json!({ "source": "x".repeat(129), "names": ["default"] }), + "backup.pinned.source_too_long", + ), + ( + serde_json::json!({ "source": "a", "names": ["n".repeat(257)] }), + "backup.pinned.name_too_long", + ), + ( + serde_json::json!({ "source": "a", "names": (0..1025).map(|i| i.to_string()).collect::>() }), + "backup.pinned.too_many", + ), + ] { + let (st, v) = call(&app, "PUT", "/pinned-keys", body).await; + assert_eq!(st, StatusCode::BAD_REQUEST, "{v}"); + assert_eq!(v["code"], code); + } + let tasks: Vec<_> = (0..16) + .map(|i| { + let app = app.clone(); + tokio::spawn(async move { + call( + &app, + "PUT", + "/pinned-keys", + serde_json::json!({ "source": format!("lite-{i}"), "names": [format!("k{i}")] }), + ) + .await + .0 + }) + }) + .collect(); + for t in tasks { + assert_eq!(t.await.unwrap(), StatusCode::OK); + } + let on_disk = tw_pack::disk::read_pinned(&p); + for i in 0..16 { + assert!(on_disk.contains(&format!("k{i}")), "{on_disk:?}"); + } + assert!(on_disk.contains("claude-code")); + assert_eq!(on_disk, mgr.pinned_keys()); +} + +/// 已经存着一份超大的通用偏好(上限之前存的):恢复、导入照样能做,只有要写下去的新偏好才查 +#[tokio::test] +async fn a_large_preference_already_here_does_not_block_a_restore() { + let b = bed(&mine()); + let big = serde_json::json!({ "big": "x".repeat(tw_api::CLIENT_PREFS_MAX + 1) }); + std::fs::write( + tw_pack::disk::prefs_path(&b.path()), + serde_json::to_vec(&big).unwrap(), + ) + .unwrap(); + b.state.cfg.record_startup(); + b.state + .cfg + .write( + &mine().replace("https://api.anthropic.com\n", "https://changed.example\n"), + None, + tw_config::history::Origin::Ui, + ) + .await + .unwrap(); + let (_, list) = call(&b.app, "GET", "/backups", serde_json::json!(null)).await; + let oldest = list.as_array().unwrap().last().unwrap()["id"].clone(); + let (_, preview) = call( + &b.app, + "POST", + "/backups/preview", + serde_json::json!({ "id": oldest }), + ) + .await; + let (st, v) = call( + &b.app, + "POST", + "/backups/restore", + serde_json::json!({ "id": oldest, "base_version": preview["base_version"] }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{v}"); + assert_eq!(tw_pack::disk::read_prefs(&b.path()), Some(big)); +} + +/// 启动时从备份里找回插件文件(写插件文件和写配置之间断过电):网关照找回来的重读,插件 +/// 照常跑。读不成 YAML 的原文不记成版本 +#[tokio::test] +async fn a_plugin_put_back_at_startup_runs_again() { + use tw_gateway::plugin::fake::{FakeEngine, source}; + let v1 = source( + serde_json::json!({"name": "A", "api": 1, "permissions": ["system"]}), + &["onRequest"], + ); + let v2 = source( + serde_json::json!({"name": "A2", "api": 1, "permissions": ["system"]}), + &["onRequest"], + ); + let sha = tw_gateway::plugin::load::sha256_hex(v1.as_bytes()); + let text = format!( + "{}plugins:\n - id: a\n file: plugins/a.js\n sha256: {sha}\n", + mine() + ); + let d = tempfile::tempdir().unwrap(); + let p = d.path().join("config.yaml"); + std::fs::write(&p, &text).unwrap(); + let files = |s: &str| tw_pack::PluginFiles { + script: s.as_bytes().to_vec(), + approved: Some(s.as_bytes().to_vec()), + }; + let one = |s: &str| std::collections::BTreeMap::from([("a".to_string(), files(s))]); + tw_pack::disk::write_plugins(&p, &Default::default(), &one(&v1)).unwrap(); + tw_pack::backup::record(&p, &text, tw_config::history::Origin::Ui).unwrap(); + // 断在半路:两份文件都已经是新的,配置还是旧的 + tw_pack::disk::write_plugins(&p, &one(&v1), &one(&v2)).unwrap(); + let gw = tw_gateway::AppState::new(tw_config::try_parse(&text).unwrap()).unwrap(); + gw.set_plugin_engine(Arc::new(FakeEngine)); + let bus = gw.bus.clone(); + let mgr = Arc::new(ConfigManager::new(p.clone(), gw.clone(), bus)); + let app = tw_control::router(state_for(p.clone(), mgr.clone(), gw)); + let status = || async { + let (_, v) = call(&app, "GET", "/plugins", serde_json::json!(null)).await; + v[0]["status"].clone() + }; + assert_eq!(status().await, serde_json::json!({"kind": "changed"})); + mgr.record_startup(); + assert_eq!(status().await, serde_json::json!({"kind": "ok"})); + assert_eq!( + std::fs::read_to_string(tw_config::plugins::file_path(d.path(), "a")).unwrap(), + v1 + ); + + let before = tw_pack::backup::list(&p).unwrap().len(); + std::fs::write(&p, "listen: [\n :::\n").unwrap(); + mgr.record_startup(); + assert_eq!(tw_pack::backup::list(&p).unwrap().len(), before); +} diff --git a/crates/tw-pack/src/backup.rs b/crates/tw-pack/src/backup.rs index d95315ff..a4e292ff 100644 --- a/crates/tw-pack/src/backup.rs +++ b/crates/tw-pack/src/backup.rs @@ -133,7 +133,7 @@ pub fn record( text: &str, origin: Origin, ) -> Result, BackupError> { - if origin == Origin::Rotation { + if origin == Origin::Rotation || !readable(text) { return Ok(None); } let pack = disk::current(config_path, text, PackOrigin::from(origin)); @@ -157,7 +157,7 @@ pub fn record_before( text: &str, origin: Origin, ) -> Result, BackupError> { - if origin == Origin::Rotation { + if origin == Origin::Rotation || !readable(text) { return Ok(None); } let all = list(config_path)?; @@ -172,6 +172,23 @@ pub fn record_before( save(config_path, &pack, Origin::External, all.last()).map(Some) } +/// 读得成 YAML(一个映射)吗。**读不成的原文不记成版本**(安全模式下启动时、改坏了的文件): +/// 恢复不了它,它还会挤掉一个恢复得了的版本,整份恢复取本机那几节时也取不到东西 +fn readable(text: &str) -> bool { + serde_yaml_ng::from_str::(text).is_ok_and(|v| v.is_mapping()) +} + +/// 从新往旧第一个读得成 YAML、和现在这份原文不一样的本机版本:配置坏到读不成 YAML 时,整份 +/// 恢复从这里取本机那几节([`crate::restore::whole`]) +pub fn last_readable(config_path: &Path, current_text: &str) -> Option { + let now = text_fingerprint(&tw_config::control_key::mask(current_text)); + list(config_path).ok()?.iter().rev().find_map(|b| { + open(b) + .ok() + .filter(|p| readable(&p.config_yaml) && text_fingerprint(&p.config_yaml) != now) + }) +} + fn masked(pack: &Pack) -> Pack { Pack { config_yaml: tw_config::control_key::mask(&pack.config_yaml), @@ -252,11 +269,16 @@ pub fn heal_plugins(config_path: &Path) -> Vec { let mut healed = Vec::new(); let mut backups: Option> = None; for (id, sha) in entries { - let files = disk::read_plugin(config_path, &id); - let fine = files.as_ref().is_some_and(|f| { - hash(&f.script) == sha || f.approved.as_ref().is_some_and(|a| hash(a) == sha) - }); - if fine { + // **只在脚本和底稿都对不上批准的哈希时**才找回来(写插件文件和写配置之间断过电)。 + // 底稿对得上、脚本对不上是用户改了脚本还没批准:不动它,网关照常把它停用、等批准。 + // 没有底稿的说不清是哪一种,也不动;脚本文件没了的找回来,盖不掉谁的改动 + let heal = match disk::read_plugin(config_path, &id) { + None => true, + Some(f) => { + hash(&f.script) != sha && f.approved.as_ref().is_some_and(|a| hash(a) != sha) + } + }; + if !heal { continue; } let packs = backups.get_or_insert_with(|| { @@ -481,7 +503,7 @@ mod tests { }; let sha = crate::model::sha256_hex(b"v1"); let text = format!( - "{}plugins:\n - id: a\n file: plugins/a.js\n sha256: {sha}\n - id: b\n file: plugins/b.js\n sha256: {sha}\n", + "{}plugins:\n - id: a\n file: plugins/a.js\n sha256: {sha}\n - id: b\n file: plugins/b.js\n sha256: {sha}\n - id: c\n file: plugins/c.js\n sha256: {sha}\n", cfg(1) ); std::fs::write(&p, &text).unwrap(); @@ -491,19 +513,22 @@ mod tests { &[ ("a".to_string(), files("v1", Some("v1"))), ("b".to_string(), files("v1", Some("v1"))), + ("c".to_string(), files("v1", Some("v1"))), ] .into_iter() .collect(), ) .unwrap(); record(&p, &text, Origin::Ui).unwrap().unwrap(); - // a:两份都换成了新的(断在半路);b:用户改了脚本,底稿还是批准的那份 + // a:两份都换成了新的(断在半路);b:用户改了脚本,底稿还是批准的那份;c:用户改了 + // 脚本,底稿没了(说不清是哪一种,不动) disk::write_plugins( &p, &Default::default(), &[ ("a".to_string(), files("v2", Some("v2"))), ("b".to_string(), files("edited", Some("v1"))), + ("c".to_string(), files("edited", None)), ] .into_iter() .collect(), @@ -512,9 +537,33 @@ mod tests { assert_eq!(heal_plugins(&p), ["a"]); assert_eq!(disk::read_plugin(&p, "a").unwrap().script, b"v1"); assert_eq!(disk::read_plugin(&p, "b").unwrap().script, b"edited"); + assert_eq!(disk::read_plugin(&p, "c").unwrap().script, b"edited"); assert!(heal_plugins(&p).is_empty()); } + /// 读不成 YAML 的原文不记成版本;整份恢复取本机那几节时从新往旧找第一个读得成的 + #[test] + fn unreadable_text_is_no_version_and_the_last_readable_one_is_found() { + let (_d, p) = setup(); + record(&p, &cfg(1), Origin::Ui).unwrap().unwrap(); + record(&p, &cfg(2), Origin::Ui).unwrap().unwrap(); + let garbage = "listen: [\n :::\n"; + assert!(serde_yaml_ng::from_str::(garbage).is_err()); + assert!(record(&p, garbage, Origin::External).unwrap().is_none()); + assert!(record_before(&p, garbage, Origin::Ui).unwrap().is_none()); + assert!( + record(&p, "just a string\n", Origin::External) + .unwrap() + .is_none() + ); + assert_eq!(list(&p).unwrap().len(), 2); + let last = last_readable(&p, garbage).unwrap(); + assert!(last.config_yaml.contains("port: 2"), "{}", last.config_yaml); + // 和现在这份一样的不算(它要是读得成,本机那几节本来就从现在这份取) + let last = last_readable(&p, &cfg(2)).unwrap(); + assert!(last.config_yaml.contains("port: 1"), "{}", last.config_yaml); + } + #[test] fn a_rotation_leaves_no_version_and_does_not_count_as_a_change() { let (_d, p) = setup(); diff --git a/crates/tw-pack/src/disk.rs b/crates/tw-pack/src/disk.rs index 16340e15..c7052385 100644 --- a/crates/tw-pack/src/disk.rs +++ b/crates/tw-pack/src/disk.rs @@ -38,9 +38,23 @@ pub fn read_prefs(config_path: &Path) -> Option { } /// 存下通用偏好。None 是删掉 +/// 这份通用偏好超出了上限吗([`tw_api::CLIENT_PREFS_MAX`],写成紧凑的 JSON 之后) +pub fn prefs_too_large(prefs: &serde_json::Value) -> bool { + prefs.to_string().len() > tw_api::CLIENT_PREFS_MAX +} + +/// 存下通用偏好,None 是删掉。**超出上限的不写**(`InvalidInput`):界面、导入、恢复、同步、 +/// 命令行写偏好都走这里,谁也绕不过去 pub fn write_prefs(config_path: &Path, prefs: Option<&serde_json::Value>) -> std::io::Result<()> { let path = prefs_path(config_path); match prefs { + Some(p) if prefs_too_large(p) => Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!( + "the client preferences are larger than {} KiB", + tw_api::CLIENT_PREFS_MAX / 1024 + ), + )), Some(p) => { let bytes = serde_json::to_vec_pretty(p).map_err(std::io::Error::other)?; write_private(&path, &bytes) @@ -52,63 +66,141 @@ pub fn write_prefs(config_path: &Path, prefs: Option<&serde_json::Value>) -> std } } -/// 一个来源(一个连着这台 core 的 Lite)报上来的固定密钥多久没更新就不算了:30 天 -pub const PINNED_TTL_MS: u64 = 30 * 24 * 3600 * 1000; +/// 最多记几个来源。多出来的去掉最久没报过的那一个:**来源不过期**(Lite 每次连上 core 都 +/// 重报一次,一个来源再报是换掉它自己那一组),这个上限挡的是没完没了的新来源 +pub const PINNED_SOURCES_MAX: usize = 64; +/// 来源标识最长几个字节 +pub const PINNED_SOURCE_MAX: usize = 128; +/// 一个来源最多报几把密钥 +pub const PINNED_NAMES_MAX: usize = 1024; +/// 一把密钥的名字最长几个字节 +pub const PINNED_NAME_MAX: usize = 256; /// 一个来源报上来的固定密钥。 #[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct PinnedSource { pub names: BTreeSet, + /// 最近一次报上来的时候。来源多过 [`PINNED_SOURCES_MAX`] 时去掉最久没报的 + #[serde(default)] pub updated_at_ms: u64, } /// 存着的固定密钥,**按来源**:几个 Lite 连着同一个 core(远程模式)时各报各的,取并集。 -/// 没有、读不懂都是空的 +/// 没有、读不懂都是空的。早先的写法(一个名字数组,或者 `{"names": [...]}`)读作来源空串 pub fn read_pinned_sources(config_path: &Path) -> BTreeMap { #[derive(serde::Deserialize)] - struct File { - sources: BTreeMap, + #[serde(untagged)] + enum File { + Sources { + sources: BTreeMap, + }, + Names { + names: BTreeSet, + }, + Plain(BTreeSet), } - std::fs::read(pinned_path(config_path)) + let Some(file) = std::fs::read(pinned_path(config_path)) .ok() .and_then(|b| serde_json::from_slice::(&b).ok()) - .map(|f| f.sources) - .unwrap_or_default() + else { + return BTreeMap::new(); + }; + match file { + File::Sources { sources } => sources, + File::Names { names } | File::Plain(names) => BTreeMap::from([( + String::new(), + PinnedSource { + names, + updated_at_ms: 0, + }, + )]), + } } -/// 各来源的并集,**过期的来源不算**([`PINNED_TTL_MS`] 没更新:那个 Lite 多半已经不在了) -pub fn pinned_union(sources: &BTreeMap, now_ms: u64) -> BTreeSet { +/// 各来源的并集 +pub fn pinned_union(sources: &BTreeMap) -> BTreeSet { sources .values() - .filter(|s| now_ms.saturating_sub(s.updated_at_ms) <= PINNED_TTL_MS) .flat_map(|s| s.names.iter().cloned()) .collect() } -/// 存着的固定密钥:各个来源的并集,过期的不算 +/// 存着的固定密钥:各个来源的并集 pub fn read_pinned(config_path: &Path) -> BTreeSet { - pinned_union(&read_pinned_sources(config_path), crate::now_ms()) + pinned_union(&read_pinned_sources(config_path)) } -/// 一个来源报上来一组固定密钥:换掉它上次报的,过期的来源顺手删掉。返回存下的全部来源 -pub fn write_pinned( - config_path: &Path, +/// 报上来的一组超出了上限 +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PinnedTooLarge { + Source, + Names, + Name, +} + +/// 一个来源报上来一组固定密钥:换掉它上次报的。来源多过上限时去掉最久没报过的。**不写盘** +pub fn with_pinned( + sources: &BTreeMap, source: &str, names: &BTreeSet, -) -> std::io::Result> { - let now = crate::now_ms(); - let mut sources = read_pinned_sources(config_path); - sources.retain(|_, s| now.saturating_sub(s.updated_at_ms) <= PINNED_TTL_MS); - sources.insert( + now_ms: u64, +) -> Result, PinnedTooLarge> { + if source.len() > PINNED_SOURCE_MAX { + return Err(PinnedTooLarge::Source); + } + if names.len() > PINNED_NAMES_MAX { + return Err(PinnedTooLarge::Names); + } + if names.iter().any(|n| n.len() > PINNED_NAME_MAX) { + return Err(PinnedTooLarge::Name); + } + let mut out = sources.clone(); + out.insert( source.to_string(), PinnedSource { names: names.clone(), - updated_at_ms: now, + updated_at_ms: now_ms, }, ); + while out.len() > PINNED_SOURCES_MAX { + let Some(oldest) = out + .iter() + .filter(|(k, _)| k.as_str() != source) + .min_by_key(|(_, s)| s.updated_at_ms) + .map(|(k, _)| k.clone()) + else { + break; + }; + out.remove(&oldest); + } + Ok(out) +} + +/// 存下全部来源 +pub fn write_pinned_sources( + config_path: &Path, + sources: &BTreeMap, +) -> std::io::Result<()> { let bytes = serde_json::to_vec_pretty(&serde_json::json!({ "sources": sources })) .map_err(std::io::Error::other)?; - write_private(&pinned_path(config_path), &bytes)?; + write_private(&pinned_path(config_path), &bytes) +} + +/// 一个来源报上来一组固定密钥,读文件、换掉、写回去。**只给单独一个进程用**(命令行、测试); +/// 跑着的 core 在锁里改它记着的那一份(`ConfigManager::set_pinned_keys`) +pub fn write_pinned( + config_path: &Path, + source: &str, + names: &BTreeSet, +) -> std::io::Result> { + let sources = with_pinned( + &read_pinned_sources(config_path), + source, + names, + crate::now_ms(), + ) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, format!("{e:?}")))?; + write_pinned_sources(config_path, &sources)?; Ok(sources) } @@ -315,6 +407,11 @@ mod tests { write_prefs(&cfg, None).unwrap(); assert_eq!(read_prefs(&cfg), None); write_prefs(&cfg, None).unwrap(); + // 超出上限的不写 + let big = serde_json::json!({ "big": "x".repeat(tw_api::CLIENT_PREFS_MAX) }); + let e = write_prefs(&cfg, Some(&big)).unwrap_err(); + assert_eq!(e.kind(), std::io::ErrorKind::InvalidInput); + assert_eq!(read_prefs(&cfg), None); std::fs::write(prefs_path(&cfg), b"{not json").unwrap(); assert_eq!(read_prefs(&cfg), None); @@ -326,10 +423,53 @@ mod tests { assert_eq!(read_pinned(&cfg), a.union(&b).cloned().collect()); write_pinned(&cfg, "lite-a", &BTreeSet::new()).unwrap(); assert_eq!(read_pinned(&cfg), b); - // 30 天没更新的来源不算 + // 来源不过期:很久以前报的照样算 let mut sources = read_pinned_sources(&cfg); sources.get_mut("lite-b").unwrap().updated_at_ms = 1; - assert!(pinned_union(&sources, crate::now_ms()).is_empty()); + assert_eq!(pinned_union(&sources), b); + } + + /// 早先的写法读作来源空串,不读成空的 + #[test] + fn an_older_pinned_file_reads_as_the_blank_source() { + let d = tempfile::tempdir().unwrap(); + let cfg = d.path().join("config.yaml"); + let codex: BTreeSet = ["codex".to_string()].into(); + for old in [&br#"{"names":["codex"]}"#[..], br#"["codex"]"#] { + std::fs::write(pinned_path(&cfg), old).unwrap(); + assert_eq!(read_pinned(&cfg), codex); + assert_eq!(read_pinned_sources(&cfg)[""].names, codex); + } + // 再报一次(有来源的):早先那一组还在,并集 + let b: BTreeSet = ["claude-code".to_string()].into(); + write_pinned(&cfg, "lite-a", &b).unwrap(); + assert_eq!(read_pinned(&cfg), codex.union(&b).cloned().collect()); + } + + #[test] + fn pinned_sources_have_limits() { + let none = BTreeMap::new(); + let one: BTreeSet = ["k".to_string()].into(); + let long = "x".repeat(PINNED_SOURCE_MAX + 1); + assert_eq!( + with_pinned(&none, &long, &one, 1), + Err(PinnedTooLarge::Source) + ); + let many: BTreeSet = (0..=PINNED_NAMES_MAX).map(|i| i.to_string()).collect(); + assert_eq!( + with_pinned(&none, "a", &many, 1), + Err(PinnedTooLarge::Names) + ); + let wide: BTreeSet = ["n".repeat(PINNED_NAME_MAX + 1)].into(); + assert_eq!(with_pinned(&none, "a", &wide, 1), Err(PinnedTooLarge::Name)); + // 来源多过上限:去掉最久没报过的那一个 + let mut all = BTreeMap::new(); + for i in 0..PINNED_SOURCES_MAX { + all = with_pinned(&all, &format!("s{i}"), &one, 10 + i as u64).unwrap(); + } + let all = with_pinned(&all, "new", &one, 1000).unwrap(); + assert_eq!(all.len(), PINNED_SOURCES_MAX); + assert!(!all.contains_key("s0") && all.contains_key("s1") && all.contains_key("new")); } #[test] diff --git a/crates/tw-pack/src/export.rs b/crates/tw-pack/src/export.rs index bc0b162b..fffad476 100644 --- a/crates/tw-pack/src/export.rs +++ b/crates/tw-pack/src/export.rs @@ -165,8 +165,15 @@ pub fn strip_secrets(snap: &mut Snapshot) { /// 不带密钥的包里的一项,**空着的密钥换成本机同一项里的**:上游的密钥、请求头的值(按名字)、 /// AWS 访问密钥、OAuth 的客户端密钥,代理的密码,网关密钥的值。比较和覆盖都照这个样子:除了 -/// 密钥都一样的就是「相同」,覆盖不会把本机的密钥抹成空的 +/// 密钥都一样的就是「相同」,覆盖不会把本机的密钥抹成空的。 +/// +/// **只在目的地不变时**([`same_destination`]):包里的上游换了主机、端口、AWS 区域或 OAuth +/// 令牌端点,代理换了地址,本机的密钥就不跟过去 —— 否则一份不带密钥的包改个地址,覆盖之后 +/// 本机的密钥就发给了包里写的那个地方 pub fn fill_secrets(id: &ItemId, item: &mut model::Item, mine: &model::Item) { + if !same_destination(id, item, mine) { + return; + } let empty = |v: Option<&Json>| v.and_then(Json::as_str) == Some(""); let take = |at: Option<&mut Json>, from: Option<&Json>| { if let (Some(v), Some(f)) = (at, from) @@ -237,6 +244,67 @@ pub fn fill_secrets(id: &ItemId, item: &mut model::Item, mine: &model::Item) { } } +/// 两份同名的项目,密钥是发去同一个地方的吗:上游的协议、主机、端口一样,AWS 区域一样, +/// OAuth 的令牌端点一样;代理的类型和地址一样。网关密钥是发给这台网关自己的,总是 +pub fn same_destination(id: &ItemId, theirs: &model::Item, mine: &model::Item) -> bool { + let at = |it: &model::Item, u: &str, path: &[&str]| -> Option { + let mut cur = it.units.get(u)?; + for p in path { + cur = cur.get(*p)?; + } + Some(cur.clone()) + }; + match id.kind { + ItemKind::Provider => { + let url = |it: &model::Item| { + at(it, unit::CONN, &["base_url"]).and_then(|v| v.as_str().and_then(destination)) + }; + url(theirs).is_some() + && url(theirs) == url(mine) + && at(theirs, unit::CONN, &["aws", "region"]) + == at(mine, unit::CONN, &["aws", "region"]) + && at(theirs, unit::CONN, &["oauth", "endpoint"]) + == at(mine, unit::CONN, &["oauth", "endpoint"]) + } + ItemKind::Proxy => { + let addr = |it: &model::Item| { + at(it, unit::ALL, &["addr"]) + .and_then(|v| v.as_str().map(|a| a.trim().to_ascii_lowercase())) + }; + addr(theirs).is_some() + && addr(theirs) == addr(mine) + && at(theirs, unit::ALL, &["type"]) == at(mine, unit::ALL, &["type"]) + } + _ => true, + } +} + +/// 一个地址发去哪儿:协议、主机(小写)、端口(没写时按协议的缺省)。读不懂是 None +pub fn destination(url: &str) -> Option<(String, String, u16)> { + let (scheme, rest) = url.trim().split_once("://")?; + let scheme = scheme.to_ascii_lowercase(); + let authority = rest.split(['/', '?', '#']).next()?; + let hostport = authority.rsplit('@').next()?; + let (host, port) = if let Some(v6) = hostport.strip_prefix('[') { + let (h, after) = v6.split_once(']')?; + (h.to_string(), after.strip_prefix(':')) + } else { + match hostport.rsplit_once(':') { + Some((h, p)) => (h.to_string(), Some(p)), + None => (hostport.to_string(), None), + } + }; + let port = match port { + Some(p) => p.parse().ok()?, + None => match scheme.as_str() { + "https" => 443, + "http" => 80, + _ => return None, + }, + }; + (!host.is_empty()).then(|| (scheme, host.to_ascii_lowercase(), port)) +} + /// 一项里有没有被去掉的密钥(空字符串) pub fn missing_secret(id: &ItemId, item: &model::Item) -> bool { let empty = |v: Option<&Json>| v.and_then(Json::as_str) == Some(""); @@ -260,6 +328,12 @@ pub fn missing_secret(id: &ItemId, item: &model::Item) -> bool { .is_some_and(|o| empty(o.get("refresh")) || empty(o.get("client_secret"))) } ItemKind::Key => empty(item.units.get(unit::ALL).and_then(|a| a.get("key"))), + ItemKind::Proxy => empty( + item.units + .get(unit::ALL) + .and_then(|a| a.get("auth")) + .and_then(|a| a.get("pass")), + ), _ => false, } } @@ -288,6 +362,26 @@ fn civil(days: u64) -> (i64, u32, u32) { mod tests { use super::*; + #[test] + fn a_destination_is_scheme_host_and_port() { + let d = |u| destination(u); + assert_eq!( + d("https://Relay.example/v1"), + Some(("https".into(), "relay.example".into(), 443)) + ); + assert_eq!( + d("https://relay.example:443/x"), + d("https://relay.example/y") + ); + assert_ne!(d("https://relay.example:8443"), d("https://relay.example")); + assert_ne!(d("http://relay.example"), d("https://relay.example")); + assert_eq!( + d("http://user:pw@[::1]:8080/p"), + Some(("http".into(), "::1".into(), 8080)) + ); + assert_eq!(d("not a url"), None); + } + #[test] fn the_file_name_carries_the_date() { assert_eq!(file_name(0), "ThinkWatch-1970-01-01.twconfig"); diff --git a/crates/tw-pack/src/import.rs b/crates/tw-pack/src/import.rs index b11daf90..d61b6ea8 100644 --- a/crates/tw-pack/src/import.rs +++ b/crates/tw-pack/src/import.rs @@ -501,7 +501,9 @@ pub fn apply_with( codec: &dyn PluginCodec, ) -> Result { let l = model::snapshot_with(local, codec)?; - let mut i = with_local_secrets(&l, model::snapshot_with(incoming, codec)?, incoming); + // **不在这里沿用本机的密钥**:改名导入的是另一个项目,不该带走本机那一项的密钥;覆盖时 + // 逐项沿用(目的地不变才沿用,见 `export::fill_secrets`) + let mut i = model::snapshot_with(incoming, codec)?; let rows = preview_with(local, incoming, ctx, codec)?; let errors: BTreeMap<&ItemId, &Msg> = rows .iter() @@ -548,12 +550,21 @@ pub fn apply_with( { return Err(ImportError::Item(d.item.clone(), Box::new((*e).clone()))); } - let Some(item) = i.items.get(&id) else { + let Some(raw) = i.items.get(&id) else { if d.action == Action::Skip { continue; } return Err(ImportError::NotInPack(d.item.clone())); }; + // 不带密钥的包:覆盖本机同名的那一项时沿用本机的密钥(目的地不变才沿用) + let mut item = raw.clone(); + if !incoming.manifest.secrets + && !matches!(d.action, Action::Rename(_)) + && let Some(mine) = l.items.get(&id) + { + export::fill_secrets(&id, &mut item, mine); + } + let item = &item; match &d.action { Action::Skip => continue, Action::Import if l.items.contains_key(&id) && !is_global(id.kind) => { diff --git a/crates/tw-pack/src/lib.rs b/crates/tw-pack/src/lib.rs index f32736b5..746c8be0 100644 --- a/crates/tw-pack/src/lib.rs +++ b/crates/tw-pack/src/lib.rs @@ -308,6 +308,10 @@ pub enum PackError { /// 要求加密的地方读到了一个没加密的包([`read_strict`]) #[error("{}", self.msg())] NotEncrypted, + /// 口令加密的工作量超出了这一版肯解的上限([`MAX_WORK_FACTOR`])。不是包坏了:别的工具 + /// 用更高的工作量加密的 + #[error("{}", self.msg())] + TooMuchWork { required: u8, max: u8 }, } impl PackError { @@ -358,6 +362,11 @@ impl PackError { "pack.too_large", limit = limit / (1024 * 1024) => "This configuration pack is larger than {limit} MiB." ), + PackError::TooMuchWork { required, max } => msg!( + "pack.too_much_work", required = required, max = max => + "The passphrase encryption of this configuration pack takes more work than this \ + version allows (work factor {required}, at most {max}), so it was not opened." + ), } } } @@ -524,6 +533,10 @@ fn decrypt(bytes: &[u8], unseal: Unseal) -> Result, PackError> { age::DecryptError::DecryptionFailed | age::DecryptError::NoMatchingKeys => { PackError::WrongPassphrase } + age::DecryptError::ExcessiveWork { required, .. } => PackError::TooMuchWork { + required, + max: MAX_WORK_FACTOR, + }, other => corrupt(other), })? } else { diff --git a/crates/tw-pack/src/restore.rs b/crates/tw-pack/src/restore.rs index 289756b6..281002f6 100644 --- a/crates/tw-pack/src/restore.rs +++ b/crates/tw-pack/src/restore.rs @@ -40,6 +40,20 @@ pub struct Row { pub kept: Option, } +/// 恢复的条件。 +#[derive(Debug, Clone, Default)] +pub struct Options { + /// 固定密钥的名字:接管的客户端在用,不删 + pub pinned: BTreeSet, + /// 通用偏好算不算。经远程连接恢复时不算:偏好在 Lite 那边 + pub with_prefs: bool, + /// 只恢复这几项(连同它们要的,见 [`plan`]),None 是全部 + pub items: Option>, + /// 不参与比较、保持现在样子的项目:恢复同步位置上的版本时,本机不同步的上游和随之不同步 + /// 的项目(规格 7.2)—— 那一版里本来就没有它们,不是删了 + pub skip: BTreeSet, +} + /// 那一版和现在的差异。`pinned` 是固定密钥的名字。`with_prefs` 为 false 时通用偏好不算 /// (经远程连接恢复:偏好在 Lite 那边) pub fn preview( @@ -48,19 +62,130 @@ pub fn preview( pinned: &BTreeSet, with_prefs: bool, ) -> Result, ModelError> { - preview_with(local, backup, pinned, with_prefs, model::codec()) + preview_opts( + local, + backup, + &Options { + pinned: pinned.clone(), + with_prefs, + ..Default::default() + }, + ) +} + +/// 照这些条件恢复会动哪几项。**和 [`apply_opts`] 是同一个计划**:只选了几项时,预览列出的 +/// 就是写下去的(连同一起恢复的、留着的) +pub fn preview_opts(local: &Pack, backup: &Pack, opts: &Options) -> Result, ModelError> { + preview_with(local, backup, opts, model::codec()) } pub fn preview_with( local: &Pack, backup: &Pack, - pinned: &BTreeSet, - with_prefs: bool, + opts: &Options, codec: &dyn PluginCodec, ) -> Result, ModelError> { let l = model::snapshot_with(local, codec)?; let b = model::snapshot_with(backup, codec)?; - Ok(plan(&l, &b, pinned, with_prefs, None).0) + Ok(plan(&l, &b, &local_refs(local), opts).0) +} + +/// 本机那几节([`model::LOCAL_SECTIONS`])引用的项目:同步设置走的出站代理。恢复留着本机那 +/// 几节,它们引用的就得留着 —— 那一版里没有这个代理的话,删了它同步设置就不成立 +fn local_refs(local: &Pack) -> BTreeSet { + local_refs_in(&local.config_yaml) +} + +fn local_refs_in(text: &str) -> BTreeSet { + fn walk(v: &serde_yaml_ng::Value, out: &mut BTreeSet) { + match v { + serde_yaml_ng::Value::Mapping(m) => { + for (k, v) in m { + if k.as_str() == Some("proxy") + && let Some(name) = v.as_str() + && name != tw_config::DIRECT + && name != tw_config::SYSTEM + { + out.insert(ItemId::new(ItemKind::Proxy, name)); + } + walk(v, out); + } + } + serde_yaml_ng::Value::Sequence(xs) => xs.iter().for_each(|x| walk(x, out)), + _ => {} + } + } + let mut out = BTreeSet::new(); + if let Ok(doc) = serde_yaml_ng::from_str::(text) { + for section in model::LOCAL_SECTIONS { + if let Some(v) = doc.get(*section) { + walk(v, &mut out); + } + } + } + out +} + +/// 那一版,`skip` 里的项目换成现在的样子:现在有的照现在的,现在没有的不在;排列顺序里它们 +/// 照现在的位置 +fn overlay(l: &Snapshot, b: &Snapshot, skip: &BTreeSet) -> Snapshot { + if skip.is_empty() { + return b.clone(); + } + let mut out = b.clone(); + for id in skip { + match l.items.get(id) { + Some(it) => { + out.items.insert(id.clone(), it.clone()); + } + None => { + out.items.remove(id); + } + } + } + for kind in [ItemKind::Provider, ItemKind::Route, ItemKind::Plugin] { + let order = ItemId::new(kind.order().expect("has an order"), ""); + let skipped = |n: &String| skip.contains(&ItemId::new(kind, n.clone())); + let here = names_of(l.items.get(&order)); + let mut list: Vec = names_of(out.items.get(&order)) + .into_iter() + .filter(|n| !skipped(n)) + .collect(); + // 跳过的那几个插回现在的位置:紧跟在现在排在它前面、那一版里也有的那一个后面 + for (i, n) in here.iter().enumerate().filter(|(_, n)| skipped(n)) { + let at = here[..i] + .iter() + .rev() + .find_map(|p| list.iter().position(|x| x == p)) + .map_or(0, |p| p + 1); + list.insert(at, n.clone()); + } + if list.is_empty() && !out.items.contains_key(&order) { + continue; + } + out.items.insert( + order, + model::Item { + units: std::collections::BTreeMap::from([( + unit::ALL.to_string(), + Json::Array(list.into_iter().map(Json::String).collect()), + )]), + ..Default::default() + }, + ); + } + out.relink(); + out +} + +fn names_of(it: Option<&model::Item>) -> Vec { + match it.and_then(|it| it.units.get(unit::ALL)) { + Some(Json::Array(a)) => a + .iter() + .filter_map(|n| n.as_str().map(str::to_string)) + .collect(), + _ => Vec::new(), + } } /// 两份之间的差异,按项目 @@ -95,24 +220,26 @@ fn hard(from: &ItemId, to: &ItemId) -> bool { /// - 是这次要删的:留着([`Kept::Referenced`]),比如留着的固定密钥绑的那条路由; /// - 是那一版里有、没选上的:一起恢复,比如只选了一个已删的上游、它用的出站代理也删了。 /// -/// 一直补到没有缺的为止 +/// 一直补到没有缺的为止。路由的去处写的是一个名字:上游里找不到时按同名的策略组找(组删了 +/// 之后它读成一个不存在的上游)。`local` 是本机那几节引用的项目([`local_refs`]),和别的 +/// 引用一样总得留着 fn plan( l: &Snapshot, b: &Snapshot, - pinned: &BTreeSet, - with_prefs: bool, - selected: Option<&BTreeSet>, + local: &BTreeSet, + opts: &Options, ) -> (Vec, Snapshot) { - let changes = diff(l, b, with_prefs); + let b = &overlay(l, b, &opts.skip); + let changes = diff(l, b, opts.with_prefs); let mut chosen: BTreeSet = changes .iter() .map(|(id, _)| id.clone()) - .filter(|id| selected.is_none_or(|s| s.contains(id))) + .filter(|id| opts.items.as_ref().is_none_or(|s| s.contains(id))) .collect(); let mut kept: std::collections::BTreeMap = changes .iter() .filter(|(id, c)| { - *c == Change::Remove && id.kind == ItemKind::Key && pinned.contains(&id.name) + *c == Change::Remove && id.kind == ItemKind::Key && opts.pinned.contains(&id.name) }) .map(|(id, _)| (id.clone(), Kept::KeyInUse)) .collect(); @@ -143,8 +270,28 @@ fn plan( .cloned() .collect::>() }) + .chain( + local + .iter() + .filter(|r| !target.items.contains_key(*r)) + .cloned(), + ) .collect(); for r in missing { + // 指向一个名字的:上游里没有就是同名的策略组 + let group = ItemId::new(ItemKind::Group, r.name.clone()); + let r = if r.kind == ItemKind::Provider + && !l.items.contains_key(&r) + && !b.items.contains_key(&r) + && (l.items.contains_key(&group) || b.items.contains_key(&group)) + { + group + } else { + r + }; + if target.items.contains_key(&r) { + continue; + } if l.items.contains_key(&r) { // 这次要删的:留着 if let std::collections::btree_map::Entry::Vacant(e) = kept.entry(r) { @@ -179,6 +326,8 @@ pub struct Applied { pub changed_keys: Vec, /// 留着没删的:固定密钥,和留下来的项目还引用着的 pub kept: Vec, + /// 实际动了的那几项(连同一起恢复的、留着的),和同样条件下的预览一样 + pub rows: Vec, } /// 恢复。`items` 是只恢复这几项,None 是全部。所选项目引用的、本机已经没有的项目一起恢复 @@ -189,21 +338,31 @@ pub fn apply( pinned: &BTreeSet, with_prefs: bool, ) -> Result { - apply_with(local, backup, items, pinned, with_prefs, model::codec()) + apply_opts( + local, + backup, + &Options { + pinned: pinned.clone(), + with_prefs, + items: items.map(|xs| xs.iter().cloned().collect()), + ..Default::default() + }, + ) +} + +pub fn apply_opts(local: &Pack, backup: &Pack, opts: &Options) -> Result { + apply_with(local, backup, opts, model::codec()) } pub fn apply_with( local: &Pack, backup: &Pack, - items: Option<&[ItemId]>, - pinned: &BTreeSet, - with_prefs: bool, + opts: &Options, codec: &dyn PluginCodec, ) -> Result { let l = model::snapshot_with(local, codec)?; let b = model::snapshot_with(backup, codec)?; - let picked: Option> = items.map(|xs| xs.iter().cloned().collect()); - let (rows, mut target) = plan(&l, &b, pinned, with_prefs, picked.as_ref()); + let (rows, mut target) = plan(&l, &b, &local_refs(local), opts); let kept = rows .iter() .filter(|r| r.kept.is_some()) @@ -234,6 +393,7 @@ pub fn apply_with( changed_keys: import::changed_keys(local, &pack), pack, kept, + rows, }) } @@ -333,6 +493,38 @@ pub fn whole(backup: &Pack, current_text: &str, last_local: Option<&Pack>) -> Pa text = t; } } + // 本机那几节引用的出站代理(同步设置走的)那一版里没有:从取本机那几节的地方一起取来 + if let Some(src) = &source { + let named = |d: &Yaml, name: &str| -> Option { + d.get("proxies")? + .as_sequence()? + .iter() + .find(|p| p.get("name").and_then(Yaml::as_str) == Some(name)) + .cloned() + }; + for r in local_refs_in(&text) { + let Some(now) = doc(&text) else { break }; + if named(&now, &r.name).is_some() { + continue; + } + let Some(px) = named(src, &r.name) else { + continue; + }; + let mut list = now + .get("proxies") + .and_then(Yaml::as_sequence) + .cloned() + .unwrap_or_default(); + list.push(px); + if let Ok(t) = tw_config::edit::set( + &text, + &[tw_yaml::Step::key("proxies")], + Some(&Yaml::Sequence(list)), + ) { + text = t; + } + } + } if let Some(k) = real { let at: Vec = tw_api::control::KEY_PATH .iter() diff --git a/crates/tw-pack/src/restore/tests.rs b/crates/tw-pack/src/restore/tests.rs index 8e838879..aaa60de5 100644 --- a/crates/tw-pack/src/restore/tests.rs +++ b/crates/tw-pack/src/restore/tests.rs @@ -32,13 +32,22 @@ fn then() -> Pack { )) } +fn opts(items: Option<&[ItemId]>) -> Options { + Options { + pinned: pinned(), + with_prefs: true, + items: items.map(|xs| xs.iter().cloned().collect()), + ..Default::default() + } +} + fn pinned() -> BTreeSet { ["claude-code".to_string()].into_iter().collect() } #[test] fn the_preview_lists_what_differs_by_item() { - let rows = preview_with(&now(), &then(), &pinned(), true, &WholeFile).unwrap(); + let rows = preview_with(&now(), &then(), &opts(None), &WholeFile).unwrap(); let got: Vec<(String, Change, Option)> = rows .iter() .map(|r| (r.item.to_string(), r.change, r.kept)) @@ -62,7 +71,7 @@ fn the_preview_lists_what_differs_by_item() { #[test] fn restoring_everything_keeps_local_settings_logins_and_keys_in_use() { - let out = apply_with(&now(), &then(), None, &pinned(), true, &WholeFile).unwrap(); + let out = apply_with(&now(), &then(), &opts(None), &WholeFile).unwrap(); let cfg = tw_config::try_parse(&out.pack.config_yaml) .unwrap_or_else(|e| panic!("{e}\n{}", out.pack.config_yaml)); assert_eq!(cfg.listen.gateway.port, 9000, "本机设置留着现在的"); @@ -87,7 +96,7 @@ fn restoring_everything_keeps_local_settings_logins_and_keys_in_use() { #[test] fn restoring_a_few_items_leaves_the_rest_as_it_is() { let pick = [ItemId::new(ItemKind::Provider, "deleted")]; - let out = apply_with(&now(), &then(), Some(&pick), &pinned(), true, &WholeFile).unwrap(); + let out = apply_with(&now(), &then(), &opts(Some(&pick)), &WholeFile).unwrap(); let cfg = tw_config::try_parse(&out.pack.config_yaml).unwrap(); let names: Vec<&str> = cfg.providers.iter().map(|p| p.name.as_str()).collect(); assert_eq!(names, ["kept", "chatgpt", "deleted"]); @@ -122,7 +131,16 @@ fn oauth_tokens_come_from_now_or_from_the_version_it_comes_back_from() { oauth("gone", "rt-gone"), " - name: blank\n base_url: https://blank.example\n disabled: true\n oauth:\n refresh: ''\n endpoint: https://auth.example/token\n", )); - let out = apply_with(&now, &then, None, &BTreeSet::new(), true, &WholeFile).unwrap(); + let out = apply_with( + &now, + &then, + &Options { + with_prefs: true, + ..Default::default() + }, + &WholeFile, + ) + .unwrap(); let cfg = tw_config::try_parse(&out.pack.config_yaml) .unwrap_or_else(|e| panic!("{e}\n{}", out.pack.config_yaml)); let p = |n: &str| cfg.providers.iter().find(|p| p.name == n).unwrap(); diff --git a/crates/tw-pack/src/tests.rs b/crates/tw-pack/src/tests.rs index 1a7a78ed..d08c05d5 100644 --- a/crates/tw-pack/src/tests.rs +++ b/crates/tw-pack/src/tests.rs @@ -292,6 +292,14 @@ fn every_error_speaks_with_its_own_code() { "pack.missing_entry", ), (PackError::TooLarge { limit: MAX_BYTES }, "pack.too_large"), + (PackError::NotEncrypted, "pack.not_encrypted"), + ( + PackError::TooMuchWork { + required: 22, + max: 18, + }, + "pack.too_much_work", + ), ]; for (e, code) in all { assert_eq!(e.msg().code, code); @@ -363,3 +371,28 @@ fn a_strict_read_refuses_a_plain_pack_when_a_key_is_given() { // `read` 照收:导入的是用户自己挑的文件 assert!(read(&plain, Unseal::Identity(&id)).is_ok()); } + +/// 口令加密的工作量超出上限:说的是这件事,不是「包坏了」。**解之前就拒**,不会先占掉内存 +#[test] +fn a_pack_that_takes_too_much_work_says_so() { + fast(); + let bytes = write(&sample(), Seal::Passphrase("pw")).unwrap(); + // 头里的 `-> scrypt 10` 改成 22:头的校验在解出文件密钥之后,工作量在那之前就查 + let head = bytes.iter().position(|b| *b == b'\n').unwrap() + 1; + let line_end = head + bytes[head..].iter().position(|b| *b == b'\n').unwrap(); + let line = std::str::from_utf8(&bytes[head..line_end]).unwrap(); + assert!( + line.starts_with("-> scrypt ") && line.ends_with(" 10"), + "{line}" + ); + let mut patched = bytes[..line_end - 2].to_vec(); + patched.extend_from_slice(b"22"); + patched.extend_from_slice(&bytes[line_end..]); + assert_eq!( + read(&patched, Unseal::Passphrase("pw")).unwrap_err(), + PackError::TooMuchWork { + required: 22, + max: MAX_WORK_FACTOR + } + ); +} diff --git a/crates/tw-pack/tests/regressions.rs b/crates/tw-pack/tests/regressions.rs index 2fdf0340..44544fe3 100644 --- a/crates/tw-pack/tests/regressions.rs +++ b/crates/tw-pack/tests/regressions.rs @@ -258,3 +258,258 @@ fn an_oauth_upstream_whose_endpoint_changed_comes_back() { let cfg = tw_config::try_parse(&out.pack.config_yaml).unwrap(); assert!(cfg.providers[0].disabled); } + +/// 不带密钥的包把上游指去了别的地方(换主机、换端口、换协议),或者改名导入:本机的密钥 +/// **不跟过去** —— 留空、标「缺少密钥」、停用。代理换了地址也一样。只换路径照样沿用(见上一条) +#[test] +fn a_pack_that_moves_an_upstream_does_not_take_the_key_along() { + let l = local(); + let relay = ItemId::new(ItemKind::Provider, "relay"); + let px = ItemId::new(ItemKind::Proxy, "px"); + let with = |base_url: &str, addr: &str| { + let mut p = export(&l, false, PackOrigin::Export); + p.config_yaml = format!( + "version: 1\nproxies:\n - name: px\n type: http\n addr: {addr}\n auth: {{ user: u, pass: '' }}\nproviders:\n - name: relay\n base_url: {base_url}\n key: ''\n proxy: px\n headers:\n X-Token: ''\n" + ); + p + }; + for base_url in [ + "https://evil.example", + "https://relay.example:8443", + "http://relay.example", + ] { + let incoming = with(base_url, "10.0.0.1:8080"); + let rows = tw_pack::import::preview(&l, &incoming, &ctx()).unwrap(); + let row = rows.iter().find(|r| r.item == relay).unwrap(); + assert_eq!(row.status, Status::Conflict, "{base_url}"); + assert_eq!(row.flags, [Flag::MissingKey], "{base_url}"); + let out = + tw_pack::import::apply(&l, &incoming, &[decide(&relay, Action::Overwrite)], &ctx()) + .unwrap(); + let cfg = tw_config::try_parse(&out.pack.config_yaml).unwrap(); + let p = cfg.providers.iter().find(|p| p.name == "relay").unwrap(); + assert_eq!(p.base_url, base_url); + assert_eq!(p.key.as_ref().map(|k| k.raw()), Some(""), "{base_url}"); + assert_eq!( + p.headers.get("X-Token").unwrap().value.raw(), + "", + "{base_url}" + ); + assert!(p.disabled, "{base_url}"); + } + + // 改名导入是另一个项目:地址没变也不带走本机那一家的密钥 + let incoming = with("https://relay.example", "10.0.0.1:8080"); + let out = tw_pack::import::apply( + &l, + &incoming, + &[decide(&relay, Action::Rename("relay (2)".into()))], + &ctx(), + ) + .unwrap(); + let cfg = tw_config::try_parse(&out.pack.config_yaml).unwrap(); + let p = |n: &str| cfg.providers.iter().find(|p| p.name == n).unwrap(); + assert_eq!(p("relay (2)").key.as_ref().map(|k| k.raw()), Some("")); + assert!(p("relay (2)").disabled); + assert_eq!(p("relay").key.as_ref().map(|k| k.raw()), Some("sk-old")); + + // 代理换了地址:密码不跟过去 + let incoming = with("https://relay.example", "203.0.113.9:3128"); + let rows = tw_pack::import::preview(&l, &incoming, &ctx()).unwrap(); + let row = rows.iter().find(|r| r.item == px).unwrap(); + assert_eq!(row.status, Status::Conflict); + assert_eq!(row.flags, [Flag::MissingKey]); + let out = + tw_pack::import::apply(&l, &incoming, &[decide(&px, Action::Overwrite)], &ctx()).unwrap(); + let cfg = tw_config::try_parse(&out.pack.config_yaml).unwrap(); + assert_eq!(cfg.proxies[0].auth.as_ref().unwrap().pass.raw(), ""); +} + +/// AWS 区域、OAuth 令牌端点换了也是换了目的地 +#[test] +fn a_moved_aws_region_or_oauth_endpoint_does_not_take_the_secret_along() { + let l = pack(&format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n key: tw-local\nproviders:\n - name: br\n base_url: https://bedrock.corp.example\n protocol: bedrock\n aws:\n access_key_id: AKIAOLD\n secret_access_key: old-secret\n region: us-east-1\n - name: oa\n base_url: https://api.example\n oauth:\n refresh: r-old\n endpoint: https://auth.example/token\n client_id: cid\n client_secret: cs-old\n" + )); + let incoming = { + let mut p = export(&l, false, PackOrigin::Export); + p.config_yaml = "version: 1\nproviders:\n - name: br\n base_url: https://bedrock.corp.example\n protocol: bedrock\n aws:\n access_key_id: ''\n secret_access_key: ''\n region: eu-west-1\n - name: oa\n base_url: https://api.example\n oauth:\n refresh: ''\n endpoint: https://other.example/token\n client_id: cid\n client_secret: ''\n".into(); + p + }; + let br = ItemId::new(ItemKind::Provider, "br"); + let oa = ItemId::new(ItemKind::Provider, "oa"); + let out = tw_pack::import::apply( + &l, + &incoming, + &[ + decide(&br, Action::Overwrite), + decide(&oa, Action::Overwrite), + ], + &ctx(), + ) + .unwrap(); + let cfg = tw_config::try_parse(&out.pack.config_yaml) + .unwrap_or_else(|e| panic!("{e}\n{}", out.pack.config_yaml)); + let p = |n: &str| cfg.providers.iter().find(|p| p.name == n).unwrap(); + let aws = p("br").aws.as_ref().unwrap(); + assert_eq!(aws.secret_access_key.as_ref().map(|k| k.raw()), Some("")); + assert!(p("br").disabled); + let oauth = p("oa").oauth.as_ref().unwrap(); + assert_eq!(oauth.client_secret.as_deref(), Some("")); + assert!(p("oa").disabled); +} + +/// 留着的固定密钥绑的路由指向一个策略组,那一版里路由和组都没有:路由和组一起留着,恢复照样 +/// 成。只选这条已删路由时,组一起恢复 +#[test] +fn a_kept_route_that_points_at_a_group_keeps_the_group() { + let now = pack(&format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n key: tw-a\n - name: codex\n key: tw-cx\n route: fast\nproviders:\n - name: p1\n base_url: https://p1.example\n key: sk-1\ngroups:\n - name: g\n type: fallback\n providers: [p1]\nroutes:\n - name: fast\n rules:\n - name: all\n to: g\n" + )); + let then = pack(&format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n key: tw-a\nproviders:\n - name: p1\n base_url: https://p1.example\n key: sk-1\n" + )); + let pinned: BTreeSet = ["codex".to_string()].into(); + let rows = tw_pack::restore::preview(&now, &then, &pinned, true).unwrap(); + let kept = |kind, name: &str| { + rows.iter() + .find(|r| r.item == ItemId::new(kind, name)) + .and_then(|r| r.kept) + }; + assert_eq!(kept(ItemKind::Key, "codex"), Some(Kept::KeyInUse)); + assert_eq!(kept(ItemKind::Route, "fast"), Some(Kept::Referenced)); + assert_eq!(kept(ItemKind::Group, "g"), Some(Kept::Referenced)); + let out = tw_pack::restore::apply(&now, &then, None, &pinned, true).unwrap(); + let cfg = tw_config::try_parse(&out.pack.config_yaml).unwrap(); + assert_eq!(cfg.groups.len(), 1); + + // 反过来:现在没有路由和组,只选那一版里那条指向组的路由 + let now2 = then.clone(); + let then2 = now.clone(); + let pick = [ItemId::new(ItemKind::Route, "fast")]; + let out = tw_pack::restore::apply(&now2, &then2, Some(&pick), &BTreeSet::new(), true).unwrap(); + let cfg = tw_config::try_parse(&out.pack.config_yaml) + .unwrap_or_else(|e| panic!("{e}\n{}", out.pack.config_yaml)); + assert_eq!( + cfg.groups + .iter() + .map(|g| g.name.as_str()) + .collect::>(), + ["g"] + ); + assert!( + out.rows + .iter() + .any(|r| r.item == ItemId::new(ItemKind::Group, "g") && r.change == Change::Add) + ); +} + +const SYNC: &str = "sync:\n device_id: 3f9a2c1e-0b1d-4e2f-9a3b-5c6d7e8f9a0b\n device_name: srv\n encrypt: false\n target:\n type: webdav\n url: https://dav.example.com/tw\n user: u\n pass: p\n proxy: corp\n"; + +/// 同步设置走的出站代理那一版里没有:恢复留着它(本机那几节引用的),整份恢复从现在这份 +/// 原文里一起取来 +#[test] +fn the_proxy_the_sync_settings_use_stays() { + let now = pack(&format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n key: tw-a\nproxies:\n - name: corp\n type: socks5h\n addr: 127.0.0.1:7890\n{SYNC}" + )); + tw_config::try_parse(&now.config_yaml).unwrap(); + let then = pack(&format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n key: tw-a\n" + )); + let rows = tw_pack::restore::preview(&now, &then, &BTreeSet::new(), true).unwrap(); + let corp = ItemId::new(ItemKind::Proxy, "corp"); + assert_eq!( + rows.iter().find(|r| r.item == corp).and_then(|r| r.kept), + Some(Kept::Referenced) + ); + let out = tw_pack::restore::apply(&now, &then, None, &BTreeSet::new(), true).unwrap(); + let cfg = tw_config::try_parse(&out.pack.config_yaml).unwrap(); + assert_eq!(cfg.proxies.len(), 1); + assert_eq!(out.kept, [corp]); + + let synced = export(&then, true, PackOrigin::Sync); + let w = tw_pack::restore::whole( + &synced, + &format!("{}not_a_field: 1\n", now.config_yaml), + None, + ); + let cfg = + tw_config::try_parse(&w.config_yaml).unwrap_or_else(|e| panic!("{e}\n{}", w.config_yaml)); + assert_eq!(cfg.proxies[0].addr, "127.0.0.1:7890"); + assert_eq!(cfg.sync.unwrap().target.proxy(), "corp"); +} + +/// 恢复同步位置上的版本:本机不同步的上游(和随之不同步的路由)那一版里本来就没有,不列成 +/// 删除,排列顺序里也照现在的位置 +#[test] +fn restoring_a_synced_version_leaves_what_this_machine_does_not_sync() { + let now = pack(&format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n key: tw-a\nproviders:\n - name: a\n base_url: https://a.example\n key: sk-a\n - name: ollama\n base_url: http://127.0.0.1:11434\n - name: b\n base_url: https://b.example\n key: sk-b\nroutes:\n - name: local\n rules:\n - name: all\n to: ollama\n" + )); + tw_config::try_parse(&now.config_yaml).unwrap(); + // 那一版:同步位置上的,没有 ollama 和指向它的路由;b 在 a 前面,a 的密钥不一样 + let then = pack(&format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n key: tw-a\nproviders:\n - name: b\n base_url: https://b.example\n key: sk-b\n - name: a\n base_url: https://a.example\n key: sk-a2\n" + )); + let l = tw_pack::model::snapshot(&now).unwrap(); + let b = tw_pack::model::snapshot(&then).unwrap(); + let skip = { + // 和同步那一路的算法一样:不同步的上游,连同直接或间接引用它们的 + let mut out: BTreeSet = [ItemId::new(ItemKind::Provider, "ollama")].into(); + for s in [&l, &b] { + for (id, it) in &s.items { + if !id.kind.is_order() && it.refs.iter().any(|r| out.contains(r)) { + out.insert(id.clone()); + } + } + } + out + }; + assert!(skip.contains(&ItemId::new(ItemKind::Route, "local"))); + let opts = tw_pack::restore::Options { + with_prefs: true, + skip, + ..Default::default() + }; + let rows = tw_pack::restore::preview_opts(&now, &then, &opts).unwrap(); + assert!( + rows.iter().all(|r| r.change != Change::Remove), + "{:?}", + rows.iter() + .map(|r| (r.item.to_string(), r.change)) + .collect::>() + ); + let out = tw_pack::restore::apply_opts(&now, &then, &opts).unwrap(); + assert_eq!(out.rows, rows); + let cfg = tw_config::try_parse(&out.pack.config_yaml).unwrap(); + let names: Vec<&str> = cfg.providers.iter().map(|p| p.name.as_str()).collect(); + assert_eq!(names, ["b", "a", "ollama"]); + assert_eq!(cfg.routes.len(), 1); + let a = cfg.providers.iter().find(|p| p.name == "a").unwrap(); + assert_eq!(a.key.as_ref().map(|k| k.raw()), Some("sk-a2")); +} + +/// 只选几项时,预览和写入走同一个计划:预览列出的就是写下去的 +#[test] +fn a_partial_preview_is_what_the_restore_writes() { + let now = pack(&format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n key: tw-a\nproviders:\n - name: p1\n base_url: https://p1.example\n key: sk-1\n" + )); + let then = pack(&format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n key: tw-a\nproxies:\n - name: px\n type: http\n addr: 10.0.0.1:8080\nproviders:\n - name: p1\n base_url: https://p1.example\n key: sk-1\n - name: gone\n base_url: https://gone.example\n key: sk-g\n proxy: px\n" + )); + let opts = tw_pack::restore::Options { + with_prefs: true, + items: Some([ItemId::new(ItemKind::Provider, "gone")].into()), + ..Default::default() + }; + let rows = tw_pack::restore::preview_opts(&now, &then, &opts).unwrap(); + let ids: BTreeSet = rows.iter().map(|r| r.item.to_string()).collect(); + assert!( + ids.contains(&ItemId::new(ItemKind::Proxy, "px").to_string()), + "{ids:?}" + ); + let out = tw_pack::restore::apply_opts(&now, &then, &opts).unwrap(); + assert_eq!(out.rows, rows); +} diff --git a/docs/config.md b/docs/config.md index 32abe538..071833a5 100644 --- a/docs/config.md +++ b/docs/config.md @@ -142,7 +142,12 @@ clients set up on this machine use. An import adds what is not here and asks, item by item, what to do with what differs (`--on-conflict` answers for all). A pack carries no settings of this machine and no ChatGPT sign-in; without `--no-secrets` it holds keys in plain text unless `--encrypt` is given (the -passphrase is asked for, or read from `TWCORE_PASSPHRASE`). +passphrase is asked for, or read from `TWCORE_PASSPHRASE`). Overwriting an +item here with one from a pack without keys keeps this machine's keys only +while the item still points at the same place (scheme, host and port; AWS +region; OAuth token endpoint; proxy address); otherwise, and for an item +imported under another name, the upstream comes in disabled until its key +is entered. These commands work on the files directly, so they work when core is not running, which is when a restore is most needed; when the file does not diff --git a/docs/config.zh-CN.md b/docs/config.zh-CN.md index b9b7ee4b..aa16e5ac 100644 --- a/docs/config.zh-CN.md +++ b/docs/config.zh-CN.md @@ -81,7 +81,7 @@ twcore import all.twconfig --dry-run # 导入会做什么 twcore config set /listen/gateway/port 8790 --int ``` -恢复时保留本机设置(监听、控制面密钥、保留期限、同步设置)、ChatGPT 的登录,以及本机已接管的客户端正在使用的网关密钥。导入时加入本机没有的项目,与本机不同的项目逐项选择处理方式(`--on-conflict` 一次指定全部)。配置包不含本机设置与 ChatGPT 登录;未指定 `--no-secrets` 时其中以明文保存密钥,除非指定 `--encrypt`(口令在终端输入,或取自环境变量 `TWCORE_PASSPHRASE`)。 +恢复时保留本机设置(监听、控制面密钥、保留期限、同步设置)、ChatGPT 的登录,以及本机已接管的客户端正在使用的网关密钥。导入时加入本机没有的项目,与本机不同的项目逐项选择处理方式(`--on-conflict` 一次指定全部)。配置包不含本机设置与 ChatGPT 登录;未指定 `--no-secrets` 时其中以明文保存密钥,除非指定 `--encrypt`(口令在终端输入,或取自环境变量 `TWCORE_PASSPHRASE`)。以不含密钥的配置包覆盖本机项目时,只有目的地不变(协议、主机与端口,AWS 区域,OAuth 令牌端点,代理地址)才沿用本机的密钥;目的地变了的、改名导入的上游以停用状态导入,填入密钥后再启用。 这些命令直接操作文件,因此 core 没有运行时也能使用,而那往往正是最需要恢复的时候;配置完全无法读取时,恢复把整个版本写回,控制面密钥保持现在的值。正在运行的 core 会像对待其他保存一样接收这些改动。