From 8bae8879e3a478d9c5eb056377f1fa62d41c132c Mon Sep 17 00:00:00 2001 From: kev1n77 Date: Wed, 22 Jul 2026 03:09:08 +0800 Subject: [PATCH] fix: retry transient proxy restoration --- crates/proxy-recovery/src/lib.rs | 57 +++++++- .../tests/macos_privileged_helper.rs | 135 +++++++++--------- crates/proxy-recovery/tests/recovery.rs | 67 ++++++++- docs/progress.html | 4 +- 4 files changed, 189 insertions(+), 74 deletions(-) diff --git a/crates/proxy-recovery/src/lib.rs b/crates/proxy-recovery/src/lib.rs index 225d605..36bc630 100644 --- a/crates/proxy-recovery/src/lib.rs +++ b/crates/proxy-recovery/src/lib.rs @@ -190,6 +190,14 @@ pub enum RecoveryError { PlatformOutputInvalid(String), AuthenticatedProxyUnsupported(String), PrivilegedHelper(String), + OperationFailed { + operation: &'static str, + source: Box, + }, + RestoreRetryFailed { + first: Box, + retry: Box, + }, RollbackRecoveryFailed { apply: Box, rollback: Box, @@ -243,6 +251,18 @@ impl fmt::Display for RecoveryError { Self::PrivilegedHelper(detail) => { write!(formatter, "privileged proxy helper failed: {detail}") } + Self::OperationFailed { operation, source } => { + write!( + formatter, + "proxy recovery failed while {operation}: {source}" + ) + } + Self::RestoreRetryFailed { first, retry } => { + write!( + formatter, + "proxy recovery failed, then its idempotent retry also failed ({first}; retry: {retry})" + ) + } Self::RollbackRecoveryFailed { apply, rollback, @@ -262,6 +282,8 @@ impl std::error::Error for RecoveryError { match self { Self::Io(error) => Some(error), Self::Json(error) => Some(error), + Self::OperationFailed { source, .. } => Some(source.as_ref()), + Self::RestoreRetryFailed { retry, .. } => Some(retry.as_ref()), Self::RollbackRecoveryFailed { apply, .. } => Some(apply.as_ref()), _ => None, } @@ -274,6 +296,15 @@ impl From for RecoveryError { } } +impl RecoveryError { + fn while_doing(self, operation: &'static str) -> Self { + Self::OperationFailed { + operation, + source: Box::new(self), + } + } +} + pub struct ProxySession { backend: B, journal_path: PathBuf, @@ -339,19 +370,35 @@ impl ProxySession { } pub fn restore(mut self) -> Result<(), RecoveryError> { + let first = match self.restore_inner() { + Ok(()) => return Ok(()), + Err(error) => error, + }; self.restore_inner() + .map_err(|retry| RecoveryError::RestoreRetryFailed { + first: Box::new(first), + retry: Box::new(retry), + }) } fn restore_inner(&mut self) -> Result<(), RecoveryError> { if self.restored { return Ok(()); } - self.backend.restore(&self.original)?; - mark_restored(&self.journal_path)?; - if let Some(mut watchdog) = self.watchdog.take() { - watchdog.disarm()?; + self.backend + .restore(&self.original) + .map_err(|error| error.while_doing("restoring the original proxy snapshot"))?; + mark_restored(&self.journal_path) + .map_err(|error| error.while_doing("marking the recovery journal restored"))?; + if let Some(watchdog) = self.watchdog.as_mut() { + watchdog + .disarm() + .map_err(|error| error.while_doing("disarming the proxy watchdog"))?; } - remove_if_exists(&self.journal_path)?; + self.watchdog = None; + remove_if_exists(&self.journal_path) + .map_err(RecoveryError::Io) + .map_err(|error| error.while_doing("removing the restored recovery journal"))?; self.restored = true; Ok(()) } diff --git a/crates/proxy-recovery/tests/macos_privileged_helper.rs b/crates/proxy-recovery/tests/macos_privileged_helper.rs index 4d08380..d299c09 100644 --- a/crates/proxy-recovery/tests/macos_privileged_helper.rs +++ b/crates/proxy-recovery/tests/macos_privileged_helper.rs @@ -13,77 +13,80 @@ use codeischeap_proxy_recovery::{ }; #[test] -fn pid_bound_helper_restores_pac_state_on_command_and_disconnect() { - for disconnect in [false, true] { - let root = test_directory(if disconnect { "disconnect" } else { "restore" }); - let state = root.join("proxy-state.json"); - let journal = root.join(MACOS_PROXY_RECOVERY_JOURNAL_FILENAME); - let status = root.join(format!(".codeischeap-proxy-helper-{disconnect}.status")); - let socket = env::temp_dir().join(format!( - "codeischeap-proxy-helper-{}-{disconnect}.sock", - std::process::id() - )); - let _ = fs::remove_file(&socket); - let backend = FileProxyBackend::new(&state); - let original = ProxySettings::AutoConfig { - url: "https://config.example.test/proxy.pac".to_owned(), - }; - let desired = ProxySettings::Manual { - http_proxy: "http://127.0.0.1:43125".to_owned(), - https_proxy: "http://127.0.0.1:43125".to_owned(), - bypass: vec!["localhost".to_owned()], - }; - backend.apply(&original).expect("seed PAC state"); - let owner_pid = std::process::id(); - let owner_uid = fs::metadata(&root).expect("root metadata").uid(); - let server_backend = backend.clone(); - let server_journal = journal.clone(); - let server_status = status.clone(); - let server_socket = socket.clone(); - let server = thread::spawn(move || { - run_macos_proxy_helper_session( - server_backend, - desired, - server_journal, - server_status, - server_socket, - owner_pid, - owner_uid, - env!("CARGO_BIN_EXE_proxy-watchdog"), - ) - }); +fn pid_bound_helper_restores_pac_state_on_command() { + run_helper_scenario(false); +} + +#[test] +fn pid_bound_helper_restores_pac_state_on_disconnect() { + run_helper_scenario(true); +} - let session = MacOsPrivilegedProxySession::connect( - &status, - &socket, +fn run_helper_scenario(disconnect: bool) { + let root = test_directory(if disconnect { "disconnect" } else { "restore" }); + let state = root.join("proxy-state.json"); + let journal = root.join(MACOS_PROXY_RECOVERY_JOURNAL_FILENAME); + let status = root.join(format!(".codeischeap-proxy-helper-{disconnect}.status")); + let socket = env::temp_dir().join(format!( + "codeischeap-proxy-helper-{}-{disconnect}.sock", + std::process::id() + )); + let _ = fs::remove_file(&socket); + let backend = FileProxyBackend::new(&state); + let original = ProxySettings::AutoConfig { + url: "https://config.example.test/proxy.pac".to_owned(), + }; + let desired = ProxySettings::Manual { + http_proxy: "http://127.0.0.1:43125".to_owned(), + https_proxy: "http://127.0.0.1:43125".to_owned(), + bypass: vec!["localhost".to_owned()], + }; + backend.apply(&original).expect("seed PAC state"); + let owner_pid = std::process::id(); + let owner_uid = fs::metadata(&root).expect("root metadata").uid(); + let server_backend = backend.clone(); + let server_journal = journal.clone(); + let server_status = status.clone(); + let server_socket = socket.clone(); + let server = thread::spawn(move || { + run_macos_proxy_helper_session( + server_backend, + desired, + server_journal, + server_status, + server_socket, + owner_pid, owner_uid, - Duration::from_secs(15), + env!("CARGO_BIN_EXE_proxy-watchdog"), ) - .expect("owner must attach"); - assert_ne!( - backend.snapshot().expect("proxy snapshot"), - ProxySnapshot::File { - settings: original.clone(), - } - ); - if disconnect { - drop(session); - } else { - session.restore().expect("helper restore command"); + }); + + let session = + MacOsPrivilegedProxySession::connect(&status, &socket, owner_uid, Duration::from_secs(15)) + .expect("owner must attach"); + assert_ne!( + backend.snapshot().expect("proxy snapshot"), + ProxySnapshot::File { + settings: original.clone(), } - server - .join() - .expect("helper thread must join") - .expect("helper session must finish"); - assert_eq!( - backend.snapshot().expect("restored proxy snapshot"), - ProxySnapshot::File { settings: original } - ); - assert!(!journal.exists()); - assert!(!status.exists()); - assert!(!socket.exists()); - fs::remove_dir_all(root).expect("test directory cleanup"); + ); + if disconnect { + drop(session); + } else { + session.restore().expect("helper restore command"); } + server + .join() + .expect("helper thread must join") + .expect("helper session must finish"); + assert_eq!( + backend.snapshot().expect("restored proxy snapshot"), + ProxySnapshot::File { settings: original } + ); + assert!(!journal.exists()); + assert!(!status.exists()); + assert!(!socket.exists()); + fs::remove_dir_all(root).expect("test directory cleanup"); } fn test_directory(label: &str) -> PathBuf { diff --git a/crates/proxy-recovery/tests/recovery.rs b/crates/proxy-recovery/tests/recovery.rs index 1076de7..6aeaadb 100644 --- a/crates/proxy-recovery/tests/recovery.rs +++ b/crates/proxy-recovery/tests/recovery.rs @@ -2,7 +2,8 @@ use std::env; use std::fs; use std::path::PathBuf; use std::process::{Child, Command, Stdio}; -use std::sync::Mutex; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; use std::thread; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -32,6 +33,35 @@ struct ApplyAndRollbackFailureBackend { inner: FileProxyBackend, } +#[derive(Debug, Clone)] +struct FailOnceRestoreBackend { + inner: FileProxyBackend, + restore_attempts: Arc, +} + +impl ProxyBackend for FailOnceRestoreBackend { + fn descriptor(&self) -> codeischeap_proxy_recovery::BackendDescriptor { + self.inner.descriptor() + } + + fn snapshot(&self) -> Result { + self.inner.snapshot() + } + + fn apply(&self, settings: &ProxySettings) -> Result<(), RecoveryError> { + self.inner.apply(settings) + } + + fn restore(&self, snapshot: &ProxySnapshot) -> Result<(), RecoveryError> { + if self.restore_attempts.fetch_add(1, Ordering::SeqCst) == 0 { + return Err(RecoveryError::PlatformCommandFailed( + "fault-injected transient restore".to_owned(), + )); + } + self.inner.restore(snapshot) + } +} + impl ProxyBackend for ApplyAndRollbackFailureBackend { fn descriptor(&self) -> codeischeap_proxy_recovery::BackendDescriptor { self.inner.descriptor() @@ -105,6 +135,41 @@ fn normal_restore_returns_to_the_exact_snapshot() { fs::remove_dir_all(root).expect("test directory must clean up"); } +#[test] +fn transient_restore_failure_is_retried_before_disarming_the_watchdog() { + let _lock = PROCESS_TEST_LOCK + .lock() + .expect("process test lock must work"); + let root = test_directory("restore-retry"); + let state = root.join("proxy-state.json"); + let journal = root.join("recovery.json"); + let inner = FileProxyBackend::new(&state); + inner + .apply(&original_settings()) + .expect("original state must write"); + let restore_attempts = Arc::new(AtomicUsize::new(0)); + let backend = FailOnceRestoreBackend { + inner: inner.clone(), + restore_attempts: restore_attempts.clone(), + }; + + let session = ProxySession::begin( + backend, + desired_settings(), + &journal, + env!("CARGO_BIN_EXE_proxy-watchdog"), + ) + .expect("session must begin"); + session + .restore() + .expect("the idempotent retry must restore the session"); + + assert_eq!(restore_attempts.load(Ordering::SeqCst), 2); + assert_eq!(file_settings(&inner), original_settings()); + assert!(!journal.exists()); + fs::remove_dir_all(root).expect("test directory must clean up"); +} + #[test] fn watchdog_recovers_pac_snapshot_when_apply_and_inline_rollback_fail() { let _lock = PROCESS_TEST_LOCK diff --git a/docs/progress.html b/docs/progress.html index cb5c904..43b4c45 100644 --- a/docs/progress.html +++ b/docs/progress.html @@ -83,7 +83,7 @@

5. 当前迭代：S5 / S6

CAP-003打包 mitmproxy sidecar 与最小 addonIn progress96%Codex / Release Owner TBD2026-08-25三平台 bundle、manifest、SBOM、原子安装、Tauri 资源、运行时校验、最小环境、独立 readiness token 与 Windows Job Object/Unix 进程组清理完成；正式 Authenticode/Developer ID 制品待发布凭据 CAP-004显式代理与 TLS flowDone100%Codex / TBD2026-07-15桌面 sidecar、认证 IPC、模式与暂停控制、进程树清理、非目标 TLS 直通、gzip/brotli、SSE/NDJSON/JSON-seq 脱敏完成；真实 TLS+ALPN HTTP/2 客户端/服务端证明代理请求响应与直连基线一致 CAP-005证书生命周期In progress85%Codex / TBD2026-08-25跨平台 CA 元数据、私钥权限与系统信任可核验；Windows CurrentUser ROOT 与 macOS User domain 均支持严格校验后的幂等安装/卸载，卸载前自动退出 Proxy；Admin/System 只读，真实交互验收待完成 - CAP-006系统代理快照与 watchdogIn progress92%Codex / TBD2026-08-25Windows 桌面 Proxy 模式已接入 CurrentUser WinINet 事务、私有 journal、同一签名可执行文件 watchdog、活 owner 防误恢复和启动修复；sidecar 独立退出会按运行时代际安全回退 Gateway 并恢复代理；macOS 以静态逐参数 quote 的授权命令启动同一受校验可执行文件，持久 UID/PID 绑定控制连接覆盖正常退出与 owner 断连，root watchdog 覆盖 helper 崩溃，armed journal 在下次启动经单次授权恢复；真实管理员交互与强杀矩阵待验收 + CAP-006系统代理快照与 watchdogIn progress92%Codex / TBD2026-08-25Windows 桌面 Proxy 模式已接入 CurrentUser WinINet 事务、私有 journal、同一签名可执行文件 watchdog、活 owner 防误恢复和启动修复；sidecar 独立退出会按运行时代际安全回退 Gateway 并恢复代理；macOS 以静态逐参数 quote 的授权命令启动同一受校验可执行文件，持久 UID/PID 绑定控制连接覆盖正常退出与 owner 断连，root watchdog 覆盖 helper 崩溃；恢复事务保留 watchdog 直至解除成功，并对瞬态失败执行一次幂等重试；真实管理员交互与强杀矩阵待验收 CAP-007进程/应用归因与兼容诊断Done100%Codex / TBD2026-07-18应用标签/User-Agent/捕获模式归因及兼容诊断树完成；Windows/macOS/Linux Gateway 与 Proxy 均通过操作系统 TCP 连接精确匹配 PID，失败不推断，sidecar PID 不受信且临时端点不进入 Envelope、持久化或导出 DAT-001SQLCipher、迁移、WAL 与 OS 凭据库Done100%Codex / TBD2026-07-21schema v3 可从 v0/v1/v2 升级，新增加密应用设置；新版本仍可只读打开安装前的 v2 恢复快照，响应类型、状态码和耗时可查询；备份恢复、DB/WAL canary 与 OS 凭据库均通过 DAT-002写入、查询、保留与磁盘压力Done100%Codex / TBD2026-07-15批量写入原子提交；默认保留 30 天/50,000 条并以 500 条事务清理；低于 256 MiB 或 SQLITE_FULL 时暂停记录但保持 Gateway 转发 @@ -100,7 +100,7 @@

5. 当前迭代：S5 / S6

PAR-007token、成本和语义指纹Done100%Codex / TBD2026-07-17四厂商 reported usage 归一化、显式 estimated 估算、版本化价格匹配、未知价格留空与 BLAKE3-256 语义指纹完成 TST-001协议 fixture 与 golden testsDone100%Codex / TBD2026-07-17版本化能力矩阵覆盖 OpenAI、Anthropic、Gemini 与 Ollama 的请求、响应、流式、工具、多模态、错误和 Raw fallback；声明均由 fixture 与 golden 验证 TST-002Gateway/Proxy 集成测试In progress82%Codex / TBD2026-08-25认证 IPC、暂停丢弃、真实模式切换与进程树清理、压缩、流式脱敏、非目标 TLS、HTTP/2 基线一致性及 Windows CA 精确增删通过；真实 sidecar 在客户端取消后保持存活，IPC 停止消费且 72 个事件超过容量 64 时，36 个目标响应仍在 15 秒内完成；真实 ROOT 往返和其余协议一致性待扩展 - TST-003平台故障注入In progress80%Codex / TBD2026-08-25Windows/macOS 强杀恢复、活 owner 防误恢复、sidecar 独立退出回退和端口冲突拒绝已通过；PAC 原状态下应用与同步回滚双重失败会交由独立 watchdog 恢复；macOS CA 拒绝恢复及 helper 协议、权限、命令转义和恢复文件校验已覆盖；授权桌面 helper 的真实故障矩阵待完成 + TST-003平台故障注入In progress82%Codex / TBD2026-08-25Windows/macOS 强杀恢复、活 owner 防误恢复、sidecar 独立退出回退和端口冲突拒绝已通过；PAC 原状态下应用与同步回滚双重失败会交由独立 watchdog 恢复；恢复阶段错误带操作上下文，瞬态失败重试有故障注入覆盖；macOS CA 拒绝恢复及 helper 协议、权限、命令转义和恢复文件校验已覆盖；授权桌面 helper 的真实故障矩阵待完成 TST-004UI、性能与可访问性In progress80%Codex / TBD2026-09-08axe、WAI-ARIA tabs、方向键、焦点循环/恢复、键盘分隔条和双主题对比度已门禁；Ubuntu Chromium 固定 960×620 明色与 1440×900 暗色像素基线，千条列表要求首屏 <2 秒、首尾滚动 <2 秒、筛选 <750 ms 且可见 DOM <40 行；Gateway fake provider 基准要求额外延迟 P95 <20 ms，最新本地基线为 0.609 ms；屏幕阅读器人工验收待完成 REL-001签名安装、升级和回滚In progress80%Codex / Release Owner TBD2026-09-22应用内签名更新、预更新恢复快照与失败后只读历史已实现；发布流水线会生成并验证 Windows/macOS 安装包、updater archive、`.sig`、`latest.json` 与哈希清单，先上传草稿再公开。正式证书/密钥托管、首批签名制品与隔离设备真实回滚演练待完成 REL-002诊断与支持包Done100%Codex / Support Owner TBD2026-07-21版本化 JSON 支持包可预览、复制和保存；接收端严格拒绝超限、重复键、未知格式、异常隐私声明、请求内容字段与凭据模式，只生成无内容诊断摘要；分流、升级、保留和删除流程已发布