fix: retry transient proxy restoration

crates/proxy-recovery/src/lib.rs

@@ -190,6 +190,14 @@ pub enum RecoveryError {
     PlatformOutputInvalid(String),
     AuthenticatedProxyUnsupported(String),
     PrivilegedHelper(String),
+    OperationFailed {
+        operation: &'static str,
+        source: Box<RecoveryError>,
+    },
+    RestoreRetryFailed {
+        first: Box<RecoveryError>,
+        retry: Box<RecoveryError>,
+    },
     RollbackRecoveryFailed {
         apply: Box<RecoveryError>,
         rollback: Box<RecoveryError>,
@@ -243,6 +251,18 @@ impl fmt::Display for RecoveryError {
             Self::PrivilegedHelper(detail) => {
                 write!(formatter, "privileged proxy helper failed: {detail}")
             }
+            Self::OperationFailed { operation, source } => {
+                write!(
+                    formatter,
+                    "proxy recovery failed while {operation}: {source}"
+                )
+            }
+            Self::RestoreRetryFailed { first, retry } => {
+                write!(
+                    formatter,
+                    "proxy recovery failed, then its idempotent retry also failed ({first}; retry: {retry})"
+                )
+            }
             Self::RollbackRecoveryFailed {
                 apply,
                 rollback,
@@ -262,6 +282,8 @@ impl std::error::Error for RecoveryError {
         match self {
             Self::Io(error) => Some(error),
             Self::Json(error) => Some(error),
+            Self::OperationFailed { source, .. } => Some(source.as_ref()),
+            Self::RestoreRetryFailed { retry, .. } => Some(retry.as_ref()),
             Self::RollbackRecoveryFailed { apply, .. } => Some(apply.as_ref()),
             _ => None,
         }
@@ -274,6 +296,15 @@ impl From<io::Error> for RecoveryError {
     }
 }
 
+impl RecoveryError {
+    fn while_doing(self, operation: &'static str) -> Self {
+        Self::OperationFailed {
+            operation,
+            source: Box::new(self),
+        }
+    }
+}
+
 pub struct ProxySession<B: ProxyBackend> {
     backend: B,
     journal_path: PathBuf,
@@ -339,19 +370,35 @@ impl<B: ProxyBackend> ProxySession<B> {
     }
 
     pub fn restore(mut self) -> Result<(), RecoveryError> {
+        let first = match self.restore_inner() {
+            Ok(()) => return Ok(()),
+            Err(error) => error,
+        };
         self.restore_inner()
+            .map_err(|retry| RecoveryError::RestoreRetryFailed {
+                first: Box::new(first),
+                retry: Box::new(retry),
+            })
     }
 
     fn restore_inner(&mut self) -> Result<(), RecoveryError> {
         if self.restored {
             return Ok(());
         }
-        self.backend.restore(&self.original)?;
-        mark_restored(&self.journal_path)?;
-        if let Some(mut watchdog) = self.watchdog.take() {
-            watchdog.disarm()?;
+        self.backend
+            .restore(&self.original)
+            .map_err(|error| error.while_doing("restoring the original proxy snapshot"))?;
+        mark_restored(&self.journal_path)
+            .map_err(|error| error.while_doing("marking the recovery journal restored"))?;
+        if let Some(watchdog) = self.watchdog.as_mut() {
+            watchdog
+                .disarm()
+                .map_err(|error| error.while_doing("disarming the proxy watchdog"))?;
         }
-        remove_if_exists(&self.journal_path)?;
+        self.watchdog = None;
+        remove_if_exists(&self.journal_path)
+            .map_err(RecoveryError::Io)
+            .map_err(|error| error.while_doing("removing the restored recovery journal"))?;
         self.restored = true;
         Ok(())
     }

crates/proxy-recovery/tests/macos_privileged_helper.rs

@@ -13,77 +13,80 @@ use codeischeap_proxy_recovery::{
 };
 
 #[test]
-fn pid_bound_helper_restores_pac_state_on_command_and_disconnect() {
-    for disconnect in [false, true] {
-        let root = test_directory(if disconnect { "disconnect" } else { "restore" });
-        let state = root.join("proxy-state.json");
-        let journal = root.join(MACOS_PROXY_RECOVERY_JOURNAL_FILENAME);
-        let status = root.join(format!(".codeischeap-proxy-helper-{disconnect}.status"));
-        let socket = env::temp_dir().join(format!(
-            "codeischeap-proxy-helper-{}-{disconnect}.sock",
-            std::process::id()
-        ));
-        let _ = fs::remove_file(&socket);
-        let backend = FileProxyBackend::new(&state);
-        let original = ProxySettings::AutoConfig {
-            url: "https://config.example.test/proxy.pac".to_owned(),
-        };
-        let desired = ProxySettings::Manual {
-            http_proxy: "http://127.0.0.1:43125".to_owned(),
-            https_proxy: "http://127.0.0.1:43125".to_owned(),
-            bypass: vec!["localhost".to_owned()],
-        };
-        backend.apply(&original).expect("seed PAC state");
-        let owner_pid = std::process::id();
-        let owner_uid = fs::metadata(&root).expect("root metadata").uid();
-        let server_backend = backend.clone();
-        let server_journal = journal.clone();
-        let server_status = status.clone();
-        let server_socket = socket.clone();
-        let server = thread::spawn(move || {
-            run_macos_proxy_helper_session(
-                server_backend,
-                desired,
-                server_journal,
-                server_status,
-                server_socket,
-                owner_pid,
-                owner_uid,
-                env!("CARGO_BIN_EXE_proxy-watchdog"),
-            )
-        });
+fn pid_bound_helper_restores_pac_state_on_command() {
+    run_helper_scenario(false);
+}
+
+#[test]
+fn pid_bound_helper_restores_pac_state_on_disconnect() {
+    run_helper_scenario(true);
+}
 
-        let session = MacOsPrivilegedProxySession::connect(
-            &status,
-            &socket,
+fn run_helper_scenario(disconnect: bool) {
+    let root = test_directory(if disconnect { "disconnect" } else { "restore" });
+    let state = root.join("proxy-state.json");
+    let journal = root.join(MACOS_PROXY_RECOVERY_JOURNAL_FILENAME);
+    let status = root.join(format!(".codeischeap-proxy-helper-{disconnect}.status"));
+    let socket = env::temp_dir().join(format!(
+        "codeischeap-proxy-helper-{}-{disconnect}.sock",
+        std::process::id()
+    ));
+    let _ = fs::remove_file(&socket);
+    let backend = FileProxyBackend::new(&state);
+    let original = ProxySettings::AutoConfig {
+        url: "https://config.example.test/proxy.pac".to_owned(),
+    };
+    let desired = ProxySettings::Manual {
+        http_proxy: "http://127.0.0.1:43125".to_owned(),
+        https_proxy: "http://127.0.0.1:43125".to_owned(),
+        bypass: vec!["localhost".to_owned()],
+    };
+    backend.apply(&original).expect("seed PAC state");
+    let owner_pid = std::process::id();
+    let owner_uid = fs::metadata(&root).expect("root metadata").uid();
+    let server_backend = backend.clone();
+    let server_journal = journal.clone();
+    let server_status = status.clone();
+    let server_socket = socket.clone();
+    let server = thread::spawn(move || {
+        run_macos_proxy_helper_session(
+            server_backend,
+            desired,
+            server_journal,
+            server_status,
+            server_socket,
+            owner_pid,
             owner_uid,
-            Duration::from_secs(15),
+            env!("CARGO_BIN_EXE_proxy-watchdog"),
         )
-        .expect("owner must attach");
-        assert_ne!(
-            backend.snapshot().expect("proxy snapshot"),
-            ProxySnapshot::File {
-                settings: original.clone(),
-            }
-        );
-        if disconnect {
-            drop(session);
-        } else {
-            session.restore().expect("helper restore command");
+    });
+
+    let session =
+        MacOsPrivilegedProxySession::connect(&status, &socket, owner_uid, Duration::from_secs(15))
+            .expect("owner must attach");
+    assert_ne!(
+        backend.snapshot().expect("proxy snapshot"),
+        ProxySnapshot::File {
+            settings: original.clone(),
         }
-        server
-            .join()
-            .expect("helper thread must join")
-            .expect("helper session must finish");
-        assert_eq!(
-            backend.snapshot().expect("restored proxy snapshot"),
-            ProxySnapshot::File { settings: original }
-        );
-        assert!(!journal.exists());
-        assert!(!status.exists());
-        assert!(!socket.exists());
-        fs::remove_dir_all(root).expect("test directory cleanup");
+    );
+    if disconnect {
+        drop(session);
+    } else {
+        session.restore().expect("helper restore command");
     }
+    server
+        .join()
+        .expect("helper thread must join")
+        .expect("helper session must finish");
+    assert_eq!(
+        backend.snapshot().expect("restored proxy snapshot"),
+        ProxySnapshot::File { settings: original }
+    );
+    assert!(!journal.exists());
+    assert!(!status.exists());
+    assert!(!socket.exists());
+    fs::remove_dir_all(root).expect("test directory cleanup");
 }
 
 fn test_directory(label: &str) -> PathBuf {

crates/proxy-recovery/tests/recovery.rs

@@ -2,7 +2,8 @@ use std::env;
 use std::fs;
 use std::path::PathBuf;
 use std::process::{Child, Command, Stdio};
-use std::sync::Mutex;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::{Arc, Mutex};
 use std::thread;
 use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
 
@@ -32,6 +33,35 @@ struct ApplyAndRollbackFailureBackend {
     inner: FileProxyBackend,
 }
 
+#[derive(Debug, Clone)]
+struct FailOnceRestoreBackend {
+    inner: FileProxyBackend,
+    restore_attempts: Arc<AtomicUsize>,
+}
+
+impl ProxyBackend for FailOnceRestoreBackend {
+    fn descriptor(&self) -> codeischeap_proxy_recovery::BackendDescriptor {
+        self.inner.descriptor()
+    }
+
+    fn snapshot(&self) -> Result<ProxySnapshot, RecoveryError> {
+        self.inner.snapshot()
+    }
+
+    fn apply(&self, settings: &ProxySettings) -> Result<(), RecoveryError> {
+        self.inner.apply(settings)
+    }
+
+    fn restore(&self, snapshot: &ProxySnapshot) -> Result<(), RecoveryError> {
+        if self.restore_attempts.fetch_add(1, Ordering::SeqCst) == 0 {
+            return Err(RecoveryError::PlatformCommandFailed(
+                "fault-injected transient restore".to_owned(),
+            ));
+        }
+        self.inner.restore(snapshot)
+    }
+}
+
 impl ProxyBackend for ApplyAndRollbackFailureBackend {
     fn descriptor(&self) -> codeischeap_proxy_recovery::BackendDescriptor {
         self.inner.descriptor()
@@ -105,6 +135,41 @@ fn normal_restore_returns_to_the_exact_snapshot() {
     fs::remove_dir_all(root).expect("test directory must clean up");
 }
 
+#[test]
+fn transient_restore_failure_is_retried_before_disarming_the_watchdog() {
+    let _lock = PROCESS_TEST_LOCK
+        .lock()
+        .expect("process test lock must work");
+    let root = test_directory("restore-retry");
+    let state = root.join("proxy-state.json");
+    let journal = root.join("recovery.json");
+    let inner = FileProxyBackend::new(&state);
+    inner
+        .apply(&original_settings())
+        .expect("original state must write");
+    let restore_attempts = Arc::new(AtomicUsize::new(0));
+    let backend = FailOnceRestoreBackend {
+        inner: inner.clone(),
+        restore_attempts: restore_attempts.clone(),
+    };
+
+    let session = ProxySession::begin(
+        backend,
+        desired_settings(),
+        &journal,
+        env!("CARGO_BIN_EXE_proxy-watchdog"),
+    )
+    .expect("session must begin");
+    session
+        .restore()
+        .expect("the idempotent retry must restore the session");
+
+    assert_eq!(restore_attempts.load(Ordering::SeqCst), 2);
+    assert_eq!(file_settings(&inner), original_settings());
+    assert!(!journal.exists());
+    fs::remove_dir_all(root).expect("test directory must clean up");
+}
+
 #[test]
 fn watchdog_recovers_pac_snapshot_when_apply_and_inline_rollback_fail() {
     let _lock = PROCESS_TEST_LOCK

docs/progress.html

@@ -83,7 +83,7 @@ <h2 id="current-sprint">5. 当前迭代：S5 / S6</h2>
         <tr><td class="task-id">CAP-003</td><td>打包 mitmproxy sidecar 与最小 addon</td><td><span class="status active">In progress</span></td><td>96%</td><td>Codex / Release Owner TBD</td><td>2026-08-25</td><td>三平台 bundle、manifest、SBOM、原子安装、Tauri 资源、运行时校验、最小环境、独立 readiness token 与 Windows Job Object/Unix 进程组清理完成；正式 Authenticode/Developer ID 制品待发布凭据</td></tr>
         <tr><td class="task-id">CAP-004</td><td>显式代理与 TLS flow</td><td><span class="status done">Done</span></td><td>100%</td><td>Codex / TBD</td><td>2026-07-15</td><td>桌面 sidecar、认证 IPC、模式与暂停控制、进程树清理、非目标 TLS 直通、gzip/brotli、SSE/NDJSON/JSON-seq 脱敏完成；真实 TLS+ALPN HTTP/2 客户端/服务端证明代理请求响应与直连基线一致</td></tr>
         <tr><td class="task-id">CAP-005</td><td>证书生命周期</td><td><span class="status active">In progress</span></td><td>85%</td><td>Codex / TBD</td><td>2026-08-25</td><td>跨平台 CA 元数据、私钥权限与系统信任可核验；Windows CurrentUser ROOT 与 macOS User domain 均支持严格校验后的幂等安装/卸载，卸载前自动退出 Proxy；Admin/System 只读，真实交互验收待完成</td></tr>
-        <tr><td class="task-id">CAP-006</td><td>系统代理快照与 watchdog</td><td><span class="status active">In progress</span></td><td>92%</td><td>Codex / TBD</td><td>2026-08-25</td><td>Windows 桌面 Proxy 模式已接入 CurrentUser WinINet 事务、私有 journal、同一签名可执行文件 watchdog、活 owner 防误恢复和启动修复；sidecar 独立退出会按运行时代际安全回退 Gateway 并恢复代理；macOS 以静态逐参数 quote 的授权命令启动同一受校验可执行文件，持久 UID/PID 绑定控制连接覆盖正常退出与 owner 断连，root watchdog 覆盖 helper 崩溃，armed journal 在下次启动经单次授权恢复；真实管理员交互与强杀矩阵待验收</td></tr>
+        <tr><td class="task-id">CAP-006</td><td>系统代理快照与 watchdog</td><td><span class="status active">In progress</span></td><td>92%</td><td>Codex / TBD</td><td>2026-08-25</td><td>Windows 桌面 Proxy 模式已接入 CurrentUser WinINet 事务、私有 journal、同一签名可执行文件 watchdog、活 owner 防误恢复和启动修复；sidecar 独立退出会按运行时代际安全回退 Gateway 并恢复代理；macOS 以静态逐参数 quote 的授权命令启动同一受校验可执行文件，持久 UID/PID 绑定控制连接覆盖正常退出与 owner 断连，root watchdog 覆盖 helper 崩溃；恢复事务保留 watchdog 直至解除成功，并对瞬态失败执行一次幂等重试；真实管理员交互与强杀矩阵待验收</td></tr>
         <tr><td class="task-id">CAP-007</td><td>进程/应用归因与兼容诊断</td><td><span class="status done">Done</span></td><td>100%</td><td>Codex / TBD</td><td>2026-07-18</td><td>应用标签/User-Agent/捕获模式归因及兼容诊断树完成；Windows/macOS/Linux Gateway 与 Proxy 均通过操作系统 TCP 连接精确匹配 PID，失败不推断，sidecar PID 不受信且临时端点不进入 Envelope、持久化或导出</td></tr>
         <tr><td class="task-id">DAT-001</td><td>SQLCipher、迁移、WAL 与 OS 凭据库</td><td><span class="status done">Done</span></td><td>100%</td><td>Codex / TBD</td><td>2026-07-21</td><td>schema v3 可从 v0/v1/v2 升级，新增加密应用设置；新版本仍可只读打开安装前的 v2 恢复快照，响应类型、状态码和耗时可查询；备份恢复、DB/WAL canary 与 OS 凭据库均通过</td></tr>
         <tr><td class="task-id">DAT-002</td><td>写入、查询、保留与磁盘压力</td><td><span class="status done">Done</span></td><td>100%</td><td>Codex / TBD</td><td>2026-07-15</td><td>批量写入原子提交；默认保留 30 天/50,000 条并以 500 条事务清理；低于 256 MiB 或 SQLITE_FULL 时暂停记录但保持 Gateway 转发</td></tr>
@@ -100,7 +100,7 @@ <h2 id="current-sprint">5. 当前迭代：S5 / S6</h2>
         <tr><td class="task-id">PAR-007</td><td>token、成本和语义指纹</td><td><span class="status done">Done</span></td><td>100%</td><td>Codex / TBD</td><td>2026-07-17</td><td>四厂商 reported usage 归一化、显式 estimated 估算、版本化价格匹配、未知价格留空与 BLAKE3-256 语义指纹完成</td></tr>
         <tr><td class="task-id">TST-001</td><td>协议 fixture 与 golden tests</td><td><span class="status done">Done</span></td><td>100%</td><td>Codex / TBD</td><td>2026-07-17</td><td>版本化能力矩阵覆盖 OpenAI、Anthropic、Gemini 与 Ollama 的请求、响应、流式、工具、多模态、错误和 Raw fallback；声明均由 fixture 与 golden 验证</td></tr>
         <tr><td class="task-id">TST-002</td><td>Gateway/Proxy 集成测试</td><td><span class="status active">In progress</span></td><td>82%</td><td>Codex / TBD</td><td>2026-08-25</td><td>认证 IPC、暂停丢弃、真实模式切换与进程树清理、压缩、流式脱敏、非目标 TLS、HTTP/2 基线一致性及 Windows CA 精确增删通过；真实 sidecar 在客户端取消后保持存活，IPC 停止消费且 72 个事件超过容量 64 时，36 个目标响应仍在 15 秒内完成；真实 ROOT 往返和其余协议一致性待扩展</td></tr>
-        <tr><td class="task-id">TST-003</td><td>平台故障注入</td><td><span class="status active">In progress</span></td><td>80%</td><td>Codex / TBD</td><td>2026-08-25</td><td>Windows/macOS 强杀恢复、活 owner 防误恢复、sidecar 独立退出回退和端口冲突拒绝已通过；PAC 原状态下应用与同步回滚双重失败会交由独立 watchdog 恢复；macOS CA 拒绝恢复及 helper 协议、权限、命令转义和恢复文件校验已覆盖；授权桌面 helper 的真实故障矩阵待完成</td></tr>
+        <tr><td class="task-id">TST-003</td><td>平台故障注入</td><td><span class="status active">In progress</span></td><td>82%</td><td>Codex / TBD</td><td>2026-08-25</td><td>Windows/macOS 强杀恢复、活 owner 防误恢复、sidecar 独立退出回退和端口冲突拒绝已通过；PAC 原状态下应用与同步回滚双重失败会交由独立 watchdog 恢复；恢复阶段错误带操作上下文，瞬态失败重试有故障注入覆盖；macOS CA 拒绝恢复及 helper 协议、权限、命令转义和恢复文件校验已覆盖；授权桌面 helper 的真实故障矩阵待完成</td></tr>
         <tr><td class="task-id">TST-004</td><td>UI、性能与可访问性</td><td><span class="status active">In progress</span></td><td>80%</td><td>Codex / TBD</td><td>2026-09-08</td><td>axe、WAI-ARIA tabs、方向键、焦点循环/恢复、键盘分隔条和双主题对比度已门禁；Ubuntu Chromium 固定 960×620 明色与 1440×900 暗色像素基线，千条列表要求首屏 &lt;2 秒、首尾滚动 &lt;2 秒、筛选 &lt;750 ms 且可见 DOM &lt;40 行；Gateway fake provider 基准要求额外延迟 P95 &lt;20 ms，最新本地基线为 0.609 ms；屏幕阅读器人工验收待完成</td></tr>
         <tr><td class="task-id">REL-001</td><td>签名安装、升级和回滚</td><td><span class="status active">In progress</span></td><td>80%</td><td>Codex / Release Owner TBD</td><td>2026-09-22</td><td>应用内签名更新、预更新恢复快照与失败后只读历史已实现；发布流水线会生成并验证 Windows/macOS 安装包、updater archive、`.sig`、`latest.json` 与哈希清单，先上传草稿再公开。正式证书/密钥托管、首批签名制品与隔离设备真实回滚演练待完成</td></tr>
         <tr><td class="task-id">REL-002</td><td>诊断与支持包</td><td><span class="status done">Done</span></td><td>100%</td><td>Codex / Support Owner TBD</td><td>2026-07-21</td><td>版本化 JSON 支持包可预览、复制和保存；接收端严格拒绝超限、重复键、未知格式、异常隐私声明、请求内容字段与凭据模式，只生成无内容诊断摘要；<a href="support.html">分流、升级、保留和删除流程</a>已发布</td></tr>