diff --git a/desktop-tauri/src-tauri/Cargo.toml b/desktop-tauri/src-tauri/Cargo.toml index 20c3c8fdb..934f5a2b4 100644 --- a/desktop-tauri/src-tauri/Cargo.toml +++ b/desktop-tauri/src-tauri/Cargo.toml @@ -1,6 +1,9 @@ [package] name = "workbuddy-proxy-desktop" version = "3.5.0-beta.1" +# workspace 里还有 `agent2api-server` 这个 headless 二进制;`tauri dev` 走 +# `cargo run` 时不带 `--bin`,没有 default-run 会报「无法确定运行哪个二进制」。 +default-run = "workbuddy-proxy-desktop" description = "Agent2API 桌面端 — AI 路由网关与 Agent 客户端接管" authors = ["Agent2API"] edition = "2021" diff --git a/desktop-tauri/src-tauri/server/src/server/api/accounts.rs b/desktop-tauri/src-tauri/server/src/server/api/accounts.rs index 1d3095bb7..a46f5ce0e 100644 --- a/desktop-tauri/src-tauri/server/src/server/api/accounts.rs +++ b/desktop-tauri/src-tauri/server/src/server/api/accounts.rs @@ -68,6 +68,25 @@ use crate::server::http::{ok_json, parse_body, query_param}; use crate::server::logging; use crate::server::ServerState; +/// 给账号列表附上熔断状态(账号页「熔断」徽章)。 +/// +/// 熔断状态在 `RouteState` 的运行时表里,不在账号库;所以统一在列表出口注入。 +/// `/api/session` 与 `/api/accounts` 共用这一处口径,字段名固定为 +/// `breakerOpenUntil`(毫秒时间戳,0 = 当前未熔断)。 +pub(super) fn with_breaker(mut list: Value, state: &ServerState) -> Value { + let now = logging::now_ms(); + if let Some(accounts) = list.as_array_mut() { + for account in accounts { + let Some(id) = account.get("id").and_then(Value::as_str) else { continue }; + let open_until = state.upstream().breaker_open_until(id, now).unwrap_or(0); + if let Some(object) = account.as_object_mut() { + object.insert("breakerOpenUntil".into(), open_until.into()); + } + } + } + list +} + /// 账号存储错误 → 管理信封(打日志 + 原状态码)。 /// /// 可见性是 `pub(super)`:`api::session` 的 Loomy 验证码登录要落账号 @@ -396,7 +415,7 @@ pub async fn dispatch( // ① 精确命中 `/api/accounts` if rest.is_none() { match method.as_str() { - "GET" => return ok_json(state.store().list_visible_accounts()), + "GET" => return ok_json(with_breaker(state.store().list_visible_accounts(), &state)), "POST" => return add_account(&state, body).await, // Node 版对 `/api/accounts` 上的其它方法同样走 404 信封 _ => { @@ -450,6 +469,12 @@ pub async fn dispatch( return clear_rate_limits(&state, &id, body); } } + if let Some(id) = rest.strip_suffix("/breaker/clear") { + let id = decode_segment(id); + if !id.is_empty() { + return clear_breaker(&state, &id); + } + } // ZCode「限时套餐」:探测(只读、不要验证码)与领取(要验证码)。 // 两个后缀互不包含(`/zcode-claim/preview` 不以 `/zcode-claim` 结尾), // 因此这里的先后不影响命中 —— 但读的时候按「先探测后领取」排列, @@ -1614,6 +1639,23 @@ pub async fn move_account(state: &ServerState, id: &str, body: &Bytes) -> Respon } } +// ─── POST /api/accounts/{id}/breaker/clear ───────────────────── + +/// 手动清除某账号的熔断状态(账号页「熔断 ×」)。 +/// +/// 熔断是运行时状态,不在账号库里:这里只删 `RouteState` 的那一条, +/// 失败计数与半开计数一并归零。返回 `{cleared: 是否确实删掉了}`。 +pub fn clear_breaker(state: &ServerState, id: &str) -> Response { + if id.is_empty() { + return management_error(400, "缺少账号 id"); + } + let cleared = state.upstream().breaker_clear(id); + if cleared { + logging::log("[Upstream]", &format!("账号 {id} 的熔断状态已手动清除")); + } + ok_json(json!({ "cleared": cleared })) +} + // ─── POST /api/accounts/{id}/rate-limits/clear ───────────────── /// 清除账号的限流标记:body `{model?}`,给了模型只清那一个,否则全清。 diff --git a/desktop-tauri/src-tauri/server/src/server/api/session.rs b/desktop-tauri/src-tauri/server/src/server/api/session.rs index 7fb98d195..5f3fdbaa5 100644 --- a/desktop-tauri/src-tauri/server/src/server/api/session.rs +++ b/desktop-tauri/src-tauri/server/src/server/api/session.rs @@ -47,7 +47,7 @@ pub async fn get_session(State(state): State) -> Response { let snapshot = config::current(); let plugin_startup = crate::server::core::plugin_host::startup::snapshot(); // 可见账号与路由页预演共用同一份快照,插件未就绪的账号不参与。 - let accounts = state.store().list_visible_accounts(); + let accounts = super::accounts::with_breaker(state.store().list_visible_accounts(), &state); let routed = super::routing_api::queue_for_accounts( &state, snapshot.last_request_model().map(str::to_string), &crate::server::core::routing::accounts_of(&accounts), ) diff --git a/desktop-tauri/src-tauri/server/src/server/core/limiter/evaluate.rs b/desktop-tauri/src-tauri/server/src/server/core/limiter/evaluate.rs index 06495315e..9acc0b8a5 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/limiter/evaluate.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/limiter/evaluate.rs @@ -1,6 +1,7 @@ use serde_json::Value; use super::{counter, groups, sources, types::*}; +use crate::server::core::{custom_providers, providers}; #[cfg(test)] #[path = "evaluate_tests.rs"] @@ -125,12 +126,118 @@ pub fn state(account: &Value, group: &Group, facts: &sources::Facts, counts: &co GroupState { hit, account: hit && (group.target == Target::Account || account_hit), models, conditions: readings } } +/// 模型条目里的倍率读数;`None` = 目录没有可判定的计费信息。 +/// +/// 字段优先级:`billingEffectiveMultiplier` > `billingMultiplier` > +/// `pointsMultiplier` > `multiplier` > `rate` > `credits` 展示串。 +/// 0 与负数都按「免费」处理(负数只可能是上游异常读数,按不消耗积分更安全); +/// 非有限值直接丢弃,避免把 NaN / Infinity 当成免费。 +fn multiplier_of(model: &Value) -> Option { + for key in [ + "billingEffectiveMultiplier", + "billingMultiplier", + "pointsMultiplier", + "multiplier", + "rate", + ] { + if let Some(value) = model.get(key).and_then(Value::as_f64) { + if value.is_finite() { + return Some(value); + } + } + } + // 展示串形态:`x0.00 credits` / `x0.0` / `x0.5 credits` / `0.7x` / `免费`。 + // `creditMultiplier` 是 Tabbit 的字段名;`credits` 是其余各家的通用字段。 + for key in ["credits", "creditMultiplier"] { + let Some(value) = model.get(key) else { continue }; + if let Some(number) = value.as_f64() { + if number.is_finite() { + return Some(number); + } + } + let Some(text) = value.as_str() else { continue }; + let text = text.trim(); + if text == "免费" { + return Some(0.0); + } + let text = text + .strip_prefix('x') + .or_else(|| text.strip_prefix('X')) + .unwrap_or(text) + .trim(); + let Some(number) = text.split_whitespace().next() else { continue }; + let number = number + .strip_suffix('x') + .or_else(|| number.strip_suffix('X')) + .unwrap_or(number); + if let Ok(value) = number.parse::() { + if value.is_finite() { + return Some(value); + } + } + } + None +} + +/// 某家当前生效的模型清单:自定义家走自己的模型表,其余走提供商目录。 +fn manifest(provider: &str) -> Vec { + if custom_providers::is_custom_provider_id(provider) { + custom_providers::models_of(provider) + } else { + providers::catalog::manifest_for_provider(provider) + } +} + +/// 某家上游模型当前的倍率;目录里找不到该模型时返回 `None`。 +fn model_multiplier(provider: &str, model: &str) -> Option { + if provider.is_empty() || model.is_empty() { + return None; + } + let manifest = manifest(provider); + let entry = manifest.iter().find(|entry| { + entry + .get("id") + .and_then(Value::as_str) + .is_some_and(|id| id.eq_ignore_ascii_case(model)) + || entry + .get("name") + .and_then(Value::as_str) + .is_some_and(|name| name.eq_ignore_ascii_case(model)) + })?; + multiplier_of(entry) +} + +/// 倍率 0 的限时免费模型:余额/积分限制不应生效。 +fn is_free_model(provider: &str, model: &str) -> bool { + model_multiplier(provider, model).is_some_and(|multiplier| multiplier <= 0.0) +} + +/// 命中里是否只有余额/额度条件。 +/// +/// `local = true` 的是 Token / 请求数这类**本地消耗保护**,不是账号积分; +/// 免费模型可以豁免前者,但不该豁免后者。 +fn only_balance_conditions(conditions: &[Reading]) -> bool { + conditions.iter().filter(|reading| reading.hit).all(|reading| !reading.local) +} + pub fn blocked(account: &Value, model: &str, facts: &sources::Facts, counts: &counter::Facts, now: i64) -> bool { + let provider = crate::server::core::routing::provider_of(account); + // 目录查询只在「确实有余额条件命中」时发生一次:多数账号不会走到这里。 + let mut free: Option = None; groups::effective_groups(account).iter().any(|group| { if group.action != Action::Skip { return false; } let state = state(account, group, facts, counts, now); if !state.hit { return false; } - state.account || (!model.is_empty() && state.models.iter().any(|wanted| wanted.eq_ignore_ascii_case(model))) + let matched = state.account + || (!model.is_empty() && state.models.iter().any(|wanted| wanted.eq_ignore_ascii_case(model))); + if !matched { return false; } + // 倍率 0 的限时免费模型不消耗账号积分:纯余额/额度条件的命中不生效。 + // Token / 请求数限制仍然生效(那是本地保护,与积分无关)。 + if !model.is_empty() && only_balance_conditions(&state.conditions) { + let is_free = *free.get_or_insert_with(|| is_free_model(provider, model)); + if is_free { return false; } + } + true }) } diff --git a/desktop-tauri/src-tauri/server/src/server/core/limiter/evaluate_tests.rs b/desktop-tauri/src-tauri/server/src/server/core/limiter/evaluate_tests.rs index 72988641e..443dcafdb 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/limiter/evaluate_tests.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/limiter/evaluate_tests.rs @@ -163,3 +163,70 @@ fn invalid_limits_and_out_of_scope_models_are_rejected() { json!({"id":"local","label":"local","models":["allowed"]}); assert!(groups::normalize_groups(&value).is_err()); } + +#[test] +fn free_model_exemption_only_covers_balance_conditions() { + let balance = Reading { hit: true, local: false, ..Default::default() }; + let token = Reading { hit: true, local: true, ..Default::default() }; + let missed_token = Reading { hit: false, local: true, ..Default::default() }; + let missed_balance = Reading { hit: false, local: false, ..Default::default() }; + + // 纯余额命中:免费模型可以豁免。 + assert!(only_balance_conditions(&[balance.clone()])); + // Token / 请求数命中:本地保护,免费模型也不能豁免。 + assert!(!only_balance_conditions(&[token.clone()])); + // 余额 + Token 同时命中:保留拦截。 + assert!(!only_balance_conditions(&[balance.clone(), token])); + // 未命中的条件不参与判定。 + assert!(only_balance_conditions(&[balance, missed_token, missed_balance])); +} + +#[test] +fn parses_numeric_multipliers_before_credit_strings() { + // 有效倍率优先:raccoon 的免费模型同时有 0.5 的 billingMultiplier + // 与 0 的 billingEffectiveMultiplier,实际计费以 effective 为准。 + assert_eq!( + multiplier_of(&json!({ + "billingEffectiveMultiplier": 0, + "billingMultiplier": 0.5, + "pointsMultiplier": 0.5, + "credits": "x0 credits", + })), + Some(0.0) + ); + assert_eq!( + multiplier_of(&json!({"billingMultiplier": 1, "credits": "x1 credits"})), + Some(1.0) + ); +} + +#[test] +fn parses_credit_display_strings() { + assert_eq!(multiplier_of(&json!({"credits": "x0.00 credits"})), Some(0.0)); + assert_eq!(multiplier_of(&json!({"credits": "x0.0"})), Some(0.0)); + assert_eq!(multiplier_of(&json!({"credits": "x0.5 credits"})), Some(0.5)); + assert_eq!(multiplier_of(&json!({"credits": "x1.33 credits"})), Some(1.33)); + // 非数值展示(autoclaw 的低/中/高)不是计费读数,不做免费判定。 + assert_eq!(multiplier_of(&json!({"credits": "低"})), None); + assert_eq!(multiplier_of(&json!({"credits": ""})), None); + assert_eq!(multiplier_of(&json!({})), None); +} + +#[test] +fn non_finite_readings_are_ignored() { + assert_eq!(multiplier_of(&json!({"billingMultiplier": "x"})), None); + assert_eq!(multiplier_of(&json!({"credits": "xNaN credits"})), None); +} + +#[test] +fn parses_alternate_credit_fields_and_free_labels() { + // LobsterAI 把 0 倍率写成「免费」。 + assert_eq!(multiplier_of(&json!({"credits": "免费"})), Some(0.0)); + // CodeArts 的展示串是后缀形态。 + assert_eq!(multiplier_of(&json!({"credits": "0.7x"})), Some(0.7)); + // Tabbit 用 creditMultiplier,不是 credits。 + assert_eq!(multiplier_of(&json!({"creditMultiplier": "x0"})), Some(0.0)); + assert_eq!(multiplier_of(&json!({"creditMultiplier": "x1"})), Some(1.0)); + // 数字形态的 0 也按免费处理(防御性兼容)。 + assert_eq!(multiplier_of(&json!({"credits": 0})), Some(0.0)); +} diff --git a/desktop-tauri/src-tauri/server/src/server/core/plugin_host/rpc.rs b/desktop-tauri/src-tauri/server/src/server/core/plugin_host/rpc.rs index 35732695e..6de3b3b95 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/plugin_host/rpc.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/plugin_host/rpc.rs @@ -1236,8 +1236,15 @@ impl ProviderAdapter for RpcAdapter { let method = crate::server::core::providers::retry::RETRY_DECISION_METHOD; match self.control(method, Value::Object(params)).await { Ok(value) => { - let decision = serde_json::from_value(value).unwrap_or_else(|_| RetryDecision::Stop { - reason: "提供商插件返回的重试决定无法解析,停止本次请求".to_string(), + // 解析失败 ≠ 插件要求停止:按 Default 交回编排层的通用重试 / + // 换号预算。兜底成 Stop 会让一次协议错位直接吃掉整条请求的 + // 失败转移能力(实测:502 后所有候选都没走到)。 + let decision = serde_json::from_value(value).unwrap_or_else(|error| { + crate::server::logging::console_line( + "[Upstream]", + &format!("提供商插件返回的重试决定无法解析({error}),按默认重试/换号处理"), + ); + RetryDecision::Default }); if strict::applies(self.id) { strict::retry_decision(decision) } else { decision } } @@ -1250,9 +1257,15 @@ impl ProviderAdapter for RpcAdapter { None => RetryDecision::Default, } } - Err(error) => RetryDecision::Stop { - reason: format!("提供商插件无法给出重试决定,停止本次请求:{}", error.message), - }, + Err(error) => { + // 控制面 RPC 失败(超时 / 插件忙 / 通道断)不是「上游要求停止」。 + // 同样按 Default 处理:让瞬时 5xx 继续走换号,而不是整单终止。 + crate::server::logging::console_line( + "[Upstream]", + &format!("提供商插件无法给出重试决定({}),按默认重试/换号处理", error.message), + ); + RetryDecision::Default + } } }) } diff --git a/desktop-tauri/src-tauri/server/src/server/core/routing_groups.rs b/desktop-tauri/src-tauri/server/src/server/core/routing_groups.rs index a8310de33..99ca290ce 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/routing_groups.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/routing_groups.rs @@ -122,6 +122,11 @@ pub fn project_with_vision(group: &RoutingGroup, accounts: &[Value], needs: &Req let item = manifests.get(&member.provider).and_then(|items| items.iter().find(|item| model_id(item).eq_ignore_ascii_case(&member.model))).cloned(); let reason = if !key_scope::allows_provider(scope, &member.provider) { Some("网关 Key 不允许此提供商".to_string()) } else if !provider_active(&member.provider) { Some("提供商当前不可用".to_string()) } + else if !crate::server::core::model_rules::shared().default_enabled(&member.provider, &member.model) { + // 模型管理里的「启用 / 映射」开关是全局可见性:关掉之后, + // 路由组也不该再用它(组员配置保留,便于随时重新打开)。 + Some("模型已在模型管理中关闭(启用/映射开关),配置保留但不参与转发".to_string()) + } else if let Some(item) = &item { if providers::images::model_is_image(item) { Some("生图模型不能用于对话路由".to_string()) } else { diff --git a/desktop-tauri/src-tauri/server/src/server/core/upstream/mod.rs b/desktop-tauri/src-tauri/server/src/server/core/upstream/mod.rs index 3dcdb0e22..9ccd27c6f 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/upstream/mod.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/upstream/mod.rs @@ -344,6 +344,18 @@ impl UpstreamService { (snapshot.sunk, snapshot.resting, snapshot.open) } + /// 某账号熔断打开的恢复时刻(毫秒);未熔断返回 None。 + /// + /// 账号页「熔断」徽章用;与 [`Self::route_queue_facts`] 的 open 集合同源。 + pub fn breaker_open_until(&self, account_id: &str, now: i64) -> Option { + self.route.breaker_open_until(account_id, now) + } + + /// 手动清除某账号的熔断状态;返回是否确实删掉了。 + pub fn breaker_clear(&self, account_id: &str) -> bool { + self.route.breaker_clear(account_id) + } + pub fn route_sticky_sessions(&self, model: Option<&str>, now: i64) -> Vec { self.route.sticky_list(model, now) } diff --git a/desktop-tauri/src-tauri/server/src/server/core/upstream/plugin_forward.rs b/desktop-tauri/src-tauri/server/src/server/core/upstream/plugin_forward.rs index 8ace9e106..343fe8ce9 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/upstream/plugin_forward.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/upstream/plugin_forward.rs @@ -38,7 +38,9 @@ pub(super) async fn forward( let class = adapter.classify_conversation_error(&error); let detail = serde_json::json!({ "message": error.message, "code": error.code }); let policy = adapter.error_policy_async(status, &detail).await; - if policy.stop || settings.no_retry(status) || policy.suspend_account || policy.account_cooldown_until.is_some() { + // 402(积分不足)不在本插件循环里原地重发:外层会按账号级额度耗尽 + // 标记该账号×模型的冷却并换下一个候选。这里的重发只会白等一个间隔。 + if status == 402 || policy.stop || settings.no_retry(status) || policy.suspend_account || policy.account_cooldown_until.is_some() { return Err(error); } if !matches!(class, UpstreamErrorClass::TokenExpired { .. }) { diff --git a/desktop-tauri/src-tauri/server/src/server/core/upstream/provider_loop.rs b/desktop-tauri/src-tauri/server/src/server/core/upstream/provider_loop.rs index 947f8f513..6eb90822a 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/upstream/provider_loop.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/upstream/provider_loop.rs @@ -308,6 +308,29 @@ fn direct_switch_status(status: i64) -> bool { .unwrap_or(false) } +/// 402(Payment Required / 积分不足)统一归一为**账号级额度耗尽**。 +/// +/// 与 429 同一处置:标记该账号×模型的冷却,然后换下一个候选。不能只依赖 +/// 「指定错误码直接换号」名单 —— 名单是用户可清空的;清空后 strict/stateful +/// 的早退门会把 402 当普通 Fatal 直接收尾,连当前请求都不换号。而 402 本身 +/// 也没有像 429 那样落 `rateLimits` 冷却,下一条请求还会再选它。 +/// +/// 适配器已经声明 QuotaLimited / TokenExpired / ContentBlocked 的,保持原分类。 +fn credit_exhausted(status: u16, class: UpstreamErrorClass) -> UpstreamErrorClass { + if status != 402 { + return class; + } + match class { + UpstreamErrorClass::Fatal { message, upstream_code, .. } => UpstreamErrorClass::QuotaLimited { + status: 402, + message, + upstream_code, + reset_at: None, + }, + other => other, + } +} + /// 瞬时 HTTP 错误的统一退避建议(设置页「请求重试」的全局兜底)。 /// /// `remaining` 是原地重发**还剩几次**(见 [`RetryBudget`] 的说明); @@ -418,7 +441,7 @@ pub(super) async fn forward_with_providers( if ctx.routing.group.is_some() { ctx.telemetry.set_route_decision(json!({ "scope": ctx.routing.scope, "configSnapshot": ctx.routing.json(), "strategy": ctx.routing.settings.strategy.as_str(), "sticky": ctx.routing.settings.sticky.as_str(), - "ordered": [], "blocked": ctx.routing.group_projection.blocked.iter().take(12).collect::>(), + "ordered": [], "blocked": ctx.routing.group_projection.blocked.iter().take(rotate::ROUTE_SNAPSHOT_LIMIT).collect::>(), "orderedTotal": ctx.routing.group_projection.accounts.len(), "blockedTotal": ctx.routing.group_projection.blocked.len() })); } // 候选家来自 `route_for_forward`(目录里没有这个模型名时会回落成默认 @@ -621,7 +644,7 @@ async fn attempt_queue( }); if ctx.routing.settings.sticky.ttl_ms().is_some() { decision["stickyId"] = decision["sessionId"].clone(); } if let Some(group) = &ctx.routing.group { - decision["blocked"] = json!(ctx.routing.group_projection.blocked.iter().take(12).collect::>()); + decision["blocked"] = json!(ctx.routing.group_projection.blocked.iter().take(rotate::ROUTE_SNAPSHOT_LIMIT).collect::>()); decision["blockedTotal"] = ctx.routing.group_projection.blocked.len().into(); decision["groupId"] = group.id.clone().into(); } @@ -744,6 +767,11 @@ async fn attempt_queue( match attempt_custom(service, ctx, target, slot, connections, degraded).await { Ok(outcome) => { if let Some(account_id) = custom_account_id.as_deref() { + // 与内置 / 会话式成功路径同款:成功要喂熔断恢复计数。 + let route_settings = ctx.routing.settings; + if route_settings.breaker_enabled { + service.route.breaker_success(account_id, route_settings.breaker_success_threshold); + } service.route.sticky_bind( &conv_key, &model, &attempt_key, sticky_source, logging::now_ms(), ctx.routing.settings.sticky.ttl_ms().unwrap_or(0), sticky_generation, @@ -779,7 +807,8 @@ async fn attempt_queue( // 并进 message、`mark_account_limited` 内部再解析)。 // 自定义家没有「同名多池」之类的键重排,冷却键 // 与发送名同源(`custom::forward::cooldown_model`)。 - if error.is_quota_limit() { + // 402(积分不足)与 429 同一处置:标记冷却后顺延。 + if error.is_quota_limit() || error.status_code == 402 { let wire_model = if ctx.routing.group.is_some() { selected_model.clone() } else { custom_forward::cooldown_model(&custom_provider_id, &model) }; @@ -875,6 +904,9 @@ async fn attempt_queue( // 它失败了就把它记入已尝试、回到循环挑下一个账号 —— 可能已经换了一家。 // 没有下一个可用账号时,把这个错误原样透传(它的文案最贴近真实原因)。 let stateful_account_id = target.account_id.clone(); + // `attempt_stateful` 会接管 `target`;熔断日志需要账号展示名, + // 所以在移动之前留一份只读副本。 + let stateful_account = target.account.clone(); let strict_runtime = crate::server::core::providers::is_scoped_runtime_provider(provider_id); match attempt_stateful( service, @@ -891,6 +923,12 @@ async fn attempt_queue( { Ok(outcome) => { if let Some(account_id) = stateful_account_id.as_deref() { + // 会话式成功也要喂熔断恢复计数:否则半开状态的账号 + // 永远等不到「连续成功 N 次关闭」的那一步。 + let route_settings = ctx.routing.settings; + if route_settings.breaker_enabled { + service.route.breaker_success(account_id, route_settings.breaker_success_threshold); + } service.route.sticky_bind( &conv_key, &model, &attempt_key, sticky_source, logging::now_ms(), ctx.routing.settings.sticky.ttl_ms().unwrap_or(0), sticky_generation, @@ -931,21 +969,41 @@ async fn attempt_queue( // 整份 `class` 交给 `mark_conversation_limited` 而不是在这里 // 拆散取值 —— 拆散后「漏带一个字段」编译期与运行期都不报错 // (本分支此前就把 `reset_at` 写成死 `None`,见那个函数的说明)。 - let class = adapter.classify_conversation_error(&error); + let class = credit_exhausted( + u16::try_from(error.status_code).unwrap_or(0), + adapter.classify_conversation_error(&error), + ); + // ── 上游 4xx/5xx:账号级失败,按队列顺延到下一个候选 ── + // 不区分具体错误码:400/402/403/404/429/5xx 都说明「这个 + // 账号/模型这一次没答上来」,换下一个账号或模型仍可能成功。 + // 终止条件只剩:手动取消、生图(可能已计费);显式 policy.stop + // 在全局队列/单候选直接收尾,路由组内只跳过当前候选继续试。 + // + // 402 会在上面归一成 QuotaLimited,额外落一条账号×模型冷却; + // 其余码至少在本请求里把失败账号记入 tried_ids 后顺延。 + let status = u16::try_from(error.status_code).unwrap_or(0); + let switchable = (400..600).contains(&status); // 会话可能持有待回传的工具结果,显式停止策略优先于账号顺延。 - let policy = adapter.error_policy_async(error.status_code as u16, &serde_json::json!({ + // 但用户的口径是「4xx/5xx 一律换」:插件的 Stop 只跳过当前 + // 候选,不应终止整条请求。非 HTTP 错误(200 业务错误等) + // 仍直接收尾;手动取消与生图在前面的分支已经先返回。 + let policy = adapter.error_policy_async(status, &serde_json::json!({ "message": error.message, "code": error.code, "upstreamCode": error.upstream_code, })).await; if policy.stop { if !policy.reason.is_empty() { ctx.telemetry.note_attempt_notice(&policy.reason); } - return Err(error); + if !switchable { + return Err(error); + } } // Scoped runtime 的原生 loader 已拥有单次 token 恢复;宿主不再消费 - // refresh,也不把 Fatal/权限/WAF/取消/超时改成账号顺延。 + // refresh。除显式策略/取消/生图外,上游 4xx/5xx 一律顺延: + // 不再把 Fatal/权限/WAF/超时当成请求级终止。 if (strict_runtime || adapter.forwards_response() && !adapter.is_stateful()) - && !account_unavailable && !matches!(&class, UpstreamErrorClass::QuotaLimited { .. }) { + && !account_unavailable && !switchable + && !matches!(&class, UpstreamErrorClass::QuotaLimited { .. }) { return Err(error); } if let (UpstreamErrorClass::QuotaLimited { .. }, Some(account_id)) = @@ -969,6 +1027,40 @@ async fn attempt_queue( ), ); } + // ── 路由状态的失败钩子(与无状态动作 3 同款)────────── + // 会话式分支此前只记 tried_ids,不记退避 / 熔断;于是 + // Tabbit / qwenwork / CatPaw 这类会话式家连续失败也永远不熔断, + // 自定义路由组里会反复被选中。429 额度冷却走上面的专属路径, + // 不在这里重复记账。 + if let Some(account_id) = stateful_account_id.as_deref() { + if !matches!(&class, UpstreamErrorClass::QuotaLimited { .. }) { + let route_settings = ctx.routing.settings; + let now = logging::now_ms(); + if route_settings.failover_backoff { + service.route.backoff_mark(account_id, now); + } + if route_settings.breaker_enabled { + let opened = service.route.breaker_failure( + account_id, + route_settings.breaker_threshold, + route_settings.breaker_timeout_seconds, + now, + ); + if opened { + logging::console_line( + "[Upstream]", + &format!( + "⛔ 账号 {} 连续失败 {} 次,熔断 {} 秒(到点放行试一次,连续成功 {} 次恢复)", + account_display(stateful_account.as_ref().unwrap_or(&Value::Null)), + route_settings.breaker_threshold, + route_settings.breaker_timeout_seconds, + route_settings.breaker_success_threshold, + ), + ); + } + } + } + } match rotate::pick_next_account( service, provider_ids, @@ -1474,7 +1566,24 @@ async fn attempt_queue( }); } } - if failure.stop || failure.policy.stop { + // ── 插件说 Stop,但错误是上游 4xx/5xx:仍然换号 ────────── + // `RetryDecision::Stop` 的字面语义是「本次请求不再原地重发」。 + // 当上游返回任意 4xx/5xx 时,这条结论只对**当前账号/模型** + // 成立:同一份请求换到下一个候选完全可能成功。因此这里只在 + // 「非上游错误」或「显式 policy.stop」时终止整条请求;上游 + // 4xx/5xx 落到下面的动作 3,按队列继续换号。 + // + // 生图请求例外:上游可能已经消耗额度,换号重发有重复计费 + // 风险,维持原来的直接收尾。手动终止的 499 也由前面的取消 + // 判定先收尾。 + let status = u16::try_from(failure.error.status_code).unwrap_or(0); + let switchable = ctx.images.is_none() && (400..600).contains(&status); + let transient_stop = failure.stop && !failure.policy.stop && switchable; + // 显式 policy.stop 在 4xx/5xx 时降为「跳过当前候选」: + // 插件的「换账号无法解决」不等于整条请求都没救,下一个 + // 账号/模型仍可能是另一家。非 HTTP 错误仍直接收尾。 + let candidate_stop = failure.policy.stop && switchable; + if (failure.stop || failure.policy.stop) && !transient_stop && !candidate_stop { ctx.telemetry.finish_last_attempt( Some(i64::from(failure.error.status_code)), Some(&failure.error.message), @@ -1489,7 +1598,8 @@ async fn attempt_queue( // 「这一轮的结局已定」那条公共定稿出口,之后与其它错误 // 一样按动作 1 / 动作 3 换号顺延:直接换下一个账号继续试, // 换满或没有更多账号时错误才原样返回客户端。 - let named_switch = failure.policy.suspend_account + let named_switch = failure.policy.stop + || failure.policy.suspend_account || failure.policy.account_cooldown_until.is_some() || direct_switch_status(i64::from(failure.error.status_code)); // ── 动作 0:内容策略拦截 → 换中性提示词,同账号立即重试一次 ── @@ -3134,6 +3244,7 @@ async fn send_with_retry( let body = detail.to_value(); let mut class = adapter.classify_error_async(status, &body).await; super::retry_after::apply(&mut class, header_reset); + class = credit_exhausted(status, class); let policy = adapter.error_policy_async(status, &body).await; let decision = adapter .retry_decision_async(status, &body, budget.used(), budget.total) diff --git a/desktop-tauri/src-tauri/server/src/server/core/upstream/provider_loop/retry_tests.rs b/desktop-tauri/src-tauri/server/src/server/core/upstream/provider_loop/retry_tests.rs index 172f3c29b..430661709 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/upstream/provider_loop/retry_tests.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/upstream/provider_loop/retry_tests.rs @@ -469,3 +469,45 @@ fn every_retry_advice_stops_when_the_budget_is_exhausted() { }; assert!(transport_retry_advice(&error, 0).is_none()); } + +#[test] +fn payment_required_is_normalized_to_an_account_level_quota() { + let class = credit_exhausted( + 402, + UpstreamErrorClass::Fatal { + status: 402, + message: "积分不足".into(), + upstream_code: Some(7), + }, + ); + match class { + UpstreamErrorClass::QuotaLimited { status, message, upstream_code, reset_at } => { + assert_eq!(status, 402); + assert_eq!(message, "积分不足"); + assert_eq!(upstream_code, Some(7)); + assert_eq!(reset_at, None); + } + other => panic!("402 应归一为 QuotaLimited,实际 {other:?}"), + } + // 其他状态码不改变分类 + let class = credit_exhausted( + 400, + UpstreamErrorClass::Fatal { + status: 400, + message: "bad".into(), + upstream_code: None, + }, + ); + assert!(matches!(class, UpstreamErrorClass::Fatal { status: 400, .. })); + // 适配器已经声明的 QuotaLimited 保持原值(包括已有的 reset_at) + let class = credit_exhausted( + 402, + UpstreamErrorClass::QuotaLimited { + status: 402, + message: "quota".into(), + upstream_code: None, + reset_at: Some(1), + }, + ); + assert!(matches!(class, UpstreamErrorClass::QuotaLimited { reset_at: Some(1), .. })); +} diff --git a/desktop-tauri/src-tauri/server/src/server/core/upstream/rotate.rs b/desktop-tauri/src-tauri/server/src/server/core/upstream/rotate.rs index 1ecfa0988..a6302d559 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/upstream/rotate.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/upstream/rotate.rs @@ -309,7 +309,7 @@ fn select_target_account_sync( decision["scope"] = Value::String(request_routing.scope.clone()); decision["configSnapshot"] = request_routing.json(); if let Some(blocked) = decision.get_mut("blocked").and_then(Value::as_array_mut) { - blocked.extend(request_routing.group_projection.blocked.iter().take(12usize.saturating_sub(blocked.len())).cloned()); + blocked.extend(request_routing.group_projection.blocked.iter().take(ROUTE_SNAPSHOT_LIMIT.saturating_sub(blocked.len())).cloned()); } decision["blockedTotal"] = (plan.blocked.len() + request_routing.group_projection.blocked.len()).into(); if !conv_key.is_empty() { @@ -823,10 +823,19 @@ pub(super) fn pick_next_account( plan.ordered.into_iter().next().map(|entry| entry.account) } +/// 请求日志 / 选路回放里保存的候选上限。 +/// +/// 12 条在只有全局队列时够用;路由组把「成员 × 账号」摊平后经常有 20+ +/// 个候选,截断会让「为什么没选它」无据可查(实测:被 413 试过的账号 +/// 落在第 13 位之后,回放里只剩一句「未保存」)。128 覆盖常见规模, +/// 同时仍保留一个上限避免请求日志无限膨胀。 +pub(super) const ROUTE_SNAPSHOT_LIMIT: usize = 128; + /// 选路决策快照(请求日志「路由回放」的数据源)。 /// -/// 有界:排序清单与被跳过者各最多带 12 条 —— 账号几十个时全部带上只会 -/// 让每条请求日志膨胀,而前 12 位已足以回答「为什么是它、为什么不是那个」。 +/// 有界:排序清单与被跳过者各最多带 [`ROUTE_SNAPSHOT_LIMIT`] 条 —— 账号 +/// 几十个时全量保存会让每条请求日志膨胀;128 已覆盖路由组的常见规模, +/// 超出时前端会明确提示「候选快照仅保存部分记录」。 fn decision_json( plan: &routing::RoutePlan, settings: &config::RoutingSettings, @@ -850,8 +859,8 @@ fn decision_json( "sticky": settings.sticky.as_str(), "stickyHit": plan.sticky_hit, "picked": plan.ordered.first().map(|entry| entry.id.clone()).unwrap_or_default(), - "ordered": plan.ordered.iter().take(12).map(entry_json).collect::>(), - "blocked": plan.blocked.iter().take(12).map(entry_json).collect::>(), + "ordered": plan.ordered.iter().take(ROUTE_SNAPSHOT_LIMIT).map(entry_json).collect::>(), + "blocked": plan.blocked.iter().take(ROUTE_SNAPSHOT_LIMIT).map(entry_json).collect::>(), "orderedTotal": plan.ordered.len(), "blockedTotal": plan.blocked.len(), }) diff --git a/desktop-tauri/src-tauri/server/src/server/core/upstream/route_state.rs b/desktop-tauri/src-tauri/server/src/server/core/upstream/route_state.rs index 5ebf89010..c03ed8a57 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/upstream/route_state.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/upstream/route_state.rs @@ -504,6 +504,27 @@ impl RouteState { } } + /// 熔断打开中的恢复时刻(毫秒);未打开 / 半开期返回 None。 + /// + /// 账号页的「熔断」徽章用;与选路快照里的 `open` 同源(`open_until > now`)。 + pub(super) fn breaker_open_until(&self, account_id: &str, now: i64) -> Option { + let table = self.breaker.lock().ok()?; + table + .get(account_id) + .map(|entry| entry.open_until) + .filter(|until| *until > now) + } + + /// 手动清除某账号的熔断状态(账号页「熔断 ×」)。 + /// + /// 只删运行时表里的这一条;失败计数与半开计数一并归零。返回是否确实删掉了。 + pub(super) fn breaker_clear(&self, account_id: &str) -> bool { + let Ok(mut table) = self.breaker.lock() else { + return false; + }; + table.remove(account_id).is_some() + } + /// 选路快照:粘性命中 + 沉底 / 休息 / 熔断三张表(一次取齐)。 pub(super) fn snapshot(&self, conv_key: &str, now: i64, ttl_ms: i64) -> RouteSnapshot { self.snapshot_for_model(conv_key, "", now, ttl_ms) diff --git a/desktop-tauri/src-tauri/src/bridge.rs b/desktop-tauri/src-tauri/src/bridge.rs index 85e198c59..7ba851ee8 100644 --- a/desktop-tauri/src-tauri/src/bridge.rs +++ b/desktop-tauri/src-tauri/src/bridge.rs @@ -371,6 +371,9 @@ const BRIDGE_JS: &str = r#" clearRateLimits: (id, model) => call('POST', '/api/accounts/' + encodeURIComponent(id) + '/rate-limits/clear', model ? { model } : {}), + // 清除账号级熔断(账号页「熔断 ×」):运行时状态,不动账号记录。 + clearBreaker: id => + call('POST', '/api/accounts/' + encodeURIComponent(id) + '/breaker/clear', {}), // ── 账号路由(策略 / 粘性 / 退避 / 垫底 / 熔断 + 队列可见化)── getRoutingConfig: () => call('GET', '/api/routing'), getRoutingWorkspace: model => call('GET', '/api/routing/workspace' + toQuery({ model })), diff --git a/desktop-tauri/ui-islands/src/islands/accounts-data.ts b/desktop-tauri/ui-islands/src/islands/accounts-data.ts index daa3bbb70..0202867a7 100644 --- a/desktop-tauri/ui-islands/src/islands/accounts-data.ts +++ b/desktop-tauri/ui-islands/src/islands/accounts-data.ts @@ -239,6 +239,17 @@ export async function clearLimits(id: string, model?: string): Promise { } } +/** 清除账号级熔断:只清运行时状态,下一次请求会重新尝试这个账号。 */ +export async function clearBreaker(id: string): Promise { + try { + await shared().workbuddyDesktop?.clearBreaker?.(id) + toast(t('✅ 已清除该账号的熔断状态')) + await shared().wbApp?.refresh?.() + } catch (error) { + toast(t('清除失败:{message}', { message: errorMessage(error) }), 'err') + } +} + /** * 代理列选中即保存(`value` 是下拉的值)。下拉的选项只有三类: * '' → 直连(proxy = null) diff --git a/desktop-tauri/ui-islands/src/islands/accounts-page.tsx b/desktop-tauri/ui-islands/src/islands/accounts-page.tsx index f9e7ef1f9..673067fa9 100644 --- a/desktop-tauri/ui-islands/src/islands/accounts-page.tsx +++ b/desktop-tauri/ui-islands/src/islands/accounts-page.tsx @@ -11,7 +11,7 @@ import { shared, toast, type AccountRecord, type ColSettingsHandle } from './acc import { isDesktopAccount, isEnabled, providerOf, supportsUsage } from './accounts-domain' import { ACCOUNT_COLUMNS, bindColumnGrips, columnWidths, tableMinWidth } from './accounts-columns' import { - allAccounts, bump, clearFilter, clearLimits, ensureProxyPoolOptions, + allAccounts, bump, clearBreaker, clearFilter, clearLimits, ensureProxyPoolOptions, filterActive, getStore, groupSortRules, groupView, hiddenAccounts, installAccountsApi, normalizeProviderFilter, openBatchDialog, providerSummaryList, queryAllUsage, cancelAllUsage, refreshList, reorderQueue, rowContext, seats, segmentCounts, setAllGroupsCollapsed, setAllPicked, setProviderFilter, setSearchFilter, setSegmentFilter, @@ -351,7 +351,7 @@ function AccountsPage() { case 'status': return case 'limits': - return void clearLimits(account.id, model)} /> + return void clearLimits(account.id, model)} onClearBreaker={() => void clearBreaker(account.id)} /> case 'expiry': return case 'usage': diff --git a/desktop-tauri/ui-islands/src/islands/accounts-panels.tsx b/desktop-tauri/ui-islands/src/islands/accounts-panels.tsx index 65b36e4a4..34f6ed06f 100644 --- a/desktop-tauri/ui-islands/src/islands/accounts-panels.tsx +++ b/desktop-tauri/ui-islands/src/islands/accounts-panels.tsx @@ -330,52 +330,69 @@ export function StatusCell({ account }: { account: AccountRecord }) { * 三家形态的判据都是「有没有生效中的限额记录」,不为任何一家开特例 —— * 记录怎么来(429 / 限额码 / 上游的 Retry-After)是转发层的事,界面只认这一份。 */ -export function LimitsCell({ account, onClear }: { +export function LimitsCell({ account, onClear, onClearBreaker }: { account: AccountRecord onClear: (model: string) => void + onClearBreaker: () => void }) { const entries = activeLimits(account) const cooldown = upstreamCooldownOf(account) const suspendedReason = upstreamSuspendedReasonOf(account) + // 账号级熔断:与限制器 / 限流是两套状态;有熔断时额外挂一枚红色徽章。 + // 恢复时刻来自 /api/session 的 breakerOpenUntil(毫秒,0 = 当前未熔断)。 + const breakerUntil = Number(account.breakerOpenUntil || 0) + // 每秒重算一次,徽章显示「熔断 4:32」;没有熔断时不挂定时器。 + const [now, setNow] = React.useState(() => Date.now()) + const breakerOpen = breakerUntil > now + React.useEffect(() => { + if (!breakerOpen) return + const timer = window.setInterval(() => setNow(Date.now()), 1000) + return () => window.clearInterval(timer) + }, [breakerOpen, breakerUntil]) + const remaining = Math.max(0, breakerUntil - now) + const seconds = Math.ceil(remaining / 1000) + const countdown = `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, '0')}` + const breaker = breakerOpen + ? + + {t('熔断')} {countdown} + + + + : null if (suspendedReason) { - return {t('上游限制 · 已禁用')} + return <>{breaker}{t('上游限制 · 已禁用')} } if (!entries.length && !cooldown) { if (Array.isArray(account.limiterGroups) && limiterHit(account)) { - return openSettingsDialog(account.id)} /> + return <>{breaker} openSettingsDialog(account.id)} /> } if (balanceBlockedOf(account, usageEntryOf(account))) { - return ( - {breaker} {t('余额不足')} - - ) + } if (tokenBlockedOf(account, tokenUsageOf(account))) { - return ( - {breaker} {t('Token 限额')} - - ) + } - return ( - + return <>{breaker} {t('正常')} - - ) + } - return ( - {breaker} - ) + /> } /* ─── 有效期 / 连接数 / 余额列 ──────────────── */ diff --git a/desktop-tauri/ui-islands/src/islands/accounts-shared.ts b/desktop-tauri/ui-islands/src/islands/accounts-shared.ts index 89cac7e96..b18095d2d 100644 --- a/desktop-tauri/ui-islands/src/islands/accounts-shared.ts +++ b/desktop-tauri/ui-islands/src/islands/accounts-shared.ts @@ -257,6 +257,8 @@ export type AccountRecord = { source?: string proxy?: ProxyConfig | null rateLimits?: Record + /** 账号级熔断的恢复时刻(毫秒);0 / 缺省 = 当前未熔断。 */ + breakerOpenUntil?: number upstreamCooldownUntil?: number | null upstreamBlockedReason?: string [key: string]: unknown @@ -324,6 +326,8 @@ export type AccountsBridge = { saveAccountSorting(payload: { provider: string; rules: GroupSortRule[] | null; onlyIfUnset?: boolean }): Promise moveAccount(id: string, direction: 'up' | 'down'): Promise clearRateLimits(id: string, model?: string | null): Promise + /** 清除账号级熔断(运行时状态,不动账号记录)。 */ + clearBreaker(id: string): Promise /** * 账号路由(策略 / 粘性 / 退避 / 垫底 / 熔断)与候选队列可见化。 * `setRoutingConfig` 收**部分**补丁、返回生效后的全量值(与 /api/timeouts