diff --git a/config.example.yaml b/config.example.yaml index a499f56b3..da4badfa6 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -48,6 +48,41 @@ audit: # MCP 状态监控执行记录保留（tool_executions 表） monitor: retention_days: 90 # 省略时默认 90；0 表示不自动清理 +# 运行空间垃圾清理（系统设置 -> 存储清理） +# 覆盖 Agent 工作区、工具输出缓存、C2 产物、对话上传件等磁盘产物的保留策略。 +# auto_clean 默认关闭：升级后不会在管理员不知情的情况下删除既有数据； +# 关闭时仍可在「存储清理」页手动预览并执行清理。 +storage: + auto_clean: false # true 开启后台定时清理 + interval_minutes: 60 # 后台清理间隔（分钟）；最小 5 + orphan_grace_days: 1 # 会话/项目已删除但目录残留时，闲置多少天后回收 + active_grace_hours: 24 # 最近有活动的会话一律跳过，避免误删正在跑的任务数据 + # 各类别 retention_days：省略时使用下方默认值；0 表示不按保留期清理（孤儿目录仍会回收） + categories: + workspace: # Agent 工作区 tmp/workspace；默认 30 天 + enabled: true + retention_days: 30 + reduction: # 超长工具输出落盘 tmp/reduction；默认 7 天（纯派生数据） + enabled: true + retention_days: 7 + conversation_artifacts: # 摘要与超长输入台账 data/conversation_artifacts；默认 30 天 + enabled: true + retention_days: 30 + plantask: # 多代理计划看板 skills/.eino/plantask；默认 30 天（纯派生数据） + enabled: true + retention_days: 30 + c2_artifacts: # C2 回传截图/上传件/下发文件/payload tmp/c2；默认 30 天 + enabled: true # 注意：清理 payload 后对应下载链接会失效 + retention_days: 30 + chat_uploads: # 对话上传附件 chat_uploads；默认 90 天 + enabled: true + retention_days: 90 + workflow_checkpoints: # 工作流断点 data/workflow-checkpoints；默认 7 天 + enabled: true + retention_days: 7 + diagnostic_logs: # 诊断日志 log/diagnostic-*.log；默认 14 天 + enabled: true # 与 log.diagnostic_retention_days 各自独立生效，取先到者 + retention_days: 14 # ============================================ # 对话相关配置 # ============================================ diff --git a/docs/en-US/configuration.md b/docs/en-US/configuration.md index f49d01294..d3915885f 100644 --- a/docs/en-US/configuration.md +++ b/docs/en-US/configuration.md @@ -122,6 +122,38 @@ When debugging, inspect both the child config and the fallback parent. | `similarity_threshold` | 0.35-0.45 | 0.5+ | recall vs precision | | `audit.retention_days` | 15-30 | 90+ | compliance and disk | +## Runtime storage cleanup + +```yaml +storage: + auto_clean: false + interval_minutes: 60 + orphan_grace_days: 1 + active_grace_hours: 24 + categories: + workspace: + enabled: true + retention_days: 30 +``` + +Reclaims disk space used by runtime artifacts. Manage it from **System settings → Storage cleanup**, or trigger it with `POST /api/storage/cleanup`. + +- `auto_clean` is off by default: upgrading never deletes existing data behind an administrator's back. Manual preview and cleanup still work while it is off. +- `interval_minutes` is the background sweep interval (floor 5). `orphan_grace_days` is the minimum age before a directory whose conversation/project no longer exists is reclaimed. Sessions active within `active_grace_hours` are always skipped. +- A category `retention_days` of 0 disables age-based cleanup, but **orphaned directories are still reclaimed** — a directory whose session is gone has no retention value. +- Deletion is irreversible. The API requires `dry_run=false` together with `confirm=true` for a real cleanup; omitting `dry_run` is treated as a preview. + +| Category | Default retention | Target directory | +| --- | --- | --- | +| `workspace` | 30 days | agent workspaces, `tmp/workspace` | +| `reduction` | 7 days | oversized tool-output spill, `tmp/reduction` | +| `conversation_artifacts` | 30 days | summaries and user-input ledger, `data/conversation_artifacts` | +| `plantask` | 30 days | multi-agent plan boards, `skills/.eino/plantask` | +| `c2_artifacts` | 30 days | C2 results/uploads/downstream/payloads, `tmp/c2` (cleaning a payload invalidates its download link) | +| `chat_uploads` | 90 days | conversation attachments, `chat_uploads` | +| `workflow_checkpoints` | 7 days | workflow run checkpoints, `data/workflow-checkpoints` | +| `diagnostic_logs` | 14 days | diagnostic logs, `log/diagnostic-*.log` | + ## Change Template Before changing config, write down: diff --git a/docs/zh-CN/configuration.md b/docs/zh-CN/configuration.md index 0db87d4a4..300b3a699 100644 --- a/docs/zh-CN/configuration.md +++ b/docs/zh-CN/configuration.md @@ -215,6 +215,38 @@ monitor: - `audit` 记录平台操作，不记录对话正文和每次工具调用正文。 - `monitor` 管理工具执行记录保留时间。 +## 存储清理 + +```yaml +storage: + auto_clean: false + interval_minutes: 60 + orphan_grace_days: 1 + active_grace_hours: 24 + categories: + workspace: + enabled: true + retention_days: 30 +``` + +清理运行期间产生的磁盘垃圾，入口在「系统设置 → 存储清理」，也可用 `POST /api/storage/cleanup` 触发。 + +- `auto_clean` 默认关闭：升级后不会在管理员不知情的情况下删除既有数据；关闭时仍可手动预览与清理。 +- `interval_minutes` 后台清理间隔，最小 5；`orphan_grace_days` 是会话/项目已删除但目录残留时的最小保留天数；`active_grace_hours` 内的活动会话一律跳过。 +- 各类别 `retention_days` 为 0 表示不按保留期清理，但**孤儿目录仍会回收**（会话已删除的目录没有保留价值）。 +- 删除不可逆。API 层要求真实删除必须同时传 `dry_run=false` 与 `confirm=true`，省略 `dry_run` 按预览处理。 + +| 类别 | 默认保留 | 目标目录 | +| --- | --- | --- | +| `workspace` | 30 天 | Agent 工作区 `tmp/workspace` | +| `reduction` | 7 天 | 超长工具输出落盘 `tmp/reduction` | +| `conversation_artifacts` | 30 天 | 摘要与超长输入台账 `data/conversation_artifacts` | +| `plantask` | 30 天 | 多代理计划看板 `skills/.eino/plantask` | +| `c2_artifacts` | 30 天 | C2 回传/上传/下发/payload `tmp/c2`（清理 payload 后下载链接失效） | +| `chat_uploads` | 90 天 | 对话上传附件 `chat_uploads` | +| `workflow_checkpoints` | 7 天 | 工作流断点 `data/workflow-checkpoints` | +| `diagnostic_logs` | 14 天 | 诊断日志 `log/diagnostic-*.log` | + ## C2、WebShell、项目 ```yaml diff --git a/internal/app/app.go b/internal/app/app.go index 2ca1417e8..a294df855 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -33,6 +33,7 @@ import ( "cyberstrike-ai/internal/robot" "cyberstrike-ai/internal/security" "cyberstrike-ai/internal/skillpackage" + "cyberstrike-ai/internal/storage" "cyberstrike-ai/internal/toolguard" "github.com/gin-gonic/gin" @@ -72,6 +73,7 @@ type App struct { c2Watchdog *c2.SessionWatchdog // C2 会话看门狗 c2WatchdogCancel context.CancelFunc // 看门狗取消函数 c2Handler *handler.C2Handler // C2 REST（与 Manager 生命周期同步） + storageHandler *handler.StorageHandler // 运行空间占用统计与垃圾清理 auditSvc *audit.Service } @@ -367,6 +369,48 @@ func New(cfg *config.Config, log *logger.Logger, configPath string) (*App, error reductionRoot := strings.TrimSpace(cfg.MultiAgent.EinoMiddleware.ReductionRootDir) workspaceRoot := strings.TrimSpace(cfg.Agent.WorkspaceRootDir) db.SetEinoConversationDirs(plantaskBase, checkpointBase, reductionRoot, workspaceRoot) + + // 运行空间垃圾清理：根目录一律复用上面已解析好的同一批值， + // 避免在 storage 包内重新推导导致「清理的目录」与「实际写入的目录」不一致。 + workspaceRootDir := strings.TrimSpace(workspaceRoot) + if workspaceRootDir == "" { + workspaceRootDir = filepath.Join("tmp", "workspace") + } + reductionRootDir := strings.TrimSpace(reductionRoot) + if reductionRootDir == "" { + reductionRootDir = filepath.Join("tmp", "reduction") + } + diagnosticLogDir := strings.TrimSpace(cfg.Log.DiagnosticDir) + if diagnosticLogDir == "" { + diagnosticLogDir = "log" + } + // chat_uploads 与 tmp/c2 目前均为相对进程工作目录的固定路径 + // （见 handler.chatUploadsRootDirName 与 app/c2_lifecycle.go 的 c2.NewManager）。 + chatUploadsRoot := "chat_uploads" + c2Root := filepath.Join("tmp", "c2") + // 让 DeleteConversation 一并删除上传附件：其 chat_upload_artifacts 行已由 + // ON DELETE CASCADE 清除，此前磁盘文件会永久残留。 + db.SetChatUploadsDir(chatUploadsRoot) + storageCleaner := storage.NewCleaner(storage.Options{ + Config: cfg, + Paths: storage.Paths{ + Workspace: workspaceRootDir, + Reduction: reductionRootDir, + ConversationArtifact: db.ConversationArtifactsBaseDir(), + Plantask: plantaskBase, + C2: c2Root, + ChatUploads: chatUploadsRoot, + WorkflowCheckpoints: filepath.Join(filepath.Dir(dbPath), "workflow-checkpoints"), + DiagnosticLogs: diagnosticLogDir, + }, + Activity: db, + Logger: log.Logger, + }) + storageService := storage.NewService(storageCleaner, cfg, log.Logger) + storage.StartRetentionLoop(storageService, log.Logger) + storageHandler := handler.NewStorageHandler(storageCleaner, cfg, log.Logger) + storageHandler.SetAudit(auditSvc) + agent.SetPromptBaseDir(configDir) agentsDir := cfg.AgentsDir @@ -479,6 +523,7 @@ func New(cfg *config.Config, log *logger.Logger, configPath string) (*App, error c2Watchdog: c2Watchdog, c2WatchdogCancel: watchdogCancel, c2Handler: c2Handler, + storageHandler: storageHandler, auditSvc: auditSvc, } // 飞书/钉钉长连接（无需公网），启用时在后台启动；后续前端应用配置时会通过 RestartRobotConnections 重启 @@ -1085,6 +1130,11 @@ func setupRoutes( protected.GET("/audit/logs/export", auditHandler.ExportLogs) protected.GET("/audit/logs/:id", auditHandler.GetLog) + // 运行空间占用与垃圾清理 + protected.GET("/storage/meta", app.storageHandler.Meta) + protected.GET("/storage/status", app.storageHandler.Status) + protected.POST("/storage/cleanup", app.storageHandler.Cleanup) + // 外部MCP管理 protected.GET("/external-mcp", externalMCPHandler.GetExternalMCPs) protected.GET("/external-mcp/stats", externalMCPHandler.GetExternalMCPStats) diff --git a/internal/config/config.go b/internal/config/config.go index 4fb988a42..c74b842f5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -37,6 +37,7 @@ type Config struct { Auth AuthConfig `yaml:"auth"` Audit AuditConfig `yaml:"audit,omitempty" json:"audit,omitempty"` Monitor MonitorConfig `yaml:"monitor,omitempty" json:"monitor,omitempty"` + Storage StorageConfig `yaml:"storage,omitempty" json:"storage,omitempty"` ExternalMCP ExternalMCPConfig `yaml:"external_mcp,omitempty"` Knowledge KnowledgeConfig `yaml:"knowledge,omitempty"` C2 C2Config `yaml:"c2,omitempty" json:"c2,omitempty"` // 内置 C2 总开关；未配置时默认启用 @@ -1311,6 +1312,132 @@ func (m MonitorConfig) RetentionDaysEffective() int { return *m.RetentionDays } +// 运行空间垃圾清理的类别键。顺序即前端展示顺序，勿依赖 map 迭代顺序。 +const ( + StorageCategoryWorkspace = "workspace" + StorageCategoryReduction = "reduction" + StorageCategoryConversationArtifact = "conversation_artifacts" + StorageCategoryPlantask = "plantask" + StorageCategoryC2Artifacts = "c2_artifacts" + StorageCategoryChatUploads = "chat_uploads" + StorageCategoryWorkflowCheckpoints = "workflow_checkpoints" + StorageCategoryDiagnosticLogs = "diagnostic_logs" +) + +// StorageCategoryOrder 列出全部可清理类别，供 UI 与报表稳定排序。 +var StorageCategoryOrder = []string{ + StorageCategoryWorkspace, + StorageCategoryReduction, + StorageCategoryConversationArtifact, + StorageCategoryPlantask, + StorageCategoryC2Artifacts, + StorageCategoryChatUploads, + StorageCategoryWorkflowCheckpoints, + StorageCategoryDiagnosticLogs, +} + +// StorageCategoryDefaults 各类别默认保留天数。取较短值的是纯派生产物 +// （reduction/checkpoint），取较长值的是可能仍需人工回看的上传件。 +var StorageCategoryDefaults = map[string]int{ + StorageCategoryWorkspace: 30, + StorageCategoryReduction: 7, + StorageCategoryConversationArtifact: 30, + StorageCategoryPlantask: 30, + StorageCategoryC2Artifacts: 30, + StorageCategoryChatUploads: 90, + StorageCategoryWorkflowCheckpoints: 7, + StorageCategoryDiagnosticLogs: 14, +} + +// StorageCategoryConfig 单个清理类别的策略覆盖。 +type StorageCategoryConfig struct { + // Enabled 省略时默认 true；显式 false 表示该类别既不自动清理也不出现在手动清理范围内。 + Enabled *bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` + // RetentionDays 省略时使用 StorageCategoryDefaults；0 表示不按保留期清理（孤儿目录仍会回收）。 + RetentionDays *int `yaml:"retention_days,omitempty" json:"retention_days,omitempty"` +} + +// StorageConfig 运行空间垃圾清理策略。 +// 自动清理默认关闭：与 Argo ttlStrategy / K8s ttlSecondsAfterFinished 的 unset 语义一致， +// 升级后不会在管理员不知情的情况下删除既有数据。 +type StorageConfig struct { + // AutoClean 省略或 false 关闭后台自动清理；显式 true 才启用。 + AutoClean *bool `yaml:"auto_clean,omitempty" json:"auto_clean,omitempty"` + // IntervalMinutes 后台清理轮询间隔；省略默认 60，最小 5。 + IntervalMinutes *int `yaml:"interval_minutes,omitempty" json:"interval_minutes,omitempty"` + // OrphanGraceDays 会话/项目已删除但目录残留时的最小保留天数；省略默认 1。 + OrphanGraceDays *int `yaml:"orphan_grace_days,omitempty" json:"orphan_grace_days,omitempty"` + // ActiveGraceHours 最近有活动的会话一律跳过清理；省略默认 24。 + ActiveGraceHours *int `yaml:"active_grace_hours,omitempty" json:"active_grace_hours,omitempty"` + // Categories 按类别覆盖策略；未列出的类别使用内置默认值。 + Categories map[string]StorageCategoryConfig `yaml:"categories,omitempty" json:"categories,omitempty"` +} + +// AutoCleanEffective returns true only when storage.auto_clean is explicitly true. +func (s StorageConfig) AutoCleanEffective() bool { + return s.AutoClean != nil && *s.AutoClean +} + +// IntervalMinutesEffective returns the background sweep interval; defaults to 60, floor 5. +func (s StorageConfig) IntervalMinutesEffective() int { + if s.IntervalMinutes == nil { + return 60 + } + if *s.IntervalMinutes < 5 { + return 5 + } + return *s.IntervalMinutes +} + +// OrphanGraceDaysEffective returns the minimum age before an orphaned session dir is reclaimed; defaults to 1. +func (s StorageConfig) OrphanGraceDaysEffective() int { + if s.OrphanGraceDays == nil { + return 1 + } + if *s.OrphanGraceDays < 0 { + return 0 + } + return *s.OrphanGraceDays +} + +// ActiveGraceHoursEffective returns the recent-activity protection window; defaults to 24, floor 1. +func (s StorageConfig) ActiveGraceHoursEffective() int { + if s.ActiveGraceHours == nil { + return 24 + } + if *s.ActiveGraceHours < 1 { + return 1 + } + return *s.ActiveGraceHours +} + +// CategoryEnabled reports whether a category participates in cleanup; unknown keys default to false. +func (s StorageConfig) CategoryEnabled(key string) bool { + if _, known := StorageCategoryDefaults[key]; !known { + return false + } + if c, ok := s.Categories[key]; ok && c.Enabled != nil { + return *c.Enabled + } + return true +} + +// CategoryRetentionDays returns the effective retention for a category; unknown keys yield 0 (keep forever). +func (s StorageConfig) CategoryRetentionDays(key string) int { + def, known := StorageCategoryDefaults[key] + if !known { + return 0 + } + c, ok := s.Categories[key] + if !ok || c.RetentionDays == nil { + return def + } + if *c.RetentionDays < 0 { + return 0 + } + return *c.RetentionDays +} + // AuditConfig platform operation audit log settings (not chat/tool execution bodies). type AuditConfig struct { // Enabled nil or true enables persistence; explicit false disables. diff --git a/internal/database/conversation.go b/internal/database/conversation.go index 11e1fb4d4..0770c652d 100644 --- a/internal/database/conversation.go +++ b/internal/database/conversation.go @@ -835,6 +835,8 @@ func (db *DB) removeConversationScopedDirs(conversationID, projectID string) { db.removeConversationScopedDir(db.einoPlantaskBaseDir, conversationID, "plantask") // Eino ADK runner checkpoints (checkpoint_dir//). db.removeConversationScopedDir(db.einoCheckpointBaseDir, conversationID, "eino_checkpoint") + // 上传附件始终归属单个会话，项目绑定的会话也要删，故放在 projectID 判断之外。 + db.removeChatUploadDirs(conversationID) // Eino reduction persisted tool outputs (tmp/reduction/conversations//). // Project-bound sessions share projects// — skip on single conversation delete. if strings.TrimSpace(projectID) == "" { @@ -845,6 +847,33 @@ func (db *DB) removeConversationScopedDirs(conversationID, projectID string) { } } +// removeChatUploadDirs 删除 chat_uploads/<日期>/<会话ID>/ 下属于该会话的上传目录。 +// 该根目录比其他产物多一层日期目录，无法复用 removeConversationScopedDir。 +func (db *DB) removeChatUploadDirs(conversationID string) { + base := strings.TrimSpace(db.chatUploadsDir) + if base == "" || strings.TrimSpace(conversationID) == "" { + return + } + seg := sanitizeConversationPathSegment(conversationID) + dates, err := os.ReadDir(base) + if err != nil { + return + } + for _, dateDir := range dates { + if !dateDir.IsDir() { + continue + } + dir := filepath.Join(base, dateDir.Name(), seg) + if rmErr := os.RemoveAll(dir); rmErr != nil && db.logger != nil { + db.logger.Warn("删除会话上传目录失败", + zap.String("conversationId", conversationID), + zap.String("kind", "chat_uploads"), + zap.String("dir", dir), + zap.Error(rmErr)) + } + } +} + func (db *DB) removeProjectScopedDirs(projectID string) { // Eino reduction persisted tool outputs (tmp/reduction/projects//). reductionBase := filepath.Join(db.einoReductionBaseDir(), "projects") diff --git a/internal/database/conversation_cleanup_test.go b/internal/database/conversation_cleanup_test.go index a2bc384d2..abc58121b 100644 --- a/internal/database/conversation_cleanup_test.go +++ b/internal/database/conversation_cleanup_test.go @@ -60,6 +60,67 @@ func TestDeleteConversationRemovesEinoScopedDirs(t *testing.T) { } } +// chat_uploads 比其他产物多一层日期目录（chat_uploads/<日期>/<会话ID>）， +// 历史上删除会话只清了数据库行（ON DELETE CASCADE），磁盘文件会永久残留。 +func TestDeleteConversationRemovesChatUploads(t *testing.T) { + tmp := t.TempDir() + dbPath := filepath.Join(tmp, "conversations.db") + db, err := NewDB(dbPath, zap.NewNop()) + if err != nil { + t.Fatalf("NewDB: %v", err) + } + defer db.Close() + + uploads := filepath.Join(tmp, "chat_uploads") + db.SetChatUploadsDir(uploads) + + target, err := db.CreateConversation("uploads cleanup", ConversationCreateMeta{}) + if err != nil { + t.Fatalf("CreateConversation: %v", err) + } + sibling, err := db.CreateConversation("sibling", ConversationCreateMeta{}) + if err != nil { + t.Fatalf("CreateConversation: %v", err) + } + targetSeg := sanitizeConversationPathSegment(target.ID) + siblingSeg := sanitizeConversationPathSegment(sibling.ID) + + // 同一会话跨两个日期目录都有上传件，另一个会话的上传件必须保留。 + for _, dir := range []string{ + filepath.Join(uploads, "2026-06-01", targetSeg), + filepath.Join(uploads, "2026-06-02", targetSeg), + filepath.Join(uploads, "2026-06-01", siblingSeg), + } { + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", dir, err) + } + if err := os.WriteFile(filepath.Join(dir, "report.pdf"), []byte("x"), 0o644); err != nil { + t.Fatalf("write %s: %v", dir, err) + } + } + + if err := db.DeleteConversation(target.ID); err != nil { + t.Fatalf("DeleteConversation: %v", err) + } + + for _, date := range []string{"2026-06-01", "2026-06-02"} { + dir := filepath.Join(uploads, date, targetSeg) + if _, statErr := os.Stat(dir); !os.IsNotExist(statErr) { + t.Errorf("expected removed dir %s, stat err=%v", dir, statErr) + } + } + siblingDir := filepath.Join(uploads, "2026-06-01", siblingSeg) + if _, statErr := os.Stat(filepath.Join(siblingDir, "report.pdf")); statErr != nil { + t.Errorf("其他会话的上传件被误删: %v", statErr) + } + if _, statErr := os.Stat(uploads); statErr != nil { + t.Errorf("chat_uploads 根目录不应被删除: %v", statErr) + } + if _, statErr := os.Stat(filepath.Join(uploads, "2026-06-01")); statErr != nil { + t.Errorf("仍有数据的日期目录不应被删除: %v", statErr) + } +} + func TestDeleteProjectRemovesReductionDir(t *testing.T) { tmp := t.TempDir() dbPath := filepath.Join(tmp, "conversations.db") diff --git a/internal/database/database.go b/internal/database/database.go index d6bcc996f..ddab0b15e 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -53,6 +53,7 @@ type DB struct { einoCheckpointBaseDir string // checkpoint_dir root (per-conversation subdirs) einoReductionRootDir string // reduction_root_dir or default tmp/reduction (conversations/ subdirs) einoWorkspaceRootDir string // workspace_root_dir or default tmp/workspace (projects|conversations/ subdirs) + chatUploadsDir string // chat_uploads root (/ subdirs) checkpointLoopName string checkpointStop chan struct{} checkpointDone chan struct{} @@ -178,6 +179,16 @@ func (db *DB) SetEinoConversationDirs(plantaskBase, checkpointBase, reductionRoo db.einoWorkspaceRootDir = strings.TrimSpace(workspaceRoot) } +// SetChatUploadsDir configures the chat_uploads root so DeleteConversation can remove +// uploaded attachment files. Their chat_upload_artifacts rows already disappear via +// ON DELETE CASCADE; without this the files themselves would linger forever. +func (db *DB) SetChatUploadsDir(dir string) { + if db == nil { + return + } + db.chatUploadsDir = strings.TrimSpace(dir) +} + // initTables 初始化数据库表 func (db *DB) initTables() error { // 创建对话表（last_react_input / last_react_output 存「代理消息轨迹」JSON 与助手摘要，列名保留以兼容已有库） diff --git a/internal/database/rbac.go b/internal/database/rbac.go index 192bed4dd..17db5cfa6 100644 --- a/internal/database/rbac.go +++ b/internal/database/rbac.go @@ -371,7 +371,7 @@ func grantSystemRolePermissions(tx *sql.Tx, permissions map[string]string) error if _, err := tx.Exec(`INSERT OR IGNORE INTO rbac_role_permissions (role_id, permission_key, created_at) VALUES (?, ?, ?)`, RBACSystemRoleAuditor, key, now); err != nil { return err } - case strings.HasPrefix(key, "rbac:"), strings.HasPrefix(key, "config:"), strings.HasPrefix(key, "terminal:"), strings.HasPrefix(key, "audit:"): + case strings.HasPrefix(key, "rbac:"), strings.HasPrefix(key, "config:"), strings.HasPrefix(key, "terminal:"), strings.HasPrefix(key, "audit:"), strings.HasPrefix(key, "storage:"): continue case key == "mcp:write" || key == "mcp:external:execute": continue diff --git a/internal/database/storage_activity.go b/internal/database/storage_activity.go new file mode 100644 index 000000000..151dc9553 --- /dev/null +++ b/internal/database/storage_activity.go @@ -0,0 +1,54 @@ +package database + +import ( + "database/sql" + "strings" + "time" +) + +// ConversationLastActivity 返回会话最近活动时间；ok=false 表示会话已不存在。 +// 供存储清理判断目录是否为孤儿、以及会话是否仍在活跃使用。 +func (db *DB) ConversationLastActivity(id string) (time.Time, bool, error) { + id = strings.TrimSpace(id) + if id == "" { + return time.Time{}, false, nil + } + var createdAt, updatedAt string + err := db.QueryRow( + "SELECT created_at, updated_at FROM conversations WHERE id = ? LIMIT 1", id, + ).Scan(&createdAt, &updatedAt) + if err == sql.ErrNoRows { + return time.Time{}, false, nil + } + if err != nil { + return time.Time{}, false, err + } + created, updated := parseDBTime(createdAt), parseDBTime(updatedAt) + if created.After(updated) { + return created, true, nil + } + return updated, true, nil +} + +// ProjectLastActivity 返回项目最近活动时间；ok=false 表示项目已不存在。 +func (db *DB) ProjectLastActivity(id string) (time.Time, bool, error) { + id = strings.TrimSpace(id) + if id == "" { + return time.Time{}, false, nil + } + var createdAt, updatedAt string + err := db.QueryRow( + "SELECT created_at, updated_at FROM projects WHERE id = ? LIMIT 1", id, + ).Scan(&createdAt, &updatedAt) + if err == sql.ErrNoRows { + return time.Time{}, false, nil + } + if err != nil { + return time.Time{}, false, err + } + created, updated := parseDBTime(createdAt), parseDBTime(updatedAt) + if created.After(updated) { + return created, true, nil + } + return updated, true, nil +} diff --git a/internal/handler/config.go b/internal/handler/config.go index 61daa9292..31e90a3d7 100644 --- a/internal/handler/config.go +++ b/internal/handler/config.go @@ -731,6 +731,7 @@ type UpdateConfigRequest struct { Robots *config.RobotsConfig `json:"robots,omitempty"` MultiAgent *config.MultiAgentAPIUpdate `json:"multi_agent,omitempty"` C2 *config.C2APIUpdate `json:"c2,omitempty"` + Storage *config.StorageConfig `json:"storage,omitempty"` } // AgentConfigUpdate 用于 PATCH /api/config 的 agent 段：仅 JSON 中出现的字段（指针非 nil）覆盖内存配置。 @@ -916,6 +917,66 @@ func (h *ConfigHandler) UpdateConfig(c *gin.Context) { ) } + if req.Storage != nil { + st := &h.config.Storage + if req.Storage.AutoClean != nil { + v := *req.Storage.AutoClean + st.AutoClean = &v + } + if req.Storage.IntervalMinutes != nil { + v := *req.Storage.IntervalMinutes + if v < 5 { + v = 5 + } + st.IntervalMinutes = &v + } + if req.Storage.OrphanGraceDays != nil { + v := *req.Storage.OrphanGraceDays + if v < 0 { + v = 0 + } + st.OrphanGraceDays = &v + } + if req.Storage.ActiveGraceHours != nil { + v := *req.Storage.ActiveGraceHours + if v < 1 { + v = 1 + } + st.ActiveGraceHours = &v + } + if req.Storage.Categories != nil { + if st.Categories == nil { + st.Categories = make(map[string]config.StorageCategoryConfig, len(req.Storage.Categories)) + } + // 只接受注册表内的类别键，未注册的键直接忽略，避免被写进 config.yaml。 + for _, key := range config.StorageCategoryOrder { + patch, ok := req.Storage.Categories[key] + if !ok { + continue + } + cur := st.Categories[key] + if patch.Enabled != nil { + v := *patch.Enabled + cur.Enabled = &v + } + if patch.RetentionDays != nil { + v := *patch.RetentionDays + if v < 0 { + v = 0 + } + cur.RetentionDays = &v + } + st.Categories[key] = cur + } + } + h.logger.Info("更新运行空间清理配置", + zap.Bool("auto_clean", st.AutoCleanEffective()), + zap.Int("interval_minutes", st.IntervalMinutesEffective()), + zap.Int("orphan_grace_days", st.OrphanGraceDaysEffective()), + zap.Int("active_grace_hours", st.ActiveGraceHoursEffective()), + ) + } + // 更新Knowledge配置 if req.Knowledge != nil { // 保存旧的嵌入模型配置（用于检测变更） @@ -1780,6 +1841,7 @@ func (h *ConfigHandler) saveConfig() error { updateC2Config(root, h.config.C2) updateRobotsConfig(root, h.config.Robots) updateHitlConfig(root, h.config.Hitl) + updateStorageConfig(root, h.config.Storage) updateMultiAgentConfig(root, h.config.MultiAgent) // 更新外部MCP配置（使用external_mcp.go中的函数，同一包中可直接调用） updateExternalMCPConfig(root, h.config.ExternalMCP) @@ -2164,6 +2226,24 @@ func updateHitlConfig(doc *yaml.Node, cfg config.HitlConfig) { setStringInMap(hitlNode, "audit_agent_prompt_review_edit", cfg.AuditAgentPromptReviewEdit) } +// updateStorageConfig 把运行空间清理策略写回 config.yaml，保留文件其余内容与注释。 +func updateStorageConfig(doc *yaml.Node, cfg config.StorageConfig) { + root := doc.Content[0] + storageNode := ensureMap(root, "storage") + setBoolInMap(storageNode, "auto_clean", cfg.AutoCleanEffective()) + setIntInMap(storageNode, "interval_minutes", cfg.IntervalMinutesEffective()) + setIntInMap(storageNode, "orphan_grace_days", cfg.OrphanGraceDaysEffective()) + setIntInMap(storageNode, "active_grace_hours", cfg.ActiveGraceHoursEffective()) + + // 按固定顺序输出，避免每次保存都因 map 迭代顺序不同而重排整个文件。 + categoriesNode := ensureMap(storageNode, "categories") + for _, key := range config.StorageCategoryOrder { + categoryNode := ensureMap(categoriesNode, key) + setBoolInMap(categoryNode, "enabled", cfg.CategoryEnabled(key)) + setIntInMap(categoryNode, "retention_days", cfg.CategoryRetentionDays(key)) + } +} + // UpdateHitlDefaultConfig 更新全局默认人机协同配置并写入 config.yaml。 func (h *ConfigHandler) UpdateHitlDefaultConfig(mode, reviewer string, timeoutSeconds int) error { h.mu.Lock() diff --git a/internal/handler/openapi.go b/internal/handler/openapi.go index e5d03f711..894087474 100644 --- a/internal/handler/openapi.go +++ b/internal/handler/openapi.go @@ -3468,6 +3468,103 @@ func (h *OpenAPIHandler) GetOpenAPISpec(c *gin.Context) { }, }, }, + "/api/storage/meta": map[string]interface{}{ + "get": map[string]interface{}{ + "tags": []string{"存储清理"}, + "summary": "获取存储清理策略", + "description": "返回自动清理开关、间隔、宽限窗口与全部清理类别的启用状态及保留天数", + "operationId": "getStorageMeta", + "responses": map[string]interface{}{ + "200": map[string]interface{}{ + "description": "获取成功", + }, + "401": map[string]interface{}{ + "description": "未授权", + }, + "403": map[string]interface{}{ + "description": "缺少 storage:read 权限", + }, + }, + }, + }, + "/api/storage/status": map[string]interface{}{ + "get": map[string]interface{}{ + "tags": []string{"存储清理"}, + "summary": "获取运行空间占用", + "description": "返回文件系统容量（含 inode）与各类别的占用、可回收量；结果按短 TTL 缓存", + "operationId": "getStorageStatus", + "parameters": []interface{}{ + map[string]interface{}{ + "name": "refresh", + "in": "query", + "description": "为 true 时强制重新遍历目录，绕过缓存", + "required": false, + "schema": map[string]interface{}{"type": "boolean"}, + }, + }, + "responses": map[string]interface{}{ + "200": map[string]interface{}{ + "description": "获取成功", + }, + "401": map[string]interface{}{ + "description": "未授权", + }, + "403": map[string]interface{}{ + "description": "缺少 storage:read 权限", + }, + }, + }, + }, + "/api/storage/cleanup": map[string]interface{}{ + "post": map[string]interface{}{ + "tags": []string{"存储清理"}, + "summary": "预览或执行运行空间清理", + "description": "默认 dry_run=true 只统计不删除；真实删除必须同时传 dry_run=false 与 confirm=true。仅处理已启用的类别，同一时刻只允许一轮执行", + "operationId": "runStorageCleanup", + "requestBody": map[string]interface{}{ + "required": false, + "content": map[string]interface{}{ + "application/json": map[string]interface{}{ + "schema": map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "dry_run": map[string]interface{}{ + "type": "boolean", + "description": "省略时按 true 处理", + }, + "confirm": map[string]interface{}{ + "type": "boolean", + "description": "dry_run=false 时必须为 true", + }, + "categories": map[string]interface{}{ + "type": "array", + "description": "限定类别；省略表示全部已启用类别", + "items": map[string]interface{}{"type": "string"}, + }, + }, + }, + }, + }, + }, + "responses": map[string]interface{}{ + "200": map[string]interface{}{ + "description": "执行完成（或预览完成）", + }, + "400": map[string]interface{}{ + "description": "缺少 confirm、或类别键未注册", + }, + "401": map[string]interface{}{ + "description": "未授权", + }, + "403": map[string]interface{}{ + "description": "缺少 storage:write 权限", + }, + "409": map[string]interface{}{ + "description": "已有一轮清理在执行", + }, + }, + }, + }, "/api/config/tools": map[string]interface{}{ "get": map[string]interface{}{ "tags": []string{"配置管理"}, diff --git a/internal/handler/storage.go b/internal/handler/storage.go new file mode 100644 index 000000000..80064e57e --- /dev/null +++ b/internal/handler/storage.go @@ -0,0 +1,156 @@ +package handler + +import ( + "errors" + "net/http" + "strconv" + + "cyberstrike-ai/internal/audit" + "cyberstrike-ai/internal/config" + "cyberstrike-ai/internal/storage" + + "github.com/gin-gonic/gin" + "go.uber.org/zap" +) + +// StorageHandler 提供运行空间占用统计与垃圾清理 API。 +type StorageHandler struct { + cleaner *storage.Cleaner + cfg *config.Config + audit *audit.Service + logger *zap.Logger +} + +// NewStorageHandler 创建存储清理 handler。 +func NewStorageHandler(cleaner *storage.Cleaner, cfg *config.Config, logger *zap.Logger) *StorageHandler { + return &StorageHandler{cleaner: cleaner, cfg: cfg, logger: logger} +} + +// SetAudit wires platform audit logging. +func (h *StorageHandler) SetAudit(s *audit.Service) { + if h != nil { + h.audit = s + } +} + +// storageCleanupRequest 是 POST /api/storage/cleanup 的请求体。 +type storageCleanupRequest struct { + // DryRun 省略时按 true 处理：只统计不删除。真正删除必须显式传 false。 + DryRun *bool `json:"dry_run"` + // Confirm 为 false 时即使 dry_run=false 也拒绝执行。 + // 磁盘删除不可逆，确认必须是 API 层的显式动作，而不只依赖前端弹窗。 + Confirm bool `json:"confirm"` + Categories []string `json:"categories"` +} + +// Meta GET /api/storage/meta 返回清理策略与各类别元信息。 +func (h *StorageHandler) Meta(c *gin.Context) { + st := h.effectiveConfig() + items := make([]gin.H, 0, len(config.StorageCategoryOrder)) + for _, info := range storage.DescribeCategories() { + items = append(items, gin.H{ + "key": info.Key, + "label": info.Label, + "hint": info.Hint, + "enabled": st.CategoryEnabled(info.Key), + "retention_days": st.CategoryRetentionDays(info.Key), + "default_retention": config.StorageCategoryDefaults[info.Key], + }) + } + c.JSON(http.StatusOK, gin.H{ + "auto_clean": st.AutoCleanEffective(), + "interval_minutes": st.IntervalMinutesEffective(), + "orphan_grace_days": st.OrphanGraceDaysEffective(), + "active_grace_hours": st.ActiveGraceHoursEffective(), + "categories": items, + }) +} + +// Status GET /api/storage/status 返回文件系统容量与各类别占用/可回收量。 +// ?refresh=1 强制重新遍历目录，否则使用短 TTL 缓存。 +func (h *StorageHandler) Status(c *gin.Context) { + if h.cleaner == nil { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": "存储清理未初始化"}) + return + } + refresh, _ := strconv.ParseBool(c.Query("refresh")) + rep := h.cleaner.Inspect(refresh) + c.JSON(http.StatusOK, gin.H{ + "filesystem": rep.Filesystem, + "categories": rep.Categories, + "totals": rep.Totals, + "scanned_at": rep.StartedAt, + "duration_ms": rep.DurationMS, + }) +} + +// Cleanup POST /api/storage/cleanup 执行清理（或预览）。 +func (h *StorageHandler) Cleanup(c *gin.Context) { + if h.cleaner == nil { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": "存储清理未初始化"}) + return + } + var req storageCleanupRequest + if c.Request.ContentLength > 0 { + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "无效的请求参数: " + err.Error()}) + return + } + } + dryRun := req.DryRun == nil || *req.DryRun + if !dryRun && !req.Confirm { + c.JSON(http.StatusBadRequest, gin.H{"error": "删除不可逆，执行真实清理必须同时传 dry_run=false 与 confirm=true"}) + return + } + + rep, err := h.cleaner.Clean(storage.CleanRequest{ + DryRun: dryRun, + Categories: req.Categories, + Trigger: "manual", + }) + switch { + case errors.Is(err, storage.ErrCleanupInProgress): + c.JSON(http.StatusConflict, gin.H{"error": err.Error()}) + return + case errors.Is(err, storage.ErrUnknownCategory): + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + case err != nil: + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + if !dryRun { + h.recordCleanup(c, rep) + } + c.JSON(http.StatusOK, gin.H{ + "dry_run": rep.DryRun, + "filesystem": rep.Filesystem, + "categories": rep.Categories, + "totals": rep.Totals, + "duration_ms": rep.DurationMS, + }) +} + +func (h *StorageHandler) recordCleanup(c *gin.Context, rep *storage.Report) { + if h.audit == nil { + return + } + detail := map[string]interface{}{ + "removed_units": rep.Totals.RemovedUnits, + "freed_bytes": rep.Totals.FreedBytes, + "errors": rep.Totals.Errors, + } + if rep.Totals.Errors > 0 { + h.audit.RecordFail(c, "storage", "cleanup", "运行空间清理完成但存在失败项", detail) + return + } + h.audit.RecordOK(c, "storage", "cleanup", "清理运行空间垃圾", "storage", "", detail) +} + +func (h *StorageHandler) effectiveConfig() config.StorageConfig { + if h.cfg == nil { + return config.StorageConfig{} + } + return h.cfg.Storage +} diff --git a/internal/security/rbac.go b/internal/security/rbac.go index 70158e513..9c8fa8d75 100644 --- a/internal/security/rbac.go +++ b/internal/security/rbac.go @@ -62,6 +62,8 @@ var PermissionCatalog = map[string]string{ "workflow:delete": "Delete workflows", "config:read": "View system configuration", "config:write": "Update and apply system configuration", + "storage:read": "View runtime storage usage and reclaimable space", + "storage:write": "Run runtime storage cleanup and change its retention policy", "terminal:execute": "Run terminal commands", "audit:read": "View and export audit logs", "audit:delete": "Delete audit logs", diff --git a/internal/security/rbac_middleware.go b/internal/security/rbac_middleware.go index 454953d7b..a72dfc822 100644 --- a/internal/security/rbac_middleware.go +++ b/internal/security/rbac_middleware.go @@ -135,6 +135,8 @@ func permissionForRequest(method, fullPath string) string { return crudPermission(method, "config") case strings.HasPrefix(path, "/config"): return crudPermission(method, "config") + case strings.HasPrefix(path, "/storage"): + return crudPermission(method, "storage") case strings.HasPrefix(path, "/terminal"): return "terminal:execute" case strings.HasPrefix(path, "/audit"): diff --git a/internal/storage/category.go b/internal/storage/category.go new file mode 100644 index 000000000..b294f51ee --- /dev/null +++ b/internal/storage/category.go @@ -0,0 +1,419 @@ +// Package storage 负责枚举、评估并清理运行空间产生的磁盘垃圾。 +// +// 设计约束（误删活跃任务数据的代价远高于省下磁盘，故以下均为硬约束）： +// - 每个类别只在固定根目录下的固定层级枚举删除单元，不做无界递归删除； +// - 删除单元必须通过 confined 校验，杜绝路径逃逸； +// - 目录清理采用「原子改名 + RemoveAll」，进程崩溃只留下带标记的残骸，下一轮补删； +// - 会话型单元在删除前查询数据库最近活动时间，活跃会话一律跳过；查询失败时保守跳过（fail closed）。 +package storage + +import ( + "os" + "path/filepath" + "strings" + "time" + + "cyberstrike-ai/internal/config" +) + +// deletionMarkerSuffix 标记「已判定删除、正在移除」的目录。 +// 沿用 Prometheus TSDB 的做法：改名是原子的，崩溃后不会留下半删状态的原始目录名。 +const deletionMarkerSuffix = ".tmp-for-deletion" + +// Scope 描述删除单元与会话/项目的绑定关系。 +type Scope int + +const ( + // ScopeNone 与会话无关，仅按保留期判定（日志、checkpoint、C2 产物）。 + ScopeNone Scope = iota + // ScopeConversation 目录名即会话 ID，可查询会话是否仍存在及最近活动时间。 + ScopeConversation + // ScopeProject 目录名即项目 ID。 + ScopeProject +) + +// Paths 汇总各类别根目录。 +// 由 app 层按既有解析规则注入（与 database.SetEinoConversationDirs 使用同一批值）， +// 避免本包重复推导路径导致清理目录与实际写入目录不一致。 +type Paths struct { + Workspace string + Reduction string + ConversationArtifact string + Plantask string + C2 string + ChatUploads string + WorkflowCheckpoints string + DiagnosticLogs string +} + +// Unit 是一个可独立删除的清理单元（目录或文件）。 +type Unit struct { + Path string + Session string // Scope 非 ScopeNone 时为对应 ID，否则为空 + Scope Scope + IsDir bool + ModTime time.Time + Size int64 +} + +// Age 返回单元相对 now 的闲置时长；未来时间戳（时钟异常）按 0 处理。 +func (u Unit) Age(now time.Time) time.Duration { + if u.ModTime.IsZero() { + return 0 + } + if age := now.Sub(u.ModTime); age > 0 { + return age + } + return 0 +} + +// scanner 枚举某个根目录下的全部删除单元。 +type scanner func(root string) ([]Unit, error) + +// category 是一个具名清理任务，对应 Gitea 的 [cron.*] 子任务模型。 +type category struct { + key string + label string + hint string + root func(Paths) string + scan scanner + pruneEmpty bool // 删除单元后回收空掉的父目录（chat_uploads 的日期层） +} + +// CategoryInfo 是一个清理类别的静态元信息，供 API 与前端展示。 +type CategoryInfo struct { + Key string `json:"key"` + Label string `json:"label"` + Hint string `json:"hint"` +} + +// DescribeCategories 按注册表顺序返回全部类别元信息。 +func DescribeCategories() []CategoryInfo { + all := categories() + out := make([]CategoryInfo, 0, len(all)) + for _, cat := range all { + out = append(out, CategoryInfo{Key: cat.key, Label: cat.label, Hint: cat.hint}) + } + return out +} + +// categories 按固定顺序返回全部类别；root 为空的类别会被调用方跳过。 +func categories() []category { + return []category{ + { + key: config.StorageCategoryWorkspace, + label: "Agent 工作区", + hint: "Agent 下载与扫描产物的工作目录（tmp/workspace），按项目/会话分目录。", + root: func(p Paths) string { return p.Workspace }, + scan: scanScopedDirs, + }, + { + key: config.StorageCategoryReduction, + label: "工具输出缓存", + hint: "超长工具输出落盘的截断文件（tmp/reduction），每条执行一个文件，属纯派生数据。", + root: func(p Paths) string { return p.Reduction }, + scan: scanScopedDirs, + }, + { + key: config.StorageCategoryConversationArtifact, + label: "会话产物", + hint: "摘要记录与超长用户输入台账（data/conversation_artifacts）。", + root: func(p Paths) string { return p.ConversationArtifact }, + scan: func(root string) ([]Unit, error) { return scanSessionDirs(root, ScopeConversation) }, + }, + { + key: config.StorageCategoryPlantask, + label: "计划任务看板", + hint: "Eino 多代理计划看板 JSON（skills/.eino/plantask），属纯派生数据。", + root: func(p Paths) string { return p.Plantask }, + scan: func(root string) ([]Unit, error) { return scanSessionDirs(root, ScopeConversation) }, + }, + { + key: config.StorageCategoryC2Artifacts, + label: "C2 产物", + hint: "C2 回传截图、上传件、下发文件与已生成的 payload 二进制（tmp/c2）。清理后对应 payload 下载链接会失效。", + root: func(p Paths) string { return p.C2 }, + scan: func(root string) ([]Unit, error) { + return scanSubdirFiles(root, "results", "uploads", "downstream", "payloads") + }, + }, + { + key: config.StorageCategoryChatUploads, + label: "对话上传文件", + hint: "用户在对话中上传的附件（chat_uploads/日期/会话）。", + root: func(p Paths) string { return p.ChatUploads }, + scan: scanDatedSessionDirs, + pruneEmpty: true, + }, + { + key: config.StorageCategoryWorkflowCheckpoints, + label: "工作流检查点", + hint: "工作流运行断点文件（data/workflow-checkpoints），仅用于恢复中断的运行。", + root: func(p Paths) string { return p.WorkflowCheckpoints }, + scan: func(root string) ([]Unit, error) { + return scanPatternFiles(root, "*.ckpt", "*.ckpt.tmp") + }, + }, + { + key: config.StorageCategoryDiagnosticLogs, + label: "诊断日志", + hint: "按天轮转的诊断日志（log/diagnostic-*.log）。", + root: func(p Paths) string { return p.DiagnosticLogs }, + scan: func(root string) ([]Unit, error) { + return scanPatternFiles(root, "diagnostic-*.log") + }, + }, + } +} + +// chatUploadsPlaceholderConvs 是上传时还没有会话 ID 的占位目录名， +// 不能当作会话 ID 去数据库查询，按非会话型处理（仅按保留期判定）。 +var chatUploadsPlaceholderConvs = map[string]bool{"_new": true, "_manual": true} + +// scanScopedDirs 枚举 root/projects/ 与 root/conversations/。 +func scanScopedDirs(root string) ([]Unit, error) { + scopes := []struct { + dir string + scope Scope + }{{"projects", ScopeProject}, {"conversations", ScopeConversation}} + + var units []Unit + for _, s := range scopes { + base := filepath.Join(root, s.dir) + entries, err := os.ReadDir(base) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, err + } + for _, e := range entries { + if !e.IsDir() { + continue + } + u, err := statUnit(filepath.Join(base, e.Name()), e.Name(), s.scope) + if err != nil { + continue + } + units = append(units, u) + } + } + return units, nil +} + +// scanSessionDirs 枚举 root/，目录名即会话 ID。 +func scanSessionDirs(root string, scope Scope) ([]Unit, error) { + entries, err := os.ReadDir(root) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + var units []Unit + for _, e := range entries { + if !e.IsDir() { + continue + } + u, err := statUnit(filepath.Join(root, e.Name()), e.Name(), scope) + if err != nil { + continue + } + units = append(units, u) + } + return units, nil +} + +// scanDatedSessionDirs 枚举 root//<会话ID|占位名>。 +func scanDatedSessionDirs(root string) ([]Unit, error) { + dates, err := os.ReadDir(root) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + var units []Unit + for _, d := range dates { + if !d.IsDir() { + continue + } + dateDir := filepath.Join(root, d.Name()) + convs, err := os.ReadDir(dateDir) + if err != nil { + continue + } + for _, cd := range convs { + if !cd.IsDir() { + // 直接落在日期目录下的散文件：按非会话型文件清理。 + if u, err := statUnit(filepath.Join(dateDir, cd.Name()), "", ScopeNone); err == nil { + units = append(units, u) + } + continue + } + session, scope := cd.Name(), ScopeConversation + if chatUploadsPlaceholderConvs[session] { + session, scope = "", ScopeNone + } + if u, err := statUnit(filepath.Join(dateDir, cd.Name()), session, scope); err == nil { + units = append(units, u) + } + } + } + return units, nil +} + +// scanSubdirFiles 枚举 root// 下的文件（不递归），用于 C2 产物。 +func scanSubdirFiles(root string, subdirs ...string) ([]Unit, error) { + var units []Unit + for _, sub := range subdirs { + base := filepath.Join(root, sub) + entries, err := os.ReadDir(base) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, err + } + for _, e := range entries { + if e.IsDir() { + continue + } + if u, err := statUnit(filepath.Join(base, e.Name()), "", ScopeNone); err == nil { + units = append(units, u) + } + } + } + return units, nil +} + +// scanPatternFiles 枚举 root 下匹配任一 glob 的文件。 +func scanPatternFiles(root string, patterns ...string) ([]Unit, error) { + entries, err := os.ReadDir(root) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + var units []Unit + for _, e := range entries { + if e.IsDir() { + continue + } + matched := false + for _, p := range patterns { + if ok, _ := filepath.Match(p, e.Name()); ok { + matched = true + break + } + } + if !matched { + continue + } + if u, err := statUnit(filepath.Join(root, e.Name()), "", ScopeNone); err == nil { + units = append(units, u) + } + } + return units, nil +} + +// statUnit 采集单元的大小与最近修改时间。 +// 使用 Lstat：符号链接只统计链接自身，绝不跟随到目标（否则会把工作目录外的数据算进来甚至删掉）。 +func statUnit(path, session string, scope Scope) (Unit, error) { + info, err := os.Lstat(path) + if err != nil { + return Unit{}, err + } + u := Unit{ + Path: path, + Session: session, + Scope: scope, + IsDir: info.IsDir(), + ModTime: info.ModTime(), + Size: info.Size(), + } + if u.IsDir { + size, newest := dirStats(path) + u.Size = size + if newest.After(u.ModTime) { + u.ModTime = newest + } + } + if scope == ScopeNone { + u.Session = "" + } + return u, nil +} + +// dirStats 汇总目录占用字节与其中最新的修改时间。 +// filepath.WalkDir 不跟随符号链接，且单个子树不可读时按尽力而为跳过，不中断整体统计。 +func dirStats(root string) (int64, time.Time) { + var size int64 + var newest time.Time + _ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil || d == nil { + return nil + } + info, infoErr := d.Info() + if infoErr != nil { + return nil + } + if !d.IsDir() { + size += info.Size() + } + if mt := info.ModTime(); mt.After(newest) { + newest = mt + } + return nil + }) + return size, newest +} + +// confined 校验 candidate 确实位于 root 之内，防止任何形式的路径逃逸。 +func confined(root, candidate string) bool { + if root == "" || candidate == "" { + return false + } + rel, err := filepath.Rel(root, candidate) + if err != nil { + return false + } + if rel == "." || rel == ".." || filepath.IsAbs(rel) { + return false + } + return !strings.HasPrefix(rel, ".."+string(filepath.Separator)) +} + +// pruneEmptyDirs 自底向上回收空目录，最多下探 depth 层。 +// chat_uploads 删完会话目录后会留下空的日期目录，需要一并回收。 +// 只删除「确实为空」的子孙目录，root 自身永不删除。 +func pruneEmptyDirs(root string, depth int) int { + if depth <= 0 { + return 0 + } + entries, err := os.ReadDir(root) + if err != nil { + return 0 + } + removed := 0 + for _, e := range entries { + if !e.IsDir() { + continue + } + sub := filepath.Join(root, e.Name()) + removed += pruneEmptyDirs(sub, depth-1) + if isEmptyDir(sub) && os.Remove(sub) == nil { + removed++ + } + } + return removed +} + +func isEmptyDir(path string) bool { + entries, err := os.ReadDir(path) + if err != nil { + return false + } + return len(entries) == 0 +} diff --git a/internal/storage/cleaner.go b/internal/storage/cleaner.go new file mode 100644 index 000000000..b3bb6bb1c --- /dev/null +++ b/internal/storage/cleaner.go @@ -0,0 +1,528 @@ +package storage + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "time" + + "cyberstrike-ai/internal/config" + + "go.uber.org/zap" +) + +// ErrCleanupInProgress 表示已有一轮清理在执行；handler 应映射为 409。 +var ErrCleanupInProgress = errors.New("已有一轮存储清理正在执行") + +// ErrUnknownCategory 表示请求里带了未注册的类别键。 +var ErrUnknownCategory = errors.New("未知的清理类别") + +// 判定结果原因，用于报表与日志可解释性。 +const ( + reasonExpired = "expired" // 超过保留期 + reasonOrphan = "orphan" // 会话/项目已删除，目录残留 + reasonLeftover = "leftover" // 上一轮清理崩溃残留的标记目录 + reasonActive = "active" // 最近有活动，受保护 + reasonRecent = "recent" // 尚未达到宽限期 + reasonKept = "kept" // 未到期或该类别保留期为 0 + reasonUnknown = "unknown" // 活跃状态查询失败，保守跳过 +) + +// Activity 查询会话/项目最近活动时间。实现方出错时清理会保守跳过该单元。 +type Activity interface { + ConversationLastActivity(id string) (time.Time, bool, error) + ProjectLastActivity(id string) (time.Time, bool, error) +} + +// CleanRequest 描述一次清理请求。 +type CleanRequest struct { + // DryRun 为 true 时只统计不删除。 + DryRun bool `json:"dry_run"` + // Categories 为空表示全部已启用类别。 + Categories []string `json:"categories"` + // Trigger 取值 manual / schedule，仅用于审计与日志。 + Trigger string `json:"trigger"` +} + +// CategoryReport 是单个类别的统计与执行结果。 +type CategoryReport struct { + Key string `json:"key"` + Label string `json:"label"` + Hint string `json:"hint"` + Root string `json:"root"` + Enabled bool `json:"enabled"` + RetentionDays int `json:"retention_days"` + // Missing 表示根目录尚未创建（系统还没产生过该类垃圾）。 + Missing bool `json:"missing"` + + Units int `json:"units"` + Bytes int64 `json:"bytes"` + + ReclaimableUnits int `json:"reclaimable_units"` + ReclaimableBytes int64 `json:"reclaimable_bytes"` + OrphanUnits int `json:"orphan_units"` + SkippedActive int `json:"skipped_active"` + SkippedUnsafe int `json:"skipped_unsafe"` + + RemovedUnits int `json:"removed_units"` + FreedBytes int64 `json:"freed_bytes"` + RemovedEmptyDirs int `json:"removed_empty_dirs"` + + Errors []string `json:"errors,omitempty"` +} + +// Totals 是全部类别的汇总。 +type Totals struct { + Units int `json:"units"` + Bytes int64 `json:"bytes"` + ReclaimableUnits int `json:"reclaimable_units"` + ReclaimableBytes int64 `json:"reclaimable_bytes"` + RemovedUnits int `json:"removed_units"` + FreedBytes int64 `json:"freed_bytes"` + SkippedActive int `json:"skipped_active"` + Errors int `json:"errors"` +} + +// Report 是一次统计或清理的完整结果。 +type Report struct { + DryRun bool `json:"dry_run"` + Trigger string `json:"trigger"` + StartedAt time.Time `json:"started_at"` + DurationMS int64 `json:"duration_ms"` + Filesystem Filesystem `json:"filesystem"` + Categories []CategoryReport `json:"categories"` + Totals Totals `json:"totals"` + Note string `json:"note,omitempty"` +} + +// defaultCacheTTL 限制 status 接口的目录遍历频率：大工作区下一次全量 walk 可能耗时数秒。 +const defaultCacheTTL = time.Minute + +// Options 构造 Cleaner 所需依赖。 +type Options struct { + Config *config.Config + Paths Paths + Activity Activity + Logger *zap.Logger + // Now 便于测试注入固定时钟；省略时使用 time.Now。 + Now func() time.Time + // CacheTTL 省略时使用 defaultCacheTTL；<=0 表示禁用缓存。 + CacheTTL time.Duration +} + +// Cleaner 枚举、评估并删除运行空间垃圾。 +type Cleaner struct { + cfg *config.Config + paths Paths + activity Activity + logger *zap.Logger + now func() time.Time + cacheTTL time.Duration + + // running 保证同一时刻只有一轮清理，避免两个管理员同时点「立即清理」互相踩。 + running atomic.Bool + + mu sync.Mutex + cached *Report + cachedAt time.Time +} + +// NewCleaner 创建清理器。cfg 为 nil 时使用零值配置（等价于全部默认策略）。 +func NewCleaner(opts Options) *Cleaner { + c := &Cleaner{ + cfg: opts.Config, + paths: opts.Paths, + activity: opts.Activity, + logger: opts.Logger, + now: opts.Now, + cacheTTL: opts.CacheTTL, + } + if c.cfg == nil { + c.cfg = &config.Config{} + } + if c.now == nil { + c.now = time.Now + } + if opts.CacheTTL == 0 { + c.cacheTTL = defaultCacheTTL + } else if opts.CacheTTL < 0 { + c.cacheTTL = 0 + } + return c +} + +// storageConfig 返回当前生效的存储策略（读取时取值，因此 PUT /api/config 后即时生效）。 +func (c *Cleaner) storageConfig() config.StorageConfig { + if c.cfg == nil { + return config.StorageConfig{} + } + return c.cfg.Storage +} + +// rootOf 返回类别根目录的绝对路径；未配置时返回空串。 +func (c *Cleaner) rootOf(cat category) string { + root := strings.TrimSpace(cat.root(c.paths)) + if root == "" { + return "" + } + abs, err := filepath.Abs(root) + if err != nil { + return "" + } + return filepath.Clean(abs) +} + +// selectCategories 按注册表顺序返回待处理类别，保证报表顺序稳定。 +// onlyEnabled 为 true 时跳过被显式关闭的类别。 +func (c *Cleaner) selectCategories(keys []string, onlyEnabled bool) ([]category, error) { + all := categories() + if len(keys) == 0 { + out := make([]category, 0, len(all)) + for _, cat := range all { + if onlyEnabled && !c.storageConfig().CategoryEnabled(cat.key) { + continue + } + out = append(out, cat) + } + return out, nil + } + + wanted := make(map[string]bool, len(keys)) + for _, k := range keys { + k = strings.TrimSpace(k) + if k == "" { + continue + } + wanted[k] = true + } + known := make(map[string]bool, len(all)) + for _, cat := range all { + known[cat.key] = true + } + for k := range wanted { + if !known[k] { + return nil, fmt.Errorf("%w: %s", ErrUnknownCategory, k) + } + } + out := make([]category, 0, len(wanted)) + for _, cat := range all { + if !wanted[cat.key] { + continue + } + if onlyEnabled && !c.storageConfig().CategoryEnabled(cat.key) { + continue + } + out = append(out, cat) + } + return out, nil +} + +// scanResult 是一次类别扫描的产物：报表 + 可删除单元。 +type scanResult struct { + report CategoryReport + eligible []Unit +} + +// scan 枚举类别下全部单元并逐个判定，同时产出统计与可删除清单。 +func (c *Cleaner) scan(cat category, now time.Time) scanResult { + st := c.storageConfig() + rep := CategoryReport{ + Key: cat.key, + Label: cat.label, + Hint: cat.hint, + Root: c.rootOf(cat), + Enabled: st.CategoryEnabled(cat.key), + RetentionDays: st.CategoryRetentionDays(cat.key), + } + res := scanResult{report: rep} + if rep.Root == "" { + rep.Missing = true + res.report = rep + return res + } + if _, err := os.Stat(rep.Root); err != nil { + rep.Missing = true + res.report = rep + return res + } + + units, err := cat.scan(rep.Root) + if err != nil { + rep.Errors = append(rep.Errors, err.Error()) + res.report = rep + return res + } + + for _, u := range units { + rep.Units++ + rep.Bytes += u.Size + // 纵深防御：scanner 只会产出 root 之下的路径，此处再断言一次。 + if !confined(rep.Root, u.Path) { + rep.SkippedUnsafe++ + continue + } + d := c.evaluate(u, rep.RetentionDays, now) + if !d.eligible { + if d.reason == reasonActive || d.reason == reasonUnknown { + rep.SkippedActive++ + } + continue + } + rep.ReclaimableUnits++ + rep.ReclaimableBytes += u.Size + if d.reason == reasonOrphan { + rep.OrphanUnits++ + } + res.eligible = append(res.eligible, u) + } + res.report = rep + return res +} + +// decision 是单个单元的判定结果。 +type decision struct { + eligible bool + reason string +} + +// evaluate 判定单元是否可删除。 +// 优先级：崩溃残留标记 > 最近活动保护 > 会话存活状态 > 保留期。 +func (c *Cleaner) evaluate(u Unit, retentionDays int, now time.Time) decision { + // 上一轮清理中途崩溃留下的标记目录：无条件补删。 + // 必须放在最前面，否则它的 mtime 是刚刚改名的时间，会被活跃保护永久挡住。 + if strings.HasSuffix(filepath.Base(u.Path), deletionMarkerSuffix) { + return decision{eligible: true, reason: reasonLeftover} + } + + st := c.storageConfig() + activeGrace := time.Duration(st.ActiveGraceHoursEffective()) * time.Hour + age := u.Age(now) + if age < activeGrace { + return decision{reason: reasonActive} + } + + retention := time.Duration(retentionDays) * 24 * time.Hour + + if u.Scope != ScopeNone && u.Session != "" && c.activity != nil { + last, exists, err := c.lastActivity(u) + if err != nil { + // 查不到活跃状态时保守跳过：宁可少删，不可误删正在跑的任务数据。 + return decision{reason: reasonUnknown} + } + if exists { + if now.Sub(last) < activeGrace { + return decision{reason: reasonActive} + } + if retentionDays > 0 && age >= retention { + return decision{eligible: true, reason: reasonExpired} + } + return decision{reason: reasonKept} + } + // 会话/项目已不存在 → 孤儿目录，按较短的宽限期回收。 + if age >= time.Duration(st.OrphanGraceDaysEffective())*24*time.Hour { + return decision{eligible: true, reason: reasonOrphan} + } + return decision{reason: reasonRecent} + } + + // retention_days: 0 表示不按保留期清理（沿用本项目既有约定）。 + if retentionDays <= 0 { + return decision{reason: reasonKept} + } + if age >= retention { + return decision{eligible: true, reason: reasonExpired} + } + return decision{reason: reasonKept} +} + +func (c *Cleaner) lastActivity(u Unit) (time.Time, bool, error) { + switch u.Scope { + case ScopeConversation: + return c.activity.ConversationLastActivity(u.Session) + case ScopeProject: + return c.activity.ProjectLastActivity(u.Session) + default: + return time.Time{}, false, nil + } +} + +// Inspect 统计全部类别的占用与可回收量，不删除任何文件。 +// 结果按 CacheTTL 缓存，refresh 为 true 时强制重算。 +func (c *Cleaner) Inspect(refresh bool) *Report { + c.mu.Lock() + if !refresh && c.cacheTTL > 0 && c.cached != nil && c.now().Sub(c.cachedAt) < c.cacheTTL { + cached := c.cached + c.mu.Unlock() + return cached + } + c.mu.Unlock() + + rep := c.buildReport(CleanRequest{DryRun: true, Trigger: "inspect"}, true) + + c.mu.Lock() + c.cached = rep + c.cachedAt = c.now() + c.mu.Unlock() + return rep +} + +// invalidateCache 让下一次 Inspect 重新遍历。 +func (c *Cleaner) invalidateCache() { + c.mu.Lock() + c.cached = nil + c.mu.Unlock() +} + +// Clean 执行一次清理；DryRun 为 true 时只统计。 +// 只处理已启用的类别，且同一时刻只允许一轮执行。 +func (c *Cleaner) Clean(req CleanRequest) (*Report, error) { + if !c.running.CompareAndSwap(false, true) { + return nil, ErrCleanupInProgress + } + defer c.running.Store(false) + + req.Trigger = strings.TrimSpace(req.Trigger) + if req.Trigger == "" { + req.Trigger = "manual" + } + if _, err := c.selectCategories(req.Categories, false); err != nil { + return nil, err + } + rep := c.buildReport(req, false) + c.invalidateCache() + + if !req.DryRun { + c.logClean(rep, req) + } + return rep, nil +} + +// buildReport 是 Inspect 与 Clean 的共用主体。 +// inspectAll 为 true 时统计全部类别（含被关闭的），供状态页展示； +// 为 false 时只处理已启用类别并真正执行删除。 +func (c *Cleaner) buildReport(req CleanRequest, inspectAll bool) *Report { + startedAt := c.now() + cats, err := c.selectCategories(req.Categories, !inspectAll) + rep := &Report{ + DryRun: req.DryRun, + Trigger: req.Trigger, + StartedAt: startedAt, + } + if err != nil { + rep.Note = err.Error() + rep.Categories = []CategoryReport{} + return rep + } + + for _, cat := range cats { + res := c.scan(cat, startedAt) + cr := res.report + + if !req.DryRun && cr.Enabled { + root := cr.Root + for _, u := range res.eligible { + if rmErr := removeUnit(u); rmErr != nil { + cr.Errors = append(cr.Errors, fmt.Sprintf("%s: %v", filepath.Base(u.Path), rmErr)) + continue + } + cr.RemovedUnits++ + cr.FreedBytes += u.Size + } + if cat.pruneEmpty && cr.RemovedUnits > 0 && root != "" { + // 日期层 + 会话层，最多两层。 + cr.RemovedEmptyDirs = pruneEmptyDirs(root, 2) + } + } + + rep.Categories = append(rep.Categories, cr) + } + if rep.Categories == nil { + rep.Categories = []CategoryReport{} + } + + for _, cr := range rep.Categories { + rep.Totals.Units += cr.Units + rep.Totals.Bytes += cr.Bytes + rep.Totals.ReclaimableUnits += cr.ReclaimableUnits + rep.Totals.ReclaimableBytes += cr.ReclaimableBytes + rep.Totals.RemovedUnits += cr.RemovedUnits + rep.Totals.FreedBytes += cr.FreedBytes + rep.Totals.SkippedActive += cr.SkippedActive + rep.Totals.Errors += len(cr.Errors) + } + + rep.Filesystem = c.filesystem() + rep.DurationMS = time.Since(startedAt).Milliseconds() + return rep +} + +// filesystem 取第一个存在的类别根目录所在文件系统，作为概览卡片的容量来源。 +func (c *Cleaner) filesystem() Filesystem { + probe := "" + for _, cat := range categories() { + if root := c.rootOf(cat); root != "" { + probe = root + break + } + } + if probe == "" { + probe = "." + } + fs, err := FilesystemUsage(probe) + if err != nil && c.logger != nil { + c.logger.Debug("查询文件系统容量失败", zap.String("path", probe), zap.Error(err)) + } + return fs +} + +// removeUnit 删除单元。目录先原子改名再递归删除： +// 中途崩溃只会留下带 deletionMarkerSuffix 的目录，下一轮 evaluate 会无条件补删。 +func removeUnit(u Unit) error { + if u.IsDir { + marker := u.Path + deletionMarkerSuffix + if err := os.Rename(u.Path, marker); err == nil { + return ignoreMissing(os.RemoveAll(marker)) + } + // 改名失败（跨设备、权限、同名残留）时退化为直接删除。 + } + return ignoreMissing(os.RemoveAll(u.Path)) +} + +func ignoreMissing(err error) error { + if err == nil || os.IsNotExist(err) { + return nil + } + return err +} + +func (c *Cleaner) logClean(rep *Report, req CleanRequest) { + if c.logger == nil { + return + } + summary := []zap.Field{ + zap.String("trigger", req.Trigger), + zap.Int("removed_units", rep.Totals.RemovedUnits), + zap.Int64("freed_bytes", rep.Totals.FreedBytes), + zap.Int("skipped_active", rep.Totals.SkippedActive), + zap.Int("errors", rep.Totals.Errors), + } + if rep.Totals.RemovedUnits == 0 && rep.Totals.Errors == 0 { + c.logger.Debug("运行空间清理完成，无可回收内容", summary...) + return + } + c.logger.Info("运行空间清理完成", summary...) + for _, cr := range rep.Categories { + if cr.RemovedUnits == 0 && len(cr.Errors) == 0 { + continue + } + c.logger.Info("清理类别明细", + zap.String("category", cr.Key), + zap.Int("removed", cr.RemovedUnits), + zap.Int64("freed_bytes", cr.FreedBytes), + zap.Strings("errors", cr.Errors)) + } +} diff --git a/internal/storage/cleaner_test.go b/internal/storage/cleaner_test.go new file mode 100644 index 000000000..0f293e09a --- /dev/null +++ b/internal/storage/cleaner_test.go @@ -0,0 +1,658 @@ +package storage + +import ( + "errors" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "testing" + "time" + + "cyberstrike-ai/internal/config" +) + +// fakeActivity 是 Activity 的测试替身。 +type fakeActivity struct { + conversations map[string]time.Time + projects map[string]time.Time + err error +} + +func (f fakeActivity) ConversationLastActivity(id string) (time.Time, bool, error) { + if f.err != nil { + return time.Time{}, false, f.err + } + at, ok := f.conversations[id] + return at, ok, nil +} + +func (f fakeActivity) ProjectLastActivity(id string) (time.Time, bool, error) { + if f.err != nil { + return time.Time{}, false, f.err + } + at, ok := f.projects[id] + return at, ok, nil +} + +// ageTree 在 root 下建一棵目录树，并把全部文件与目录的 mtime 统一改成 now-age。 +// 目录自身的 mtime 也要改：statUnit 取「目录与其内容的最新 mtime」， +// 只改文件的话新建目录的 mtime 仍是当下，单元会被活跃保护挡住。 +func ageTree(t *testing.T, root string, files map[string]int, age time.Duration, now time.Time) { + t.Helper() + stamp := now.Add(-age) + dirs := map[string]bool{root: true} + for rel, size := range files { + path := filepath.Join(root, rel) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", path, err) + } + if err := os.WriteFile(path, make([]byte, size), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } + if err := os.Chtimes(path, stamp, stamp); err != nil { + t.Fatalf("chtimes %s: %v", path, err) + } + for dir := filepath.Dir(path); dir != root && strings.HasPrefix(dir, root); dir = filepath.Dir(dir) { + dirs[dir] = true + } + } + // 自底向上：改动子目录会刷新父目录 mtime，顺序反了父目录仍是新的。 + order := make([]string, 0, len(dirs)) + for dir := range dirs { + order = append(order, dir) + } + sort.Slice(order, func(i, j int) bool { return len(order[i]) > len(order[j]) }) + for _, dir := range order { + if err := os.Chtimes(dir, stamp, stamp); err != nil { + t.Fatalf("chtimes dir %s: %v", dir, err) + } + } +} + +func exists(path string) bool { + _, err := os.Lstat(path) + return err == nil +} + +func newTestCleaner(t *testing.T, cfg *config.Config, paths Paths, activity Activity, now time.Time) *Cleaner { + t.Helper() + if cfg == nil { + cfg = &config.Config{} + } + return NewCleaner(Options{ + Config: cfg, + Paths: paths, + Activity: activity, + Now: func() time.Time { return now }, + CacheTTL: -1, // 测试里禁用缓存，保证每次都真实遍历 + }) +} + +func intPtr(v int) *int { return &v } +func boolPtr(v bool) *bool { return &v } + +func categoryCfg(days int) config.StorageCategoryConfig { + return config.StorageCategoryConfig{RetentionDays: intPtr(days)} +} + +// workspace 根目录布局：tmp/workspace/{projects,conversations}// +func TestCleanExpiresIdleWorkspaceAndKeepsFresh(t *testing.T) { + now := time.Date(2026, 9, 14, 12, 0, 0, 0, time.UTC) + tmp := t.TempDir() + ws := filepath.Join(tmp, "workspace") + + old := filepath.Join(ws, "conversations", "conv-old") + fresh := filepath.Join(ws, "conversations", "conv-fresh") + ageTree(t, old, map[string]int{"scan/nmap.txt": 4096}, 40*24*time.Hour, now) + ageTree(t, fresh, map[string]int{"scan/nmap.txt": 1024}, 2*24*time.Hour, now) + + activity := fakeActivity{conversations: map[string]time.Time{ + "conv-old": now.Add(-40 * 24 * time.Hour), + "conv-fresh": now.Add(-2 * 24 * time.Hour), + }} + cfg := &config.Config{Storage: config.StorageConfig{ + Categories: map[string]config.StorageCategoryConfig{ + config.StorageCategoryWorkspace: categoryCfg(30), + }, + }} + c := newTestCleaner(t, cfg, Paths{Workspace: ws}, activity, now) + + rep, err := c.Clean(CleanRequest{Categories: []string{config.StorageCategoryWorkspace}}) + if err != nil { + t.Fatalf("Clean: %v", err) + } + if !exists(fresh) { + t.Errorf("未过期的工作区被删除: %s", fresh) + } + if exists(old) { + t.Errorf("超过保留期的工作区未被删除: %s", old) + } + cat := findCategory(t, rep, config.StorageCategoryWorkspace) + if cat.RemovedUnits != 1 { + t.Errorf("RemovedUnits = %d, want 1", cat.RemovedUnits) + } + if cat.FreedBytes != 4096 { + t.Errorf("FreedBytes = %d, want 4096", cat.FreedBytes) + } +} + +func TestDryRunDeletesNothing(t *testing.T) { + now := time.Date(2026, 9, 14, 12, 0, 0, 0, time.UTC) + tmp := t.TempDir() + ws := filepath.Join(tmp, "workspace") + old := filepath.Join(ws, "conversations", "conv-old") + ageTree(t, old, map[string]int{"a.txt": 2048}, 40*24*time.Hour, now) + + activity := fakeActivity{conversations: map[string]time.Time{"conv-old": now.Add(-40 * 24 * time.Hour)}} + cfg := &config.Config{Storage: config.StorageConfig{ + Categories: map[string]config.StorageCategoryConfig{ + config.StorageCategoryWorkspace: categoryCfg(30), + }, + }} + c := newTestCleaner(t, cfg, Paths{Workspace: ws}, activity, now) + + rep, err := c.Clean(CleanRequest{DryRun: true}) + if err != nil { + t.Fatalf("Clean: %v", err) + } + if !exists(old) { + t.Fatal("dry-run 删除了文件") + } + if rep.Totals.ReclaimableBytes != 2048 { + t.Errorf("ReclaimableBytes = %d, want 2048", rep.Totals.ReclaimableBytes) + } + if rep.Totals.RemovedUnits != 0 { + t.Errorf("dry-run 不应有 RemovedUnits，got %d", rep.Totals.RemovedUnits) + } +} + +// 会话已从数据库消失 → 孤儿目录，按较短的 orphan_grace_days 回收。 +func TestOrphanReclaimedEvenWhenRetentionIsZero(t *testing.T) { + now := time.Date(2026, 9, 14, 12, 0, 0, 0, time.UTC) + tmp := t.TempDir() + ws := filepath.Join(tmp, "workspace") + orphan := filepath.Join(ws, "conversations", "conv-gone") + ageTree(t, orphan, map[string]int{"a.txt": 10}, 5*24*time.Hour, now) + + // retention_days: 0 表示不按保留期清理，但孤儿目录仍应回收。 + cfg := &config.Config{Storage: config.StorageConfig{ + OrphanGraceDays: intPtr(1), + Categories: map[string]config.StorageCategoryConfig{ + config.StorageCategoryWorkspace: categoryCfg(0), + }, + }} + c := newTestCleaner(t, cfg, Paths{Workspace: ws}, fakeActivity{}, now) + + if _, err := c.Clean(CleanRequest{}); err != nil { + t.Fatalf("Clean: %v", err) + } + if exists(orphan) { + t.Error("孤儿目录未被回收") + } +} + +// 会话仍存在且最近有活动 → 即使目录 mtime 很旧也必须保护。 +func TestActiveSessionIsProtected(t *testing.T) { + now := time.Date(2026, 9, 14, 12, 0, 0, 0, time.UTC) + tmp := t.TempDir() + ws := filepath.Join(tmp, "workspace") + dir := filepath.Join(ws, "conversations", "conv-busy") + ageTree(t, dir, map[string]int{"a.txt": 10}, 40*24*time.Hour, now) + + activity := fakeActivity{conversations: map[string]time.Time{ + "conv-busy": now.Add(-10 * time.Minute), // 10 分钟前还在跑 + }} + cfg := &config.Config{Storage: config.StorageConfig{ + Categories: map[string]config.StorageCategoryConfig{ + config.StorageCategoryWorkspace: categoryCfg(30), + }, + }} + c := newTestCleaner(t, cfg, Paths{Workspace: ws}, activity, now) + + rep, err := c.Clean(CleanRequest{}) + if err != nil { + t.Fatalf("Clean: %v", err) + } + if !exists(dir) { + t.Fatal("活跃会话的工作区被删除") + } + if rep.Totals.SkippedActive != 1 { + t.Errorf("SkippedActive = %d, want 1", rep.Totals.SkippedActive) + } +} + +// 活跃状态查询失败时必须保守跳过：宁可少删，不可误删正在跑的任务数据。 +func TestActivityLookupErrorFailsClosed(t *testing.T) { + now := time.Date(2026, 9, 14, 12, 0, 0, 0, time.UTC) + tmp := t.TempDir() + ws := filepath.Join(tmp, "workspace") + dir := filepath.Join(ws, "conversations", "conv-x") + ageTree(t, dir, map[string]int{"a.txt": 10}, 400*24*time.Hour, now) + + cfg := &config.Config{Storage: config.StorageConfig{ + Categories: map[string]config.StorageCategoryConfig{ + config.StorageCategoryWorkspace: categoryCfg(30), + }, + }} + c := newTestCleaner(t, cfg, Paths{Workspace: ws}, fakeActivity{err: errors.New("db locked")}, now) + + if _, err := c.Clean(CleanRequest{}); err != nil { + t.Fatalf("Clean: %v", err) + } + if !exists(dir) { + t.Fatal("查询失败时不应删除任何数据") + } +} + +// 指向目录的符号链接不会被任何 scanner 当作删除单元： +// os.ReadDir 的 DirEntry.IsDir() 对符号链接返回 false。 +// 这是更安全的行为 —— 工作区里被塞进一个指向 /etc 的软链时， +// 既不会跟随它，也不会把它当成会话目录处理。 +func TestSymlinkInsideRootIsNeverADeletionUnit(t *testing.T) { + now := time.Date(2026, 9, 14, 12, 0, 0, 0, time.UTC) + tmp := t.TempDir() + ws := filepath.Join(tmp, "workspace") + outside := filepath.Join(tmp, "precious") + if err := os.MkdirAll(filepath.Join(outside, "sub"), 0o755); err != nil { + t.Fatal(err) + } + victim := filepath.Join(outside, "sub", "keep.txt") + if err := os.WriteFile(victim, []byte("important"), 0o644); err != nil { + t.Fatal(err) + } + + linkParent := filepath.Join(ws, "conversations") + link := filepath.Join(linkParent, "conv-link") + if err := os.MkdirAll(linkParent, 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, link); err != nil { + t.Skipf("symlink 不可用: %v", err) + } + stamp := now.Add(-400 * 24 * time.Hour) + if err := os.Chtimes(link, stamp, stamp); err != nil { + t.Fatal(err) + } + + cfg := &config.Config{Storage: config.StorageConfig{ + Categories: map[string]config.StorageCategoryConfig{ + config.StorageCategoryWorkspace: categoryCfg(30), + }, + }} + c := newTestCleaner(t, cfg, Paths{Workspace: ws}, nil, now) + + rep, err := c.Clean(CleanRequest{}) + if err != nil { + t.Fatalf("Clean: %v", err) + } + if !exists(link) { + t.Error("符号链接不应被当作删除单元移除") + } + if !exists(victim) { + t.Fatal("符号链接目标被删除，发生路径逃逸") + } + if rep.Totals.Units != 0 { + t.Errorf("Units = %d, want 0（符号链接不计入）", rep.Totals.Units) + } +} + +// 上一轮清理崩溃留下的标记目录必须被无条件补删， +// 否则它的 mtime 是刚改名的时间，会被活跃保护永久挡住。 +func TestLeftoverDeletionMarkerIsReclaimed(t *testing.T) { + now := time.Date(2026, 9, 14, 12, 0, 0, 0, time.UTC) + tmp := t.TempDir() + ws := filepath.Join(tmp, "workspace") + marker := filepath.Join(ws, "conversations", "conv-crash"+deletionMarkerSuffix) + // mtime 就是「刚刚」，模拟崩溃后立即重跑。 + ageTree(t, marker, map[string]int{"a.txt": 10}, 0, now) + + cfg := &config.Config{Storage: config.StorageConfig{ + Categories: map[string]config.StorageCategoryConfig{ + config.StorageCategoryWorkspace: categoryCfg(30), + }, + }} + c := newTestCleaner(t, cfg, Paths{Workspace: ws}, fakeActivity{}, now) + + if _, err := c.Clean(CleanRequest{}); err != nil { + t.Fatalf("Clean: %v", err) + } + if exists(marker) { + t.Error("崩溃残留的标记目录未被补删") + } +} + +// chat_uploads 是 root/<日期>/<会话> 三层布局，删完会话目录后空的日期目录也要回收。 +func TestChatUploadsDatedLayoutAndEmptyDirPrune(t *testing.T) { + now := time.Date(2026, 9, 14, 12, 0, 0, 0, time.UTC) + tmp := t.TempDir() + uploads := filepath.Join(tmp, "chat_uploads") + dateDir := filepath.Join(uploads, "2026-06-01") + convDir := filepath.Join(dateDir, "conv-old") + ageTree(t, convDir, map[string]int{"report.pdf": 100}, 120*24*time.Hour, now) + + cfg := &config.Config{Storage: config.StorageConfig{ + Categories: map[string]config.StorageCategoryConfig{ + config.StorageCategoryChatUploads: categoryCfg(90), + }, + }} + c := newTestCleaner(t, cfg, Paths{ChatUploads: uploads}, fakeActivity{}, now) + + rep, err := c.Clean(CleanRequest{}) + if err != nil { + t.Fatalf("Clean: %v", err) + } + if exists(convDir) { + t.Error("过期上传目录未被删除") + } + if exists(dateDir) { + t.Error("空的日期目录未被回收") + } + if !exists(uploads) { + t.Error("类别根目录不应被删除") + } + // Clean 未指定类别时会返回全部已注册类别，必须按 key 取，不能依赖下标。 + cat := findCategory(t, rep, config.StorageCategoryChatUploads) + if cat.RemovedEmptyDirs != 1 { + t.Errorf("RemovedEmptyDirs = %d, want 1", cat.RemovedEmptyDirs) + } + if cat.RemovedUnits != 1 { + t.Errorf("RemovedUnits = %d, want 1", cat.RemovedUnits) + } +} + +// findCategory 按 key 取报表条目；缺失时直接失败，避免用错下标断言到别的类别。 +func findCategory(t *testing.T, rep *Report, key string) CategoryReport { + t.Helper() + for _, cat := range rep.Categories { + if cat.Key == key { + return cat + } + } + t.Fatalf("报表中缺少类别 %s", key) + return CategoryReport{} +} + +// 占位目录 _new / _manual 不是会话 ID，不能拿去查数据库，按非会话型处理。 +func TestChatUploadsPlaceholderDirIsNotSessionScoped(t *testing.T) { + now := time.Date(2026, 9, 14, 12, 0, 0, 0, time.UTC) + tmp := t.TempDir() + uploads := filepath.Join(tmp, "chat_uploads") + placeholder := filepath.Join(uploads, "2026-06-01", "_new") + ageTree(t, placeholder, map[string]int{"a.png": 10}, 120*24*time.Hour, now) + + // Activity 对任何查询都报错：若占位目录被当成会话，会因 fail-closed 而被保留。 + cfg := &config.Config{Storage: config.StorageConfig{ + Categories: map[string]config.StorageCategoryConfig{ + config.StorageCategoryChatUploads: categoryCfg(90), + }, + }} + c := newTestCleaner(t, cfg, Paths{ChatUploads: uploads}, + fakeActivity{err: errors.New("should not be called")}, now) + + if _, err := c.Clean(CleanRequest{}); err != nil { + t.Fatalf("Clean: %v", err) + } + if exists(placeholder) { + t.Error("占位目录应按保留期清理，而不是走会话查询") + } +} + +// glob 型类别：只删匹配的文件，同目录下的其他文件不受影响。 +func TestPatternCategoryOnlyMatchesItsGlob(t *testing.T) { + now := time.Date(2026, 9, 14, 12, 0, 0, 0, time.UTC) + tmp := t.TempDir() + logs := filepath.Join(tmp, "log") + if err := os.MkdirAll(logs, 0o755); err != nil { + t.Fatal(err) + } + old := now.Add(-40 * 24 * time.Hour) + stale := filepath.Join(logs, "diagnostic-2026-08-01.log") + keepName := filepath.Join(logs, "app.log") + for _, p := range []string{stale, keepName} { + if err := os.WriteFile(p, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(p, old, old); err != nil { + t.Fatal(err) + } + } + + cfg := &config.Config{Storage: config.StorageConfig{ + Categories: map[string]config.StorageCategoryConfig{ + config.StorageCategoryDiagnosticLogs: categoryCfg(14), + }, + }} + c := newTestCleaner(t, cfg, Paths{DiagnosticLogs: logs}, nil, now) + + if _, err := c.Clean(CleanRequest{}); err != nil { + t.Fatalf("Clean: %v", err) + } + if exists(stale) { + t.Error("过期诊断日志未被删除") + } + if !exists(keepName) { + t.Error("不匹配 glob 的文件被误删") + } +} + +func TestDisabledCategoryIsReportedButNotCleaned(t *testing.T) { + now := time.Date(2026, 9, 14, 12, 0, 0, 0, time.UTC) + tmp := t.TempDir() + ws := filepath.Join(tmp, "workspace") + dir := filepath.Join(ws, "conversations", "conv-old") + ageTree(t, dir, map[string]int{"a.txt": 10}, 400*24*time.Hour, now) + + cfg := &config.Config{Storage: config.StorageConfig{ + Categories: map[string]config.StorageCategoryConfig{ + config.StorageCategoryWorkspace: {Enabled: boolPtr(false), RetentionDays: intPtr(30)}, + }, + }} + c := newTestCleaner(t, cfg, Paths{Workspace: ws}, fakeActivity{}, now) + + // Inspect 仍应展示被关闭的类别，让管理员看到可回收量后再决定是否开启。 + inspected := c.Inspect(true) + var found bool + for _, cat := range inspected.Categories { + if cat.Key == config.StorageCategoryWorkspace { + found = true + if cat.Enabled { + t.Error("类别应为 disabled") + } + if cat.ReclaimableUnits != 1 { + t.Errorf("ReclaimableUnits = %d, want 1", cat.ReclaimableUnits) + } + } + } + if !found { + t.Fatal("Inspect 未返回 workspace 类别") + } + + if _, err := c.Clean(CleanRequest{}); err != nil { + t.Fatalf("Clean: %v", err) + } + if !exists(dir) { + t.Error("被关闭的类别不应被清理") + } +} + +func TestUnknownCategoryIsRejected(t *testing.T) { + now := time.Now() + c := newTestCleaner(t, nil, Paths{Workspace: t.TempDir()}, nil, now) + if _, err := c.Clean(CleanRequest{Categories: []string{"../../etc"}}); !errors.Is(err, ErrUnknownCategory) { + t.Errorf("err = %v, want ErrUnknownCategory", err) + } +} + +func TestConcurrentCleanIsRejected(t *testing.T) { + now := time.Date(2026, 9, 14, 12, 0, 0, 0, time.UTC) + tmp := t.TempDir() + ws := filepath.Join(tmp, "workspace") + for i := 0; i < 20; i++ { + dir := filepath.Join(ws, "conversations", "conv-"+string(rune('a'+i))) + ageTree(t, dir, map[string]int{"a.txt": 10}, 400*24*time.Hour, now) + } + c := newTestCleaner(t, nil, Paths{Workspace: ws}, fakeActivity{}, now) + + // 手动占住执行位，模拟「已有一轮在跑」。 + if !c.running.CompareAndSwap(false, true) { + t.Fatal("无法占用执行位") + } + if _, err := c.Clean(CleanRequest{}); !errors.Is(err, ErrCleanupInProgress) { + t.Errorf("err = %v, want ErrCleanupInProgress", err) + } + c.running.Store(false) + + if _, err := c.Clean(CleanRequest{}); err != nil { + t.Fatalf("释放后应可再次清理: %v", err) + } +} + +// 并发触发清理时只允许一轮真正执行，其余应立即得到 ErrCleanupInProgress 而不是排队删除。 +func TestParallelCleanHasSingleWinner(t *testing.T) { + now := time.Date(2026, 9, 14, 12, 0, 0, 0, time.UTC) + tmp := t.TempDir() + ws := filepath.Join(tmp, "workspace") + dir := filepath.Join(ws, "conversations", "conv-old") + ageTree(t, dir, map[string]int{"a.txt": 10}, 400*24*time.Hour, now) + + c := newTestCleaner(t, nil, Paths{Workspace: ws}, fakeActivity{}, now) + + const n = 8 + var wg sync.WaitGroup + results := make([]error, n) + start := make(chan struct{}) + for i := 0; i < n; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + _, err := c.Clean(CleanRequest{}) + results[i] = err + }(i) + } + close(start) + wg.Wait() + + inProgress := 0 + for _, err := range results { + switch { + case err == nil: + case errors.Is(err, ErrCleanupInProgress): + inProgress++ + default: + t.Errorf("意外错误: %v", err) + } + } + if inProgress == 0 { + t.Log("提示：本轮所有 goroutine 都串行完成了，未观察到并发冲突（非失败）") + } +} + +func TestConfinedRejectsEscapes(t *testing.T) { + root := filepath.Join(string(filepath.Separator), "srv", "app", "tmp", "workspace") + cases := []struct { + candidate string + want bool + }{ + {filepath.Join(root, "conversations", "abc"), true}, + {filepath.Join(root, "a", "b", "c"), true}, + {root, false}, + {filepath.Join(root, ".."), false}, + {filepath.Join(root, "..", "secrets"), false}, + {filepath.Join(string(filepath.Separator), "etc", "passwd"), false}, + {"", false}, + } + for _, tc := range cases { + if got := confined(root, tc.candidate); got != tc.want { + t.Errorf("confined(%q) = %v, want %v", tc.candidate, got, tc.want) + } + } + if confined("", filepath.Join(root, "x")) { + t.Error("空 root 不应通过校验") + } +} + +// 根目录不存在时不应报错，只标记 Missing —— 系统尚未产生该类垃圾是正常状态。 +func TestMissingRootIsNotAnError(t *testing.T) { + now := time.Now() + c := newTestCleaner(t, nil, Paths{Workspace: filepath.Join(t.TempDir(), "never-created")}, nil, now) + + rep := c.Inspect(true) + if len(rep.Categories) != len(config.StorageCategoryOrder) { + t.Fatalf("Categories = %d, want %d", len(rep.Categories), len(config.StorageCategoryOrder)) + } + for _, cat := range rep.Categories { + if !cat.Missing { + t.Errorf("类别 %s 的根目录不存在/未配置，应标记 Missing", cat.Key) + } + if cat.Units != 0 || len(cat.Errors) != 0 { + t.Errorf("类别 %s 不应有统计或错误: units=%d errors=%v", cat.Key, cat.Units, cat.Errors) + } + } + if _, err := c.Clean(CleanRequest{}); err != nil { + t.Errorf("Clean: %v", err) + } +} + +func TestInspectCachesUntilRefresh(t *testing.T) { + now := time.Date(2026, 9, 14, 12, 0, 0, 0, time.UTC) + tmp := t.TempDir() + ws := filepath.Join(tmp, "workspace") + dir := filepath.Join(ws, "conversations", "conv-a") + ageTree(t, dir, map[string]int{"a.txt": 10}, 400*24*time.Hour, now) + + c := NewCleaner(Options{ + Config: &config.Config{}, + Paths: Paths{Workspace: ws}, + Activity: fakeActivity{}, + Now: func() time.Time { return now }, + CacheTTL: time.Minute, + }) + + first := c.Inspect(false) + if first.Totals.Units != 1 { + t.Fatalf("首次统计 Units = %d, want 1", first.Totals.Units) + } + // 缓存生效期间新增目录不应被看到。 + ageTree(t, filepath.Join(ws, "conversations", "conv-b"), map[string]int{"b.txt": 10}, 400*24*time.Hour, now) + if cached := c.Inspect(false); cached.Totals.Units != 1 { + t.Errorf("缓存期内 Units = %d, want 1", cached.Totals.Units) + } + if refreshed := c.Inspect(true); refreshed.Totals.Units != 2 { + t.Errorf("refresh 后 Units = %d, want 2", refreshed.Totals.Units) + } + // 清理后缓存必须失效，否则状态页仍显示已删除的内容。 + if _, err := c.Clean(CleanRequest{}); err != nil { + t.Fatalf("Clean: %v", err) + } + if after := c.Inspect(false); after.Totals.Units != 0 { + t.Errorf("清理后 Units = %d, want 0", after.Totals.Units) + } +} + +func TestRetentionLoopDisabledByDefault(t *testing.T) { + s := NewService(nil, &config.Config{}, nil) + if s.AutoCleanEnabled() { + t.Error("auto_clean 默认必须为关闭") + } + // 关闭时 PurgeExpired 是 no-op，不应 panic。 + s.PurgeExpired() + + enabled := &config.Config{Storage: config.StorageConfig{AutoClean: boolPtr(true)}} + if !NewService(nil, enabled, nil).AutoCleanEnabled() { + t.Error("显式 true 时应开启") + } +} + +func TestIntervalHasFloor(t *testing.T) { + cfg := &config.Config{Storage: config.StorageConfig{IntervalMinutes: intPtr(1)}} + if got := NewService(nil, cfg, nil).Interval(); got != minSweepInterval { + t.Errorf("Interval = %v, want %v", got, minSweepInterval) + } + cfg = &config.Config{Storage: config.StorageConfig{IntervalMinutes: intPtr(180)}} + if got := NewService(nil, cfg, nil).Interval(); got != 3*time.Hour { + t.Errorf("Interval = %v, want 3h", got) + } +} diff --git a/internal/storage/retention.go b/internal/storage/retention.go new file mode 100644 index 000000000..4bda727ee --- /dev/null +++ b/internal/storage/retention.go @@ -0,0 +1,93 @@ +package storage + +import ( + "errors" + "time" + + "cyberstrike-ai/internal/config" + + "go.uber.org/zap" +) + +// minSweepInterval 是后台清理的下限，避免把 interval_minutes 配成极小值后持续遍历磁盘。 +const minSweepInterval = 5 * time.Minute + +// sweepCheckInterval 是后台循环的唤醒粒度。 +// 不能按 interval 直接睡一整段：那样把 interval_minutes 从 60 改成 5 后， +// 必须等当前这段 60 分钟睡眠结束才生效。短粒度唤醒 + 到期判断把延迟限制在一个粒度内。 +const sweepCheckInterval = time.Minute + +// Service 驱动后台自动清理。 +type Service struct { + cleaner *Cleaner + cfg *config.Config + logger *zap.Logger +} + +// NewService 创建后台清理服务。 +func NewService(cleaner *Cleaner, cfg *config.Config, logger *zap.Logger) *Service { + return &Service{cleaner: cleaner, cfg: cfg, logger: logger} +} + +// AutoCleanEnabled 返回后台自动清理是否开启（默认关闭）。 +func (s *Service) AutoCleanEnabled() bool { + if s == nil || s.cfg == nil { + return false + } + return s.cfg.Storage.AutoCleanEffective() +} + +// Interval 返回清理间隔；每轮重新读取，因此改配置无需重启。 +func (s *Service) Interval() time.Duration { + if s == nil || s.cfg == nil { + return minSweepInterval + } + d := time.Duration(s.cfg.Storage.IntervalMinutesEffective()) * time.Minute + if d < minSweepInterval { + return minSweepInterval + } + return d +} + +// PurgeExpired 执行一轮自动清理；未开启自动清理时直接返回。 +func (s *Service) PurgeExpired() { + if s == nil || s.cleaner == nil || !s.AutoCleanEnabled() { + return + } + if _, err := s.cleaner.Clean(CleanRequest{Trigger: "schedule"}); err != nil { + if s.logger != nil { + // 并发冲突不是故障：说明已有一轮在跑，跳过即可。 + if errors.Is(err, ErrCleanupInProgress) { + s.logger.Debug("已有存储清理在执行，跳过本轮") + return + } + s.logger.Warn("运行空间自动清理失败", zap.Error(err)) + } + } +} + +// StartRetentionLoop 按配置间隔周期性清理运行空间垃圾。 +// 以 sweepCheckInterval 粒度唤醒、到期才执行，因此 interval_minutes 的改动最多一个粒度后生效。 +func StartRetentionLoop(s *Service, logger *zap.Logger) { + if s == nil || s.cleaner == nil { + return + } + // 启动后先跑一轮，避免「配置好了但要等一个间隔才见效」； + // 放在 goroutine 里，全量目录遍历不阻塞启动流程。 + go func() { + s.PurgeExpired() + last := time.Now() + ticker := time.NewTicker(sweepCheckInterval) + defer ticker.Stop() + for range ticker.C { + if time.Since(last) < s.Interval() { + continue + } + last = time.Now() + s.PurgeExpired() + if logger != nil { + logger.Debug("storage retention tick completed") + } + } + }() +} diff --git a/internal/storage/usage.go b/internal/storage/usage.go new file mode 100644 index 000000000..77ebe3177 --- /dev/null +++ b/internal/storage/usage.go @@ -0,0 +1,19 @@ +package storage + +// Filesystem 描述某个路径所在文件系统的容量。 +type Filesystem struct { + Path string `json:"path"` + TotalBytes int64 `json:"total_bytes"` + FreeBytes int64 `json:"free_bytes"` + UsedBytes int64 `json:"used_bytes"` + UsedPercent float64 `json:"used_percent"` + InodesTotal int64 `json:"inodes_total"` + InodesFree int64 `json:"inodes_free"` + // Available 为 false 表示当前平台不支持查询，前端应隐藏容量卡片而不是显示 0。 + Available bool `json:"available"` +} + +// FilesystemUsage 返回 path 所在文件系统的容量信息。 +func FilesystemUsage(path string) (Filesystem, error) { + return filesystemUsage(path) +} diff --git a/internal/storage/usage_linux.go b/internal/storage/usage_linux.go new file mode 100644 index 000000000..062035542 --- /dev/null +++ b/internal/storage/usage_linux.go @@ -0,0 +1,64 @@ +//go:build linux + +package storage + +import ( + "os" + "path/filepath" + "syscall" +) + +func filesystemUsage(path string) (Filesystem, error) { + path = filepath.Clean(path) + if path == "" { + path = "." + } + // 路径可能尚不存在，向上找最近的存在祖先，否则 Statfs 直接 ENOENT。 + probe := path + for { + if _, err := os.Stat(probe); err == nil { + break + } + parent := filepath.Dir(probe) + if parent == probe { + break + } + probe = parent + } + + var st syscall.Statfs_t + if err := syscall.Statfs(probe, &st); err != nil { + return Filesystem{Path: path}, err + } + + bsize := int64(st.Bsize) + if bsize <= 0 { + bsize = 512 + } + total := int64(st.Blocks) * bsize + // 用 Bavail 而不是 Bfree：f_bfree 含 root 保留块（通常约 5%）， + // 对非 root 进程会高估可用空间，导致「明明还有空间却写失败」。 + free := int64(st.Bavail) * bsize + used := (int64(st.Blocks) - int64(st.Bfree)) * bsize + if used < 0 { + used = 0 + } + + fs := Filesystem{ + Path: path, + TotalBytes: total, + FreeBytes: free, + UsedBytes: used, + InodesTotal: int64(st.Files), + InodesFree: int64(st.Ffree), + Available: true, + } + // 分母用 used+free 而非 total：与 gopsutil 一致，避免 root 保留块把使用率算低。 + if denom := used + free; denom > 0 { + fs.UsedPercent = float64(used) / float64(denom) * 100 + } + if fs.InodesFree < 0 { + fs.InodesFree = 0 + } + return fs, nil +} diff --git a/internal/storage/usage_other.go b/internal/storage/usage_other.go new file mode 100644 index 000000000..ae3be3930 --- /dev/null +++ b/internal/storage/usage_other.go @@ -0,0 +1,16 @@ +//go:build !linux + +package storage + +// errPlatformUnsupported 由调用方转换为 Available=false，不视为致命错误。 +var errPlatformUnsupported = &unsupportedPlatformError{} + +type unsupportedPlatformError struct{} + +func (e *unsupportedPlatformError) Error() string { + return "filesystem usage query is only supported on linux" +} + +func filesystemUsage(path string) (Filesystem, error) { + return Filesystem{Path: path}, errPlatformUnsupported +} diff --git a/web/static/css/style.css b/web/static/css/style.css index 0589ad310..7b0f32196 100644 --- a/web/static/css/style.css +++ b/web/static/css/style.css @@ -46440,3 +46440,203 @@ html[data-theme="dark"] { --tool-blocked-color: #f4bc62; } .mcp-stats-timeline__legend-item--blocked::before { background: #d99224; } .mcp-stats-rate.is-muted, .mcp-stats-tool-item__rate.is-muted { color: var(--text-muted); } + +/* ============================================ + 系统设置 -> 存储清理（运行空间垃圾回收） + ============================================ */ +.storage-desc { + margin: 0 0 16px; + color: var(--text-secondary); + font-size: .8125rem; + line-height: 1.6; +} + +/* display:grid/flex 会盖过 UA 的 [hidden]，需显式提权 */ +.storage-overview[hidden], +.storage-result[hidden] { display: none; } + +.storage-overview { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(190px, 1fr)); + gap: 12px; + margin-bottom: 16px; +} + +.storage-overview-card { + border: 1px solid var(--border-color); + border-radius: 8px; + background: var(--bg-secondary); + padding: 12px 14px; + display: flex; + flex-direction: column; + gap: 6px; +} + +.storage-overview-card--accent { border-color: var(--warning-color); } + +.storage-overview-label { + color: var(--text-secondary); + font-size: .75rem; + font-weight: 600; +} + +.storage-overview-card strong { + font-size: 1.25rem; + color: var(--text-primary); + font-variant-numeric: tabular-nums; +} + +.storage-overview-hint { + color: var(--text-secondary); + font-size: .75rem; +} + +.storage-usage-bar { + height: 6px; + border-radius: 3px; + background: var(--bg-tertiary); + overflow: hidden; +} + +.storage-usage-bar > span { + display: block; + height: 100%; + width: 0; + background: var(--primary-color); + transition: width .2s ease; +} + +.storage-usage-bar > span.storage-usage-bar-fill--warn { background: var(--error-color); } + +.storage-toolbar { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 8px; + margin-bottom: 12px; +} + +.storage-scanned-at { + margin-left: auto; + color: var(--text-secondary); + font-size: .75rem; +} + +.storage-result { + border: 1px solid var(--border-color); + border-left: 3px solid var(--success-color); + border-radius: 8px; + background: var(--bg-secondary); + padding: 12px 14px; + margin-bottom: 16px; + font-size: .8125rem; +} + +.storage-result--error { border-left-color: var(--error-color); } + +.storage-result-head { + display: flex; + flex-wrap: wrap; + gap: 8px; + align-items: baseline; + justify-content: space-between; +} + +.storage-result-list { + list-style: none; + margin: 10px 0 0; + padding: 0; + display: flex; + flex-direction: column; + gap: 4px; +} + +.storage-result-list li { + display: flex; + justify-content: space-between; + gap: 12px; + color: var(--text-secondary); +} + +.storage-error-text { color: var(--error-color); } + +.storage-table .storage-cat-label { font-weight: 600; color: var(--text-primary); } + +.storage-table .storage-sub { + display: block; + color: var(--text-secondary); + font-size: .6875rem; + margin-top: 2px; +} + +.storage-root { + font-size: .75rem; + color: var(--text-secondary); + background: var(--bg-tertiary); + border-radius: 4px; + padding: 1px 5px; +} + +.storage-muted { color: var(--text-secondary); font-size: .75rem; } +.storage-empty { text-align: center; color: var(--text-secondary); padding: 20px 0; } +.storage-row--disabled { opacity: .55; } + +.storage-retention-input { + width: 72px; + padding: 4px 6px; + border: 1px solid var(--border-color); + border-radius: 4px; + background: var(--bg-primary, #fff); + color: var(--text-primary); + font-size: .8125rem; +} + +.storage-policy { + margin-top: 20px; + border-top: 1px solid var(--border-color); + padding-top: 16px; +} + +.storage-policy h4 { + margin: 0 0 12px; + font-size: .875rem; + color: var(--text-primary); +} + +.storage-policy-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(210px, 1fr)); + gap: 12px; +} + +.storage-policy-field { + display: flex; + flex-direction: column; + gap: 6px; + font-size: .75rem; + color: var(--text-secondary); +} + +.storage-policy-field input[type="number"] { + padding: 6px 8px; + border: 1px solid var(--border-color); + border-radius: 4px; + background: var(--bg-primary, #fff); + color: var(--text-primary); + font-size: .8125rem; +} + +.storage-policy-field--switch { + flex-direction: row; + align-items: center; + gap: 8px; + font-size: .8125rem; + color: var(--text-primary); +} + +.storage-policy-hint { + margin: 12px 0 0; + color: var(--text-secondary); + font-size: .75rem; + line-height: 1.6; +} diff --git a/web/static/i18n/en-US.json b/web/static/i18n/en-US.json index 7f44c2757..f7b363c50 100644 --- a/web/static/i18n/en-US.json +++ b/web/static/i18n/en-US.json @@ -1776,6 +1776,7 @@ "security": "Security", "rbac": "Platform permissions", "audit": "Audit logs", + "storage": "Storage cleanup", "infocollect": "Asset management" }, "infocollect": { @@ -3044,6 +3045,70 @@ "close": "×", "newTerminal": "+" }, + "settingsStorage": { + "title": "Storage cleanup", + "subtitle": "Inspect and reclaim disk space used by runtime artifacts (agent workspaces, tool-output spill, C2 artifacts, chat uploads). Deletion is irreversible - preview first.", + "diskUsed": "Disk used", + "runtimeTotal": "Runtime footprint", + "reclaimable": "Reclaimable", + "inodes": "inode usage", + "inodesHint": "Scans can exhaust inodes with huge numbers of tiny files", + "refresh": "Rescan", + "preview": "Preview reclaimable", + "cleanNow": "Clean now", + "colCategory": "Category", + "colRoot": "Directory", + "colUnits": "Items", + "colSize": "Size", + "colReclaimable": "Reclaimable", + "colRetention": "Retention days", + "colEnabled": "Enabled", + "loading": "Loading...", + "policyTitle": "Automatic cleanup policy", + "autoClean": "Enable scheduled background cleanup", + "intervalMinutes": "Interval (minutes)", + "orphanGraceDays": "Orphan grace (days)", + "activeGraceHours": "Active session protection (hours)", + "policyHint": "Retention of 0 disables age-based cleanup, but directories whose conversation is already gone are still reclaimed. Sessions with recent activity are never cleaned. Automatic cleanup is off by default, so upgrading never deletes existing data.", + "save": "Save policy", + "loadMetaFailed": "Failed to load cleanup policy", + "loadStatusFailed": "Failed to load storage usage", + "cleanupFailed": "Cleanup request failed", + "unavailable": "Unavailable", + "free": "free", + "itemsUnit": "items", + "daysUnit": "days", + "noCategories": "No cleanable categories", + "orphanSuffix": "incl. orphans", + "skippedActive": "skipped active", + "defaultRetention": "Default", + "unused": "Unused", + "scannedAt": "Scanned at", + "previewTitle": "Preview (nothing was deleted)", + "cleanDone": "Cleanup finished", + "nothingToClean": "Nothing currently matches the cleanup policy.", + "skippedActiveNote": "Skipped sessions with recent activity: ", + "failed": "Operation failed", + "confirmClean": "This permanently deletes about {size} ({count} items) of runtime files and cannot be undone. Run \"Preview reclaimable\" first. Continue?", + "saveFailed": "Save failed", + "saved": "Cleanup policy saved", + "cat.workspace": "Agent workspace", + "catHint.workspace": "Agent download & scan artifacts (tmp/workspace), split per project/conversation.", + "cat.reduction": "Tool output spill", + "catHint.reduction": "Truncated spill files of oversized tool output (tmp/reduction); one file per execution, purely derived.", + "cat.conversation_artifacts": "Conversation artifacts", + "catHint.conversation_artifacts": "Summarization transcripts and the oversized user-input ledger (data/conversation_artifacts).", + "cat.plantask": "Plan task boards", + "catHint.plantask": "Eino multi-agent plan boards (skills/.eino/plantask), purely derived.", + "cat.c2_artifacts": "C2 artifacts", + "catHint.c2_artifacts": "C2 screenshots, uploads, pushed files and generated payloads (tmp/c2). Cleaning payloads invalidates their download links.", + "cat.chat_uploads": "Chat uploads", + "catHint.chat_uploads": "Files uploaded in conversations (chat_uploads//).", + "cat.workflow_checkpoints": "Workflow checkpoints", + "catHint.workflow_checkpoints": "Workflow run checkpoints (data/workflow-checkpoints), only used to resume interrupted runs.", + "cat.diagnostic_logs": "Diagnostic logs", + "catHint.diagnostic_logs": "Daily-rotated diagnostic logs (log/diagnostic-*.log)." + }, "settingsAudit": { "title": "Audit logs", "description": "Platform admin actions (login, config, deletes). Does not log chat content, per-command terminal/WebShell runs, or per-tool invocations.", diff --git a/web/static/i18n/zh-CN.json b/web/static/i18n/zh-CN.json index 9841e46f9..37b8c7ae2 100644 --- a/web/static/i18n/zh-CN.json +++ b/web/static/i18n/zh-CN.json @@ -1764,6 +1764,7 @@ "security": "安全设置", "rbac": "平台权限", "audit": "日志审计", + "storage": "存储清理", "infocollect": "资产管理" }, "infocollect": { @@ -3032,6 +3033,70 @@ "close": "×", "newTerminal": "+" }, + "settingsStorage": { + "title": "存储清理", + "subtitle": "统计并清理运行期间产生的磁盘垃圾（Agent 工作区、工具输出缓存、C2 产物、对话上传件等）。删除不可逆，建议先预览再执行。", + "diskUsed": "磁盘已用", + "runtimeTotal": "运行空间占用", + "reclaimable": "可回收", + "inodes": "inode 使用", + "inodesHint": "扫描产生的海量小文件可能先耗尽 inode", + "refresh": "重新扫描", + "preview": "预览可清理项", + "cleanNow": "立即清理", + "colCategory": "类别", + "colRoot": "目录", + "colUnits": "项目数", + "colSize": "占用", + "colReclaimable": "可回收", + "colRetention": "保留天数", + "colEnabled": "启用", + "loading": "加载中…", + "policyTitle": "自动清理策略", + "autoClean": "开启后台定时清理", + "intervalMinutes": "清理间隔（分钟）", + "orphanGraceDays": "孤儿目录宽限（天）", + "activeGraceHours": "活跃会话保护（小时）", + "policyHint": "保留天数为 0 表示不按保留期清理，但会话已删除的孤儿目录仍会回收；最近有活动的会话在任何情况下都不会被清理。默认关闭自动清理，升级不会自动删除既有数据。", + "save": "保存策略", + "loadMetaFailed": "获取清理策略失败", + "loadStatusFailed": "获取存储占用失败", + "cleanupFailed": "清理请求失败", + "unavailable": "不可用", + "free": "可用", + "itemsUnit": "项", + "daysUnit": "天", + "noCategories": "暂无可清理类别", + "orphanSuffix": "含孤儿", + "skippedActive": "跳过活跃", + "defaultRetention": "默认", + "unused": "未使用", + "scannedAt": "扫描于", + "previewTitle": "预览结果（未删除任何文件）", + "cleanDone": "清理完成", + "nothingToClean": "当前没有符合清理条件的内容。", + "skippedActiveNote": "已跳过最近仍在活动的会话：", + "failed": "操作失败", + "confirmClean": "将永久删除约 {size}（{count} 项）运行空间文件，无法恢复。建议先执行「预览可清理项」。确认继续？", + "saveFailed": "保存失败", + "saved": "清理策略已保存", + "cat.workspace": "Agent 工作区", + "catHint.workspace": "Agent 下载与扫描产物的工作目录（tmp/workspace），按项目/会话分目录。", + "cat.reduction": "工具输出缓存", + "catHint.reduction": "超长工具输出落盘的截断文件（tmp/reduction），每条执行一个文件，属纯派生数据。", + "cat.conversation_artifacts": "会话产物", + "catHint.conversation_artifacts": "摘要记录与超长用户输入台账（data/conversation_artifacts）。", + "cat.plantask": "计划任务看板", + "catHint.plantask": "Eino 多代理计划看板 JSON（skills/.eino/plantask），属纯派生数据。", + "cat.c2_artifacts": "C2 产物", + "catHint.c2_artifacts": "C2 回传截图、上传件、下发文件与已生成的 payload 二进制（tmp/c2）。清理后对应 payload 下载链接会失效。", + "cat.chat_uploads": "对话上传文件", + "catHint.chat_uploads": "用户在对话中上传的附件（chat_uploads/日期/会话）。", + "cat.workflow_checkpoints": "工作流检查点", + "catHint.workflow_checkpoints": "工作流运行断点文件（data/workflow-checkpoints），仅用于恢复中断的运行。", + "cat.diagnostic_logs": "诊断日志", + "catHint.diagnostic_logs": "按天轮转的诊断日志（log/diagnostic-*.log）。" + }, "settingsAudit": { "title": "日志审计", "description": "记录平台管理类操作（登录、配置、删除等），不记录对话正文、终端/WebShell 每次命令与工具调用明细。", diff --git a/web/static/js/settings.js b/web/static/js/settings.js index c30d1e870..10c57b3ed 100644 --- a/web/static/js/settings.js +++ b/web/static/js/settings.js @@ -688,6 +688,9 @@ function switchSettingsSection(section) { if (section === 'audit' && typeof initAuditLogsSection === 'function') { setTimeout(initAuditLogsSection, 0); } + if (section === 'storage' && typeof initStorageSection === 'function') { + setTimeout(initStorageSection, 0); + } } // 打开设置 diff --git a/web/static/js/storage.js b/web/static/js/storage.js new file mode 100644 index 000000000..b196dfc14 --- /dev/null +++ b/web/static/js/storage.js @@ -0,0 +1,367 @@ +// 运行空间占用统计与垃圾清理（系统设置 -> 存储清理） +// +// 数据来源： +// GET /api/storage/meta 清理策略与类别元信息 +// GET /api/storage/status 文件系统容量 + 各类别占用/可回收量 +// POST /api/storage/cleanup 预览（dry_run）或执行清理 +// PUT /api/config 保存策略（storage 段） +(function () { + 'use strict'; + + var meta = null; + var status = null; + var busy = false; + + function esc(v) { + var s = v == null ? '' : String(v); + if (typeof escapeHtml === 'function') return escapeHtml(s); + return s.replace(/[&<>"']/g, function (c) { + return { '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }[c]; + }); + } + + function st(key, fallback) { + if (typeof t === 'function') { + var v = t(key); + if (v && v !== key) return v; + } + return fallback; + } + + // 类别名称/说明来自后端（中文硬编码），优先走 i18n，缺失时回退服务端文案。 + function catLabel(cat) { + if (!cat) return ''; + return st('settingsStorage.cat.' + cat.key, cat.label || cat.key); + } + + function catHint(cat) { + if (!cat) return ''; + return st('settingsStorage.catHint.' + cat.key, cat.hint || ''); + } + + function fmtBytes(n) { + var v = Number(n); + if (!Number.isFinite(v) || v <= 0) return '0 B'; + var units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB']; + var i = 0; + while (v >= 1024 && i < units.length - 1) { + v /= 1024; + i++; + } + // 字节不给小数，其余保留一位；数值较大时省去小数避免噪声 + var digits = i === 0 ? 0 : (v >= 100 ? 0 : 1); + return v.toFixed(digits) + ' ' + units[i]; + } + + function setText(id, text) { + var el = document.getElementById(id); + if (el) el.textContent = text == null ? '' : String(text); + } + + function intOr(raw, fallback) { + var v = parseInt(raw, 10); + return Number.isFinite(v) && v >= 0 ? v : fallback; + } + + async function readErr(r, fallback) { + try { + var body = await r.json(); + return (body && body.error) || fallback; + } catch (_) { + return fallback; + } + } + + async function loadMeta() { + var r = await apiFetch('/api/storage/meta'); + if (!r.ok) throw new Error(await readErr(r, st('settingsStorage.loadMetaFailed', '获取清理策略失败'))); + meta = await r.json(); + } + + async function loadStatus(refresh) { + var r = await apiFetch('/api/storage/status' + (refresh ? '?refresh=1' : '')); + if (!r.ok) throw new Error(await readErr(r, st('settingsStorage.loadStatusFailed', '获取存储占用失败'))); + status = await r.json(); + } + + async function postCleanup(body) { + var r = await apiFetch('/api/storage/cleanup', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body) + }); + if (!r.ok) throw new Error(await readErr(r, st('settingsStorage.cleanupFailed', '清理请求失败'))); + return await r.json(); + } + + function render() { + renderOverview(); + renderRows(); + renderPolicy(); + showScannedAt(); + } + + function renderOverview() { + var ov = document.getElementById('storage-overview'); + if (!ov) return; + ov.hidden = false; + var fs = (status && status.filesystem) || {}; + var totals = (status && status.totals) || {}; + + setText('storage-disk-used', fs.available ? fmtBytes(fs.used_bytes) : st('settingsStorage.unavailable', '不可用')); + var fill = document.getElementById('storage-usage-bar-fill'); + if (fill) { + var pct = fs.available ? Math.max(0, Math.min(100, Number(fs.used_percent) || 0)) : 0; + fill.style.width = pct.toFixed(1) + '%'; + fill.classList.toggle('storage-usage-bar-fill--warn', pct >= 85); + } + setText('storage-disk-detail', fs.available + ? fmtBytes(fs.free_bytes) + ' ' + st('settingsStorage.free', '可用') + ' / ' + fmtBytes(fs.total_bytes) + : ''); + + setText('storage-total-bytes', fmtBytes(totals.bytes)); + setText('storage-total-units', (totals.units || 0) + ' ' + st('settingsStorage.itemsUnit', '项')); + setText('storage-reclaimable-bytes', fmtBytes(totals.reclaimable_bytes)); + setText('storage-reclaimable-units', (totals.reclaimable_units || 0) + ' ' + st('settingsStorage.itemsUnit', '项')); + + if (fs.available && fs.inodes_total > 0) { + var usedInodes = Math.max(0, fs.inodes_total - fs.inodes_free); + var inodePct = (usedInodes / fs.inodes_total) * 100; + setText('storage-inodes', usedInodes.toLocaleString() + ' / ' + fs.inodes_total.toLocaleString() + + ' (' + inodePct.toFixed(1) + '%)'); + } else { + setText('storage-inodes', st('settingsStorage.unavailable', '不可用')); + } + } + + function renderRows() { + var tbody = document.getElementById('storage-category-rows'); + if (!tbody) return; + var categories = (meta && meta.categories) || []; + if (!categories.length) { + tbody.innerHTML = '' + + esc(st('settingsStorage.noCategories', '暂无可清理类别')) + ''; + return; + } + var byKey = {}; + ((status && status.categories) || []).forEach(function (c) { byKey[c.key] = c; }); + + tbody.innerHTML = categories.map(function (m) { + var s = byKey[m.key] || {}; + var missing = !!s.missing; + var reclaimParts = []; + if (!missing) { + reclaimParts.push((s.reclaimable_units || 0) + ' ' + st('settingsStorage.itemsUnit', '项')); + if (s.orphan_units) { + reclaimParts.push(st('settingsStorage.orphanSuffix', '含孤儿') + ' ' + s.orphan_units); + } + if (s.skipped_active) { + reclaimParts.push(st('settingsStorage.skippedActive', '跳过活跃') + ' ' + s.skipped_active); + } + } + var retentionTitle = st('settingsStorage.defaultRetention', '默认') + ' ' + + (m.default_retention != null ? m.default_retention : '-') + ' ' + + st('settingsStorage.daysUnit', '天'); + return '' + + '' + esc(catLabel(m)) + '' + + '' + (missing + ? '' + esc(st('settingsStorage.unused', '未使用')) + '' + : '' + esc(shortRoot(s.root)) + '') + '' + + '' + (missing ? '—' : (s.units || 0)) + '' + + '' + (missing ? '—' : fmtBytes(s.bytes)) + '' + + '' + (missing ? '—' : '' + fmtBytes(s.reclaimable_bytes) + '' + + '' + esc(reclaimParts.join(' · ')) + '') + '' + + '' + + '' + + ''; + }).join(''); + } + + // 目录列只展示末两级，完整路径放在 title 里，避免长绝对路径把表格撑破。 + function shortRoot(root) { + var s = String(root || ''); + if (!s) return ''; + var parts = s.split('/').filter(Boolean); + if (parts.length <= 2) return s; + return '…/' + parts.slice(-2).join('/'); + } + + function renderPolicy() { + if (!meta) return; + var auto = document.getElementById('storage-auto-clean'); + if (auto) auto.checked = !!meta.auto_clean; + setNumber('storage-interval-minutes', meta.interval_minutes); + setNumber('storage-orphan-grace-days', meta.orphan_grace_days); + setNumber('storage-active-grace-hours', meta.active_grace_hours); + } + + function setNumber(id, value) { + var el = document.getElementById(id); + if (el) el.value = value != null ? value : ''; + } + + function showScannedAt() { + var el = document.getElementById('storage-scanned-at'); + if (!el) return; + if (!status || !status.scanned_at) { + el.textContent = ''; + return; + } + var d = new Date(status.scanned_at); + el.textContent = st('settingsStorage.scannedAt', '扫描于') + ' ' + + (isNaN(d.getTime()) ? String(status.scanned_at) : d.toLocaleString()); + } + + function renderResult(rep, isPreview) { + var el = document.getElementById('storage-result'); + if (!el) return; + var totals = (rep && rep.totals) || {}; + var head = isPreview + ? st('settingsStorage.previewTitle', '预览结果（未删除任何文件）') + : st('settingsStorage.cleanDone', '清理完成'); + var summary = isPreview + ? fmtBytes(totals.reclaimable_bytes) + ' / ' + (totals.reclaimable_units || 0) + ' ' + st('settingsStorage.itemsUnit', '项') + : fmtBytes(totals.freed_bytes) + ' / ' + (totals.removed_units || 0) + ' ' + st('settingsStorage.itemsUnit', '项'); + + var rows = ((rep && rep.categories) || []) + .filter(function (c) { + return isPreview ? (c.reclaimable_units || 0) > 0 : ((c.removed_units || 0) > 0 || (c.errors || []).length > 0); + }) + .map(function (c) { + var main = isPreview + ? fmtBytes(c.reclaimable_bytes) + ' / ' + (c.reclaimable_units || 0) + ' ' + st('settingsStorage.itemsUnit', '项') + : fmtBytes(c.freed_bytes) + ' / ' + (c.removed_units || 0) + ' ' + st('settingsStorage.itemsUnit', '项'); + var errs = (c.errors || []).slice(0, 3).map(esc).join('
'); + return '
  • ' + esc(catLabel(c)) + '' + esc(main) + + (errs ? '
    ' + errs + '' : '') + + '
    • '; + }); + + var html = '
    ' + esc(head) + '' + esc(summary) + '
    '; + html += rows.length + ? '
      ' + rows.join('') + '
    ' + : '

    ' + esc(st('settingsStorage.nothingToClean', '当前没有符合清理条件的内容。')) + '

    '; + if (totals.skipped_active) { + html += '

    ' + esc(st('settingsStorage.skippedActiveNote', '已跳过最近仍在活动的会话：') + + totals.skipped_active + ' ' + st('settingsStorage.itemsUnit', '项')) + '

    '; + } + el.className = 'storage-result'; + el.innerHTML = html; + el.hidden = false; + } + + function showError(message) { + var el = document.getElementById('storage-result'); + if (!el) return; + el.className = 'storage-result storage-result--error'; + el.innerHTML = '
    ' + + esc(st('settingsStorage.failed', '操作失败')) + '' + esc(message) + '
    '; + el.hidden = false; + } + + function setButtonsDisabled(disabled) { + var section = document.getElementById('settings-section-storage'); + if (!section) return; + section.querySelectorAll('button').forEach(function (btn) { btn.disabled = disabled; }); + } + + async function withBusy(fn) { + if (busy) return; + busy = true; + setButtonsDisabled(true); + try { + await fn(); + } catch (e) { + showError(e && e.message ? e.message : String(e)); + } finally { + busy = false; + setButtonsDisabled(false); + } + } + + function collectPolicy() { + var payload = { + auto_clean: !!(document.getElementById('storage-auto-clean') || {}).checked, + interval_minutes: intOr(document.getElementById('storage-interval-minutes') && document.getElementById('storage-interval-minutes').value, 5), + orphan_grace_days: intOr(document.getElementById('storage-orphan-grace-days') && document.getElementById('storage-orphan-grace-days').value, 0), + active_grace_hours: intOr(document.getElementById('storage-active-grace-hours') && document.getElementById('storage-active-grace-hours').value, 1), + categories: {} + }; + // 先用服务端元信息铺底，保证未渲染/缺字段时不会把既有策略清成默认值。 + ((meta && meta.categories) || []).forEach(function (m) { + payload.categories[m.key] = { enabled: !!m.enabled, retention_days: intOr(m.retention_days, 0) }; + }); + document.querySelectorAll('#storage-category-rows .storage-retention-input').forEach(function (el) { + var key = el.getAttribute('data-key'); + if (!key || !payload.categories[key]) return; + payload.categories[key].retention_days = intOr(el.value, 0); + }); + document.querySelectorAll('#storage-category-rows .storage-enabled-input').forEach(function (el) { + var key = el.getAttribute('data-key'); + if (!key || !payload.categories[key]) return; + payload.categories[key].enabled = !!el.checked; + }); + return payload; + } + + window.initStorageSection = async function () { + if (typeof apiFetch !== 'function') return; + if (!document.getElementById('storage-category-rows')) return; + await withBusy(async function () { + await loadMeta(); + await loadStatus(false); + render(); + }); + }; + + window.refreshStorageStatus = async function (force) { + if (typeof apiFetch !== 'function') return; + await withBusy(async function () { + await loadStatus(force === true); + render(); + }); + }; + + window.previewStorageCleanup = async function () { + await withBusy(async function () { + var rep = await postCleanup({ dry_run: true }); + renderResult(rep, true); + }); + }; + + window.runStorageCleanup = async function () { + var totals = (status && status.totals) || {}; + var message = st('settingsStorage.confirmClean', '将永久删除约 {size}（{count} 项）运行空间文件，无法恢复。建议先执行「预览可清理项」。确认继续？') + .replace('{size}', fmtBytes(totals.reclaimable_bytes)) + .replace('{count}', String(totals.reclaimable_units || 0)); + if (!window.confirm(message)) return; + await withBusy(async function () { + // 服务端要求真实删除必须同时带 dry_run=false 与 confirm=true。 + var rep = await postCleanup({ dry_run: false, confirm: true }); + renderResult(rep, false); + await loadStatus(true); + render(); + }); + }; + + window.saveStorageSettings = async function () { + var payload = collectPolicy(); + await withBusy(async function () { + // 只调 PUT /api/config：UpdateConfig 会合并 storage 段并写回 config.yaml， + // 清理器每次执行都实时读取配置，无需 /api/config/apply 触发重启类副作用。 + var r = await apiFetch('/api/config', { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ storage: payload }) + }); + if (!r.ok) throw new Error(await readErr(r, st('settingsStorage.saveFailed', '保存失败'))); + await loadMeta(); + renderPolicy(); + renderRows(); + window.alert(st('settingsStorage.saved', '清理策略已保存')); + }); + }; +})(); diff --git a/web/templates/index.html b/web/templates/index.html index c7f8c4347..8b94a0f42 100644 --- a/web/templates/index.html +++ b/web/templates/index.html @@ -3598,6 +3598,9 @@

    系统设置

    日志审计
    +
    + 存储清理 +
    @@ -5067,6 +5070,92 @@

    修改密码

    + +
    +
    +

    存储清理

    +
    +

    统计并清理运行期间产生的磁盘垃圾（Agent 工作区、工具输出缓存、C2 产物、对话上传件等）。删除不可逆，建议先预览再执行。

    + + + +
    + + + + +
    + + + +
    + + + + + + + + + + + + + + + +
    类别目录项目数占用可回收保留天数启用
    加载中…
    +
    + +
    +

    自动清理策略

    +
    + + + + +
    +

    保留天数为 0 表示不按保留期清理，但会话已删除的孤儿目录仍会回收；最近有活动的会话在任何情况下都不会被清理。默认关闭自动清理，升级不会自动删除既有数据。

    +
    + +
    +
    +
    + @@ -6816,6 +6905,7 @@

    已绑定平台账号

    +