From 055089304bf3213fbcb47ed7f0bd499229795fbe Mon Sep 17 00:00:00 2001 From: mahoshojoHCG Date: Sat, 29 Aug 2026 23:34:06 +0800 Subject: [PATCH 1/2] feat: add durable job runtime reliability --- .../Engines/CodexAppServerEngine.cs | 32 +- .../ChatController.cs | 8 +- README.md | 6 + .../mock-server.mjs | 43 + .../src/i18n/locales/en/tasks.json | 31 + .../src/i18n/locales/ja/tasks.json | 31 + .../src/i18n/locales/zh-CN/tasks.json | 31 + .../src/pages/TasksPage.tsx | 141 ++- .../src/tasks/hooks.ts | 7 +- .../src/tasks/types.ts | 19 + .../src/tasks/utils.ts | 11 + .../DataRepository/IDurableJobRepository.cs | 110 ++ .../DataRepository/IReadinessRepository.cs | 6 + .../IScheduledTaskLeaseRepository.cs | 28 + .../IDownloadCompletionNotifier.cs | 15 + .../PluginParams/FileDownloadCompleteParam.cs | 14 +- .../Tasks/IScheduledTaskLeaseManager.cs | 23 + .../Tasks/ScheduledTaskBase.cs | 172 ++- .../Health/HealthEndpointTests.cs | 47 + .../FileMappingRepositoryPostgreSqlTests.cs | 193 +++ .../WebDavWebApplicationFactory.cs | 8 + .../CodexAppServerEngineTests.cs | 26 + .../CompleteDownloadBackgroundServiceTests.cs | 310 +++-- .../DurableJobsControllerTests.cs | 72 ++ .../ObservabilityTests.cs | 96 ++ .../ScheduledTaskBaseTests.cs | 168 +++ .../Controllers/DurableJobsController.cs | 120 ++ .../External/AppJsonSerializerContext.cs | 4 + .../Controllers/External/DurableJob.cs | 22 + ...27_AddDurableJobsAndTaskLeases.Designer.cs | 1086 +++++++++++++++++ ...60829145227_AddDurableJobsAndTaskLeases.cs | 79 ++ .../ApplicationContextModelSnapshot.cs | 103 ++ .../Models/ApplicationContext.cs | 55 + .../Models/DurableJob.cs | 22 + .../Models/ScheduledTaskState.cs | 13 + .../DurableJobMetricsBackgroundService.cs | 37 + .../Observability/ReadinessHealthChecks.cs | 111 ++ .../Observability/RuntimeTelemetry.cs | 143 +++ .../SensitiveTagRedactionProcessor.cs | 49 + SecondDimensionWatcherReDive/Program.cs | 165 ++- .../Repositories/AnimationInfoRepository.cs | 31 +- .../Repositories/DurableJobRepository.cs | 218 ++++ ...eMappingRepositoryPostgreSqlTestFixture.cs | 198 ++- .../Repositories/ReadinessRepository.cs | 11 + .../ScheduledTaskLeaseRepository.cs | 95 ++ .../SecondDimensionWatcherReDive.csproj | 7 + .../CompleteDownloadBackgroundService.cs | 392 ++++-- .../FetchRemoteTorrentBackgroundService.cs | 111 +- .../Services/MediaLibraryScanQueue.cs | 10 +- .../NullDownloadCompletionNotifier.cs | 12 + .../PostgresScheduledTaskLeaseManager.cs | 146 +++ .../ScheduledTaskBackgroundService.cs | 56 +- .../RemoteTorrentDownloadClient.cs | 6 +- .../appsettings.example.json | 15 + docs/runtime-reliability.md | 66 + packaging/appsettings.yml | 11 + 56 files changed, 4751 insertions(+), 291 deletions(-) create mode 100644 SecondDimensionWatcherReDive.Framework/DataRepository/IDurableJobRepository.cs create mode 100644 SecondDimensionWatcherReDive.Framework/DataRepository/IReadinessRepository.cs create mode 100644 SecondDimensionWatcherReDive.Framework/DataRepository/IScheduledTaskLeaseRepository.cs create mode 100644 SecondDimensionWatcherReDive.Framework/Notifications/IDownloadCompletionNotifier.cs create mode 100644 SecondDimensionWatcherReDive.Framework/Tasks/IScheduledTaskLeaseManager.cs create mode 100644 SecondDimensionWatcherReDive.IntegrationTest/Health/HealthEndpointTests.cs create mode 100644 SecondDimensionWatcherReDive.Test/DurableJobsControllerTests.cs create mode 100644 SecondDimensionWatcherReDive.Test/ObservabilityTests.cs create mode 100644 SecondDimensionWatcherReDive.Test/ScheduledTaskBaseTests.cs create mode 100644 SecondDimensionWatcherReDive/Controllers/DurableJobsController.cs create mode 100644 SecondDimensionWatcherReDive/Controllers/External/DurableJob.cs create mode 100644 SecondDimensionWatcherReDive/Migrations/20260829145227_AddDurableJobsAndTaskLeases.Designer.cs create mode 100644 SecondDimensionWatcherReDive/Migrations/20260829145227_AddDurableJobsAndTaskLeases.cs create mode 100644 SecondDimensionWatcherReDive/Models/DurableJob.cs create mode 100644 SecondDimensionWatcherReDive/Models/ScheduledTaskState.cs create mode 100644 SecondDimensionWatcherReDive/Observability/DurableJobMetricsBackgroundService.cs create mode 100644 SecondDimensionWatcherReDive/Observability/ReadinessHealthChecks.cs create mode 100644 SecondDimensionWatcherReDive/Observability/RuntimeTelemetry.cs create mode 100644 SecondDimensionWatcherReDive/Observability/SensitiveTagRedactionProcessor.cs create mode 100644 SecondDimensionWatcherReDive/Repositories/DurableJobRepository.cs create mode 100644 SecondDimensionWatcherReDive/Repositories/ReadinessRepository.cs create mode 100644 SecondDimensionWatcherReDive/Repositories/ScheduledTaskLeaseRepository.cs create mode 100644 SecondDimensionWatcherReDive/Services/NullDownloadCompletionNotifier.cs create mode 100644 SecondDimensionWatcherReDive/Services/PostgresScheduledTaskLeaseManager.cs create mode 100644 docs/runtime-reliability.md diff --git a/Plugins/SecondDimensionWatcherReDive.AI/Engines/CodexAppServerEngine.cs b/Plugins/SecondDimensionWatcherReDive.AI/Engines/CodexAppServerEngine.cs index 2735fe5..63d0ccb 100644 --- a/Plugins/SecondDimensionWatcherReDive.AI/Engines/CodexAppServerEngine.cs +++ b/Plugins/SecondDimensionWatcherReDive.AI/Engines/CodexAppServerEngine.cs @@ -226,14 +226,14 @@ await rpc.SendRequestAsync( turnId = GetRequiredNestedString(turnResult, "turn", "id"); state.SetTurnId(turnId); - while (state.Updates.TryDequeue(out var bufferedUpdate)) + while (state.TryDequeueUpdate(out var bufferedUpdate)) yield return bufferedUpdate; while (!state.IsTerminal) { var message = await rpc.ReceiveAsync(cancellationToken); await ProcessTurnMessageAsync(rpc, state, message, cancellationToken); - while (state.Updates.TryDequeue(out var update)) + while (state.TryDequeueUpdate(out var update)) yield return update; } @@ -488,8 +488,8 @@ await rpc.SendErrorAsync(requestId, -32602, } var argumentsJson = arguments.GetRawText(); - state.Updates.Enqueue(new ToolCallBegin(callId, toolName)); - state.Updates.Enqueue(new ToolCallDelta(callId, argumentsJson)); + state.TryEnqueueUpdate(new ToolCallBegin(callId, toolName)); + state.TryEnqueueUpdate(new ToolCallDelta(callId, argumentsJson)); IToolResult toolResult; var executionCanceled = false; @@ -516,7 +516,7 @@ await rpc.SendErrorAsync(requestId, -32602, var serializedResult = JsonSerializer.SerializeToElement( toolResult, toolResult.GetType(), ToolJsonOptions.Options); - state.Updates.Enqueue(new ToolResultUpdate(callId, serializedResult)); + state.TryEnqueueUpdate(new ToolResultUpdate(callId, serializedResult)); if (executionCanceled || cancellationToken.IsCancellationRequested) { @@ -827,17 +827,18 @@ private sealed class TurnState( IToolExecutor? toolExecutor, int maxDynamicToolCalls) { + private const int MaxBufferedUpdates = 1024; private readonly Dictionary _finalAgentTexts = new(StringComparer.Ordinal); private readonly HashSet _completedFinalAgentItems = new(StringComparer.Ordinal); private readonly HashSet _toolCallIds = new(StringComparer.Ordinal); + private readonly Queue _updates = new(); private int _dynamicToolCallCount; public string ThreadId { get; } = threadId; public string? TurnId { get; private set; } public IToolExecutor? ToolExecutor { get; } = toolExecutor; - public Queue Updates { get; } = new(); public bool IsTerminal { get; private set; } public bool ServerTerminalReceived { get; private set; } public string? Status { get; private set; } @@ -854,7 +855,7 @@ public void AppendFinalAgentDelta(string itemId, string delta) _finalAgentTexts[itemId] = accumulated + delta; if (delta.Length > 0) - Updates.Enqueue(new TextDelta(delta)); + TryEnqueueUpdate(new TextDelta(delta)); } public void CompleteFinalAgentItem(string itemId, string authoritativeText) @@ -870,9 +871,24 @@ public void CompleteFinalAgentItem(string itemId, string authoritativeText) var suffix = authoritativeText[accumulated.Length..]; _finalAgentTexts[itemId] = authoritativeText; if (suffix.Length > 0) - Updates.Enqueue(new TextDelta(suffix)); + TryEnqueueUpdate(new TextDelta(suffix)); } + public bool TryEnqueueUpdate(IChatUpdate update) + { + if (_updates.Count < MaxBufferedUpdates) + { + _updates.Enqueue(update); + return true; + } + + Fail($"Codex app-server buffered update limit ({MaxBufferedUpdates}) was exceeded."); + return false; + } + + public bool TryDequeueUpdate(out IChatUpdate update) => + _updates.TryDequeue(out update!); + public bool TryBeginToolCall(string callId, out string rejectionReason) { if (!_toolCallIds.Add(callId)) diff --git a/Plugins/SecondDimensionWatcherReDive.Chat/ChatController.cs b/Plugins/SecondDimensionWatcherReDive.Chat/ChatController.cs index 203ffc1..7420380 100644 --- a/Plugins/SecondDimensionWatcherReDive.Chat/ChatController.cs +++ b/Plugins/SecondDimensionWatcherReDive.Chat/ChatController.cs @@ -190,7 +190,13 @@ private async IAsyncEnumerable> StreamChatEvents( string? model, [EnumeratorCancellation] CancellationToken cancellationToken) { - var channel = Channel.CreateUnbounded>(); + var channel = Channel.CreateBounded>( + new BoundedChannelOptions(256) + { + SingleReader = true, + SingleWriter = true, + FullMode = BoundedChannelFullMode.Wait + }); // Producer: runs AI chat streaming in background, writes SSE items to channel. // Keep the task and await it during iterator disposal so a disconnected request cannot diff --git a/README.md b/README.md index a55a5f4..b6e0b47 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,8 @@ - [x] 按动画分组的主页展示（卡片 + 剧集列表） - [x] 当季番组发现（mikanani.me 爬取）+ 一键订阅 - [x] 后台任务仪表盘（查看状态、手动触发） +- [x] PostgreSQL 持久任务 / Outbox（崩溃恢复、指数重试、死信处理、多实例租约） +- [x] 存活与就绪探针、Prometheus 指标及 OpenTelemetry 链路 - [x] 一次性数据迁移框架（`MigrationMarkers` 表幂等记录） - [x] 插件事件系统（下载前 / 下载完成后钩子） - [x] 多语言界面（简体中文 / English / 日本語） @@ -105,11 +107,15 @@ bash <(curl -fsSL https://raw.githubusercontent.com/HCGStudio/SecondDimensionWat | `AI:CodexAppServer:Endpoint` / `BearerToken` / `Model` / `PermissionProfile` / `TimeoutSeconds` | Codex app-server WebSocket 端点；空模型使用服务端默认模型；权限配置默认 `:read-only`，也可填写管理员定义的 profile id | | `Inference:RateLimitDelayMs` | 推断 API 调用最小间隔（毫秒，默认 1000） | | `Valkey:ConnectionString` | Valkey / Redis 连接（可选；为空则使用内存缓存） | +| `Health:ValkeyRequired` / `QbittorrentRequired` / `StorageRequired` / `AIRequired` | `/health/ready` 的依赖要求；PostgreSQL 始终必需，AI 默认不阻塞就绪 | +| `OpenTelemetry:OtlpEndpoint` | 可选 OTLP Collector 地址；Prometheus `/metrics` 无需配置即启用 | > 使用现有媒体库导入前，必须至少配置一个 `MediaLibrary:AllowedRoots`。导入源必须位于白名单内，且不能与 `FileStore:Local` 管理的下载目录相同、互为父目录或以其他方式重叠。导入与后续对账只会修改数据库中的媒体记录和虚拟路径映射；系统绝不会移动、重命名或删除原文件。短暂缺失的条目会先撤下映射并保留观看/审核记录，超过 `MissingGracePeriod`（默认 24 小时）后才清理数据库记录。 > 从 v2.2 之前升级：旧的 `Inference:ApiKey/Provider/Model` 已迁移到 `AI:` 前缀。运行 `deployments/migrate-config.sh` 自动迁移；包管理器安装时 `postinstall.sh` 会自动执行。 +运行探针、持久任务恢复、死信操作和遥测标签约束详见 [运行可靠性与可观测性](docs/runtime-reliability.md)。 + ### 网页运行时设置 登录后打开「设置」，可修改 AI 执行模式与 Provider、AI/TMDB 密钥、qBittorrent、媒体库扫描、异常检测和 NFS。保存值存入 PostgreSQL，并覆盖部署文件或环境变量中的默认值；密钥和密码使用持久化 Data Protection 密钥环加密，API 不会回显明文。可对单个敏感项选择保留、替换、清除或恢复部署默认值。 diff --git a/SecondDimensionWatcherReDive.Client/mock-server.mjs b/SecondDimensionWatcherReDive.Client/mock-server.mjs index f5ee4d4..9c7c57f 100644 --- a/SecondDimensionWatcherReDive.Client/mock-server.mjs +++ b/SecondDimensionWatcherReDive.Client/mock-server.mjs @@ -2589,10 +2589,53 @@ async function route(method, pathname, searchParams, req, res) { }, ]; + const mockDurableJobs = + globalThis._mockDurableJobs ?? + (globalThis._mockDurableJobs = [ + { + id: randomUUID(), + type: "downloadCompletion", + status: "deadLetter", + stage: "mapFiles", + attemptCount: 8, + createdAt: new Date(Date.now() - 3_600_000).toISOString(), + updatedAt: new Date(Date.now() - 60_000).toISOString(), + nextAttemptAt: new Date(Date.now() - 60_000).toISOString(), + lastAttemptAt: new Date(Date.now() - 60_000).toISOString(), + completedAt: null, + lastError: "InvalidOperationException: No file mapping could be produced.", + }, + ]); + if (method === "GET" && pathname === "/api/tasks") { return json(res, MOCK_TASKS); } + if (method === "GET" && pathname === "/api/jobs") { + const status = searchParams.get("status"); + const items = status + ? mockDurableJobs.filter( + (job) => job.status.toLowerCase() === status.toLowerCase(), + ) + : mockDurableJobs; + return json(res, { items, totalCount: items.length }); + } + + if ( + method === "POST" && + (pathname === "/api/jobs/retry" || pathname === "/api/jobs/resolve") + ) { + const body = await readBody(req); + const ids = new Set(Array.isArray(body.ids) ? body.ids : []); + let affectedCount = 0; + for (let index = mockDurableJobs.length - 1; index >= 0; index--) { + if (!ids.has(mockDurableJobs[index].id)) continue; + mockDurableJobs.splice(index, 1); + affectedCount++; + } + return json(res, { affectedCount }); + } + // POST /api/tasks/:id/run { const m = pathname.match(/^\/api\/tasks\/(.+)\/run$/); diff --git a/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/tasks.json b/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/tasks.json index b00d2b8..551e15c 100644 --- a/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/tasks.json +++ b/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/tasks.json @@ -29,6 +29,37 @@ "success": "Task \"{{name}}\" finished", "failure": "Task \"{{name}}\" failed" }, + "deadLetters": { + "title": "Failed durable jobs", + "description": "Jobs that exhausted automatic retries. Retry them after fixing the cause, or mark them handled.", + "empty": "No durable jobs require attention.", + "loadFailed": "Could not load failed durable jobs.", + "retry": "Retry", + "resolve": "Mark handled", + "columns": { + "type": "Job", + "stage": "Failed stage", + "attempts": "Attempts", + "updated": "Last attempt", + "error": "Last error", + "actions": "Actions" + }, + "types": { + "downloadCompletion": "Download completion" + }, + "stages": { + "mapFiles": "Map files", + "notify": "Notify", + "invokePlugins": "Invoke plugins", + "done": "Finalize" + }, + "toast": { + "retrySuccess": "Job queued for retry", + "retryFailure": "Could not retry the job", + "resolveSuccess": "Job marked handled", + "resolveFailure": "Could not mark the job handled" + } + }, "metadata": { "SyncFeed": { "name": "SyncFeed", diff --git a/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/tasks.json b/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/tasks.json index 1112f71..9ef237f 100644 --- a/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/tasks.json +++ b/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/tasks.json @@ -26,6 +26,37 @@ "success": "タスク「{{name}}」を完了しました", "failure": "タスク「{{name}}」が失敗しました" }, + "deadLetters": { + "title": "失敗した永続ジョブ", + "description": "自動再試行を使い切ったジョブです。原因を修正して再試行するか、処理済みにしてください。", + "empty": "対応が必要な永続ジョブはありません。", + "loadFailed": "失敗した永続ジョブを読み込めませんでした。", + "retry": "再試行", + "resolve": "処理済みにする", + "columns": { + "type": "ジョブ", + "stage": "失敗ステージ", + "attempts": "試行回数", + "updated": "最終試行", + "error": "最終エラー", + "actions": "操作" + }, + "types": { + "downloadCompletion": "ダウンロード完了処理" + }, + "stages": { + "mapFiles": "ファイルのマッピング", + "notify": "通知", + "invokePlugins": "プラグインの呼び出し", + "done": "完了" + }, + "toast": { + "retrySuccess": "ジョブを再試行キューに追加しました", + "retryFailure": "ジョブを再試行できませんでした", + "resolveSuccess": "ジョブを処理済みにしました", + "resolveFailure": "ジョブを処理済みにできませんでした" + } + }, "metadata": { "SyncFeed": { "name": "SyncFeed", diff --git a/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/tasks.json b/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/tasks.json index fad79a7..b0ab651 100644 --- a/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/tasks.json +++ b/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/tasks.json @@ -26,6 +26,37 @@ "success": "任务「{{name}}」执行完成", "failure": "任务「{{name}}」执行失败" }, + "deadLetters": { + "title": "失败的持久任务", + "description": "这些任务已耗尽自动重试次数。修复原因后可重新执行，或将其标记为已处理。", + "empty": "当前没有需要处理的持久任务。", + "loadFailed": "无法加载失败的持久任务。", + "retry": "重试", + "resolve": "标记已处理", + "columns": { + "type": "任务", + "stage": "失败阶段", + "attempts": "尝试次数", + "updated": "最后尝试", + "error": "最后错误", + "actions": "操作" + }, + "types": { + "downloadCompletion": "下载完成处理" + }, + "stages": { + "mapFiles": "建立文件映射", + "notify": "发送通知", + "invokePlugins": "调用插件", + "done": "完成" + }, + "toast": { + "retrySuccess": "任务已加入重试队列", + "retryFailure": "无法重试该任务", + "resolveSuccess": "任务已标记为处理完成", + "resolveFailure": "无法标记该任务" + } + }, "metadata": { "SyncFeed": { "name": "SyncFeed", diff --git a/SecondDimensionWatcherReDive.Client/src/pages/TasksPage.tsx b/SecondDimensionWatcherReDive.Client/src/pages/TasksPage.tsx index 8e0626b..e23ed81 100644 --- a/SecondDimensionWatcherReDive.Client/src/pages/TasksPage.tsx +++ b/SecondDimensionWatcherReDive.Client/src/pages/TasksPage.tsx @@ -1,16 +1,23 @@ -import { AlertTriangle, Loader2, Play } from "lucide-react"; import React from "react"; import { useTranslation } from "react-i18next"; +import { + AlertTriangle, + CheckCircle2, + Loader2, + Play, + RotateCcw, +} from "lucide-react"; + import { useToast } from "../components/ToastProvider"; import { Button } from "../components/ui/Button"; import { EmptyPrompt } from "../components/ui/EmptyPrompt"; import { Spinner } from "../components/ui/Spinner"; import { Table, type TableColumn } from "../components/ui/Table"; -import { useTasks } from "../tasks/hooks"; +import { useDeadLetterJobs, useTasks } from "../tasks/hooks"; import { useTaskMetadata } from "../tasks/taskMetadata"; -import { runTask } from "../tasks/utils"; -import { ITask } from "../tasks/types"; +import { IDurableJob, ITask } from "../tasks/types"; +import { resolveJobs, retryJobs, runTask } from "../tasks/utils"; import { PageTemplate } from "./PageTemplate"; function useFormatInterval(): (interval: string) => string { @@ -43,8 +50,18 @@ export const TasksPage: React.FC = () => { const getTaskMetadata = useTaskMetadata(); const formatInterval = useFormatInterval(); const { data: tasks, error, mutate } = useTasks(); + const { + data: deadLetters, + error: deadLetterError, + mutate: mutateDeadLetters, + } = useDeadLetterJobs(); const { addToast } = useToast(); - const [runningTasks, setRunningTasks] = React.useState>(new Set()); + const [runningTasks, setRunningTasks] = React.useState>( + new Set(), + ); + const [mutatingJobs, setMutatingJobs] = React.useState>( + new Set(), + ); const onRun = React.useCallback( async (id: string) => { @@ -72,6 +89,36 @@ export const TasksPage: React.FC = () => { [mutate, addToast, t, getTaskMetadata], ); + const mutateJob = React.useCallback( + async (job: IDurableJob, action: "retry" | "resolve") => { + setMutatingJobs((previous) => new Set(previous).add(job.id)); + try { + const result = + action === "retry" + ? await retryJobs([job.id]) + : await resolveJobs([job.id]); + if (result.affectedCount !== 1) throw new Error("job state changed"); + await mutateDeadLetters(); + addToast({ + title: t(`tasks:deadLetters.toast.${action}Success`), + color: "success", + }); + } catch { + addToast({ + title: t(`tasks:deadLetters.toast.${action}Failure`), + color: "danger", + }); + } finally { + setMutatingJobs((previous) => { + const next = new Set(previous); + next.delete(job.id); + return next; + }); + } + }, + [addToast, mutateDeadLetters, t], + ); + const columns: TableColumn[] = [ { name: t("tasks:columns.name"), @@ -79,7 +126,8 @@ export const TasksPage: React.FC = () => { }, { name: t("tasks:columns.description"), - render: (_value: any, item: ITask) => getTaskMetadata(item.id).description, + render: (_value: any, item: ITask) => + getTaskMetadata(item.id).description, }, { field: "interval", @@ -122,6 +170,65 @@ export const TasksPage: React.FC = () => { }, ]; + const deadLetterColumns: TableColumn[] = [ + { + field: "type", + name: t("tasks:deadLetters.columns.type"), + render: () => t("tasks:deadLetters.types.downloadCompletion"), + }, + { + field: "stage", + name: t("tasks:deadLetters.columns.stage"), + render: (value: IDurableJob["stage"]) => + t(`tasks:deadLetters.stages.${value}`), + }, + { + field: "attemptCount", + name: t("tasks:deadLetters.columns.attempts"), + }, + { + field: "updatedAt", + name: t("tasks:deadLetters.columns.updated"), + render: (value: string) => new Date(value).toLocaleString(), + }, + { + field: "lastError", + name: t("tasks:deadLetters.columns.error"), + render: (value: string | null) => value ?? "-", + truncateText: true, + }, + { + name: t("tasks:deadLetters.columns.actions"), + render: (_value: unknown, item: IDurableJob) => { + const disabled = mutatingJobs.has(item.id); + return ( +
+ + +
+ ); + }, + width: "230px", + }, + ]; + return (

@@ -145,6 +252,28 @@ export const TasksPage: React.FC = () => { body={

{t("tasks:empty.body")}

} /> )} + +

+ {t("tasks:deadLetters.title")} +

+

+ {t("tasks:deadLetters.description")} +

+ {deadLetterError ? ( +

+ {t("tasks:deadLetters.loadFailed")} +

+ ) : !deadLetters ? ( +
+ +
+ ) : deadLetters.items.length > 0 ? ( + + ) : ( +

+ {t("tasks:deadLetters.empty")} +

+ )} ); }; diff --git a/SecondDimensionWatcherReDive.Client/src/tasks/hooks.ts b/SecondDimensionWatcherReDive.Client/src/tasks/hooks.ts index cacb821..f8361d9 100644 --- a/SecondDimensionWatcherReDive.Client/src/tasks/hooks.ts +++ b/SecondDimensionWatcherReDive.Client/src/tasks/hooks.ts @@ -1,7 +1,12 @@ import useSWR from "swr"; import fetcher from "../auth/httpClient"; -import { ITask } from "./types"; +import { IDurableJobPage, ITask } from "./types"; export const useTasks = () => useSWR("/api/tasks", fetcher, { refreshInterval: 3000 }); + +export const useDeadLetterJobs = () => + useSWR("/api/jobs?status=deadLetter&take=100", fetcher, { + refreshInterval: 5000, + }); diff --git a/SecondDimensionWatcherReDive.Client/src/tasks/types.ts b/SecondDimensionWatcherReDive.Client/src/tasks/types.ts index f065b84..2f89d3f 100644 --- a/SecondDimensionWatcherReDive.Client/src/tasks/types.ts +++ b/SecondDimensionWatcherReDive.Client/src/tasks/types.ts @@ -5,3 +5,22 @@ export interface ITask { lastRunAt: string | null; isRunning: boolean; } + +export interface IDurableJob { + id: string; + type: "downloadCompletion"; + status: "deadLetter"; + stage: "mapFiles" | "notify" | "invokePlugins" | "done"; + attemptCount: number; + createdAt: string; + updatedAt: string; + nextAttemptAt: string; + lastAttemptAt: string | null; + completedAt: string | null; + lastError: string | null; +} + +export interface IDurableJobPage { + items: IDurableJob[]; + totalCount: number; +} diff --git a/SecondDimensionWatcherReDive.Client/src/tasks/utils.ts b/SecondDimensionWatcherReDive.Client/src/tasks/utils.ts index 0c18130..aa4ec30 100644 --- a/SecondDimensionWatcherReDive.Client/src/tasks/utils.ts +++ b/SecondDimensionWatcherReDive.Client/src/tasks/utils.ts @@ -3,3 +3,14 @@ import fetcher from "../auth/httpClient"; export const runTask = async (id: string) => { return await fetcher(`/api/tasks/${id}/run`, { method: "POST" }); }; + +const mutateJobs = async (action: "retry" | "resolve", ids: string[]) => + await fetcher<{ affectedCount: number }>(`/api/jobs/${action}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ ids }), + }); + +export const retryJobs = async (ids: string[]) => mutateJobs("retry", ids); + +export const resolveJobs = async (ids: string[]) => mutateJobs("resolve", ids); diff --git a/SecondDimensionWatcherReDive.Framework/DataRepository/IDurableJobRepository.cs b/SecondDimensionWatcherReDive.Framework/DataRepository/IDurableJobRepository.cs new file mode 100644 index 0000000..7a496f0 --- /dev/null +++ b/SecondDimensionWatcherReDive.Framework/DataRepository/IDurableJobRepository.cs @@ -0,0 +1,110 @@ +namespace SecondDimensionWatcherReDive.Framework.DataRepository; + +public enum DurableJobType +{ + DownloadCompletion +} + +public enum DurableJobStatus +{ + Pending, + Processing, + Completed, + DeadLetter, + Resolved +} + +public enum DurableJobStage +{ + MapFiles, + Notify, + InvokePlugins, + Done +} + +public sealed record DurableJob( + Guid Id, + string DeduplicationKey, + DurableJobType Type, + DurableJobStatus Status, + DurableJobStage Stage, + string PayloadJson, + int AttemptCount, + DateTimeOffset CreatedAt, + DateTimeOffset UpdatedAt, + DateTimeOffset NextAttemptAt, + DateTimeOffset? LastAttemptAt, + DateTimeOffset? CompletedAt, + string? LeaseOwner, + DateTimeOffset? LeaseExpiresAt, + string? LastError); + +public sealed record DurableJobPage( + IReadOnlyList Items, + int TotalCount); + +public sealed record DurableJobStatistics( + int PendingCount, + int ProcessingCount, + int DeadLetterCount, + double OldestPendingAgeSeconds); + +public sealed record DownloadCompletionJobPayload( + Guid ItemId, + string StorePath, + string FileStore, + Guid? DownloadAttemptId); + +public interface IDurableJobRepository +{ + Task> ClaimDueAsync( + string workerId, + DateTimeOffset now, + DateTimeOffset leaseUntil, + int take, + CancellationToken cancellationToken); + + Task AdvanceStageAsync( + Guid id, + string workerId, + DurableJobStage expectedStage, + DurableJobStage nextStage, + DateTimeOffset now, + CancellationToken cancellationToken); + + Task RenewLeaseAsync( + Guid id, + string workerId, + DateTimeOffset now, + DateTimeOffset leaseUntil, + CancellationToken cancellationToken); + + Task MarkFailedAsync( + Guid id, + string workerId, + int attemptCount, + DateTimeOffset attemptedAt, + DateTimeOffset? nextAttemptAt, + string error, + CancellationToken cancellationToken); + + Task GetPageAsync( + DurableJobStatus? status, + int skip, + int take, + CancellationToken cancellationToken); + + Task RetryAsync( + IReadOnlyCollection ids, + DateTimeOffset now, + CancellationToken cancellationToken); + + Task ResolveAsync( + IReadOnlyCollection ids, + DateTimeOffset now, + CancellationToken cancellationToken); + + Task GetStatisticsAsync( + DateTimeOffset now, + CancellationToken cancellationToken); +} diff --git a/SecondDimensionWatcherReDive.Framework/DataRepository/IReadinessRepository.cs b/SecondDimensionWatcherReDive.Framework/DataRepository/IReadinessRepository.cs new file mode 100644 index 0000000..b65eba4 --- /dev/null +++ b/SecondDimensionWatcherReDive.Framework/DataRepository/IReadinessRepository.cs @@ -0,0 +1,6 @@ +namespace SecondDimensionWatcherReDive.Framework.DataRepository; + +public interface IReadinessRepository +{ + Task CanConnectAsync(CancellationToken cancellationToken); +} diff --git a/SecondDimensionWatcherReDive.Framework/DataRepository/IScheduledTaskLeaseRepository.cs b/SecondDimensionWatcherReDive.Framework/DataRepository/IScheduledTaskLeaseRepository.cs new file mode 100644 index 0000000..34b6424 --- /dev/null +++ b/SecondDimensionWatcherReDive.Framework/DataRepository/IScheduledTaskLeaseRepository.cs @@ -0,0 +1,28 @@ +namespace SecondDimensionWatcherReDive.Framework.DataRepository; + +public interface IScheduledTaskLeaseRepository +{ + Task TryAcquireAsync( + string taskId, + string ownerId, + DateTimeOffset now, + DateTimeOffset leaseUntil, + bool force, + CancellationToken cancellationToken); + + Task RenewAsync( + string taskId, + string ownerId, + DateTimeOffset now, + DateTimeOffset leaseUntil, + CancellationToken cancellationToken); + + Task CompleteAsync( + string taskId, + string ownerId, + DateTimeOffset completedAt, + DateTimeOffset leaseUntil, + bool succeeded, + string? error, + CancellationToken cancellationToken); +} diff --git a/SecondDimensionWatcherReDive.Framework/Notifications/IDownloadCompletionNotifier.cs b/SecondDimensionWatcherReDive.Framework/Notifications/IDownloadCompletionNotifier.cs new file mode 100644 index 0000000..94d95ef --- /dev/null +++ b/SecondDimensionWatcherReDive.Framework/Notifications/IDownloadCompletionNotifier.cs @@ -0,0 +1,15 @@ +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Framework.Notifications; + +/// +/// Optional notification effect in the durable download-completion workflow. +/// Implementations must treat as an idempotency key. +/// +public interface IDownloadCompletionNotifier +{ + Task NotifyAsync( + Guid eventId, + DownloadCompletionJobPayload payload, + CancellationToken cancellationToken); +} diff --git a/SecondDimensionWatcherReDive.Framework/PluginParams/FileDownloadCompleteParam.cs b/SecondDimensionWatcherReDive.Framework/PluginParams/FileDownloadCompleteParam.cs index c0187bd..e0adc16 100644 --- a/SecondDimensionWatcherReDive.Framework/PluginParams/FileDownloadCompleteParam.cs +++ b/SecondDimensionWatcherReDive.Framework/PluginParams/FileDownloadCompleteParam.cs @@ -1,8 +1,18 @@ namespace SecondDimensionWatcherReDive.Framework.PluginParams; -public class FileDownloadCompleteParam(Guid itemId, string storePath, string fileStore) +public class FileDownloadCompleteParam( + Guid itemId, + string storePath, + string fileStore, + Guid? eventId = null) { + /// + /// Stable identifier for this completion workflow. Plugin handlers can persist + /// it as an idempotency key before performing externally visible work. + /// + public Guid EventId { get; } = eventId ?? itemId; + public Guid ItemId { get; set; } = itemId; public string StorePath { get; set; } = storePath; public string FileStore { get; set; } = fileStore; -} \ No newline at end of file +} diff --git a/SecondDimensionWatcherReDive.Framework/Tasks/IScheduledTaskLeaseManager.cs b/SecondDimensionWatcherReDive.Framework/Tasks/IScheduledTaskLeaseManager.cs new file mode 100644 index 0000000..2a0f1bd --- /dev/null +++ b/SecondDimensionWatcherReDive.Framework/Tasks/IScheduledTaskLeaseManager.cs @@ -0,0 +1,23 @@ +namespace SecondDimensionWatcherReDive.Framework.Tasks; + +public sealed class ScheduledTaskLeaseUnavailableException(Exception innerException) + : Exception("The scheduled-task lease store is unavailable.", innerException); + +public interface IScheduledTaskExecutionLease : IAsyncDisposable +{ + CancellationToken LeaseLostToken { get; } + + Task CompleteAsync( + bool succeeded, + string? error, + CancellationToken cancellationToken); +} + +public interface IScheduledTaskLeaseManager +{ + Task TryAcquireAsync( + string taskId, + TimeSpan interval, + bool force, + CancellationToken cancellationToken); +} diff --git a/SecondDimensionWatcherReDive.Framework/Tasks/ScheduledTaskBase.cs b/SecondDimensionWatcherReDive.Framework/Tasks/ScheduledTaskBase.cs index 7a9a2e0..fa025c7 100644 --- a/SecondDimensionWatcherReDive.Framework/Tasks/ScheduledTaskBase.cs +++ b/SecondDimensionWatcherReDive.Framework/Tasks/ScheduledTaskBase.cs @@ -4,10 +4,16 @@ namespace SecondDimensionWatcherReDive.Framework.Tasks; public abstract class ScheduledTaskBase : IScheduledTask { - private readonly Channel _runQueue = - Channel.CreateUnbounded( - new UnboundedChannelOptions { SingleReader = true }); - + private readonly Channel _runQueue = Channel.CreateBounded( + new BoundedChannelOptions(1) + { + SingleReader = true, + SingleWriter = false, + FullMode = BoundedChannelFullMode.DropWrite + }); + private readonly object _sync = new(); + private TaskCompletionSource? _pendingRun; + private bool _pendingForce; private volatile bool _isRunning; private DateTimeOffset? _lastRunAt; @@ -19,55 +25,157 @@ public abstract class ScheduledTaskBase : IScheduledTask public async Task RunNowAsync(CancellationToken cancellationToken) { - var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - - await using var registration = cancellationToken.Register( - () => tcs.TrySetCanceled(cancellationToken)); - - await _runQueue.Writer.WriteAsync(tcs, cancellationToken); - await tcs.Task; + var completion = QueueRun(force: true); + // Cancelling one HTTP request must not cancel the shared execution that + // other callers and the periodic scheduler are awaiting. + await completion.WaitAsync(cancellationToken); } /// - /// Enqueues a run request without waiting for completion. + /// Runs a periodic signal and reports whether this instance acquired the + /// distributed lease. Hosting services use the result to poll quickly + /// while another instance owns an unfinished run. /// - public void Enqueue() - { - var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - _runQueue.Writer.TryWrite(tcs); - } + public Task RunScheduledAsync(CancellationToken cancellationToken) => + QueueRun(force: false).WaitAsync(cancellationToken); /// - /// Sequentially processes queued run requests. Called by the hosting - /// BackgroundService; runs for the lifetime of the host. + /// Coalesces a run request without waiting for completion. At most one + /// pending signal exists while the current execution is in flight. /// - public async Task ProcessQueueAsync(CancellationToken cancellationToken) + public void Enqueue() => QueueRun(force: true); + + /// + /// Sequentially processes coalesced run requests. A PostgreSQL lease + /// ensures only one application instance executes a task at a time. + /// + public async Task ProcessQueueAsync( + IScheduledTaskLeaseManager leaseManager, + CancellationToken cancellationToken) { - await foreach (var tcs in _runQueue.Reader.ReadAllAsync(cancellationToken)) + await foreach (var _ in _runQueue.Reader.ReadAllAsync(cancellationToken)) { - if (tcs.Task.IsCanceled) continue; + TaskCompletionSource? completion; + bool force; + lock (_sync) + { + completion = _pendingRun; + force = _pendingForce; + } + if (completion is null) continue; - _isRunning = true; + IScheduledTaskExecutionLease? lease; try { - await ExecuteTaskAsync(cancellationToken); - _lastRunAt = DateTimeOffset.UtcNow; - tcs.TrySetResult(); + lease = await leaseManager.TryAcquireAsync( + Id, + Interval, + force, + cancellationToken); } - catch (OperationCanceledException ex) + catch (OperationCanceledException exception) when (cancellationToken.IsCancellationRequested) { - tcs.TrySetCanceled(ex.CancellationToken); + completion.TrySetCanceled(exception.CancellationToken); + FinishRun(completion); + throw; } - catch (Exception ex) + catch (Exception exception) { - tcs.TrySetException(ex); + completion.TrySetException( + new ScheduledTaskLeaseUnavailableException(exception)); + FinishRun(completion); + continue; } - finally + + if (lease is null) + { + // Another instance owns the same periodic task. Its local timer + // will drive the execution; this duplicate signal is complete. + completion.TrySetResult(false); + FinishRun(completion); + continue; + } + + await using (lease) { - _isRunning = false; + using var executionCancellation = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + lease.LeaseLostToken); + _isRunning = true; + try + { + await ExecuteTaskAsync(executionCancellation.Token); + _lastRunAt = DateTimeOffset.UtcNow; + await lease.CompleteAsync(true, null, cancellationToken); + completion.TrySetResult(true); + } + catch (OperationCanceledException exception) + { + // On host shutdown or lease loss, leave the lease to expire so + // another instance can resume without an overlapping run. + completion.TrySetCanceled(exception.CancellationToken); + if (cancellationToken.IsCancellationRequested) + throw; + } + catch (Exception exception) + { + try + { + await lease.CompleteAsync( + false, + exception.GetType().Name, + cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception leaseException) + { + completion.TrySetException( + new ScheduledTaskLeaseUnavailableException(leaseException)); + continue; + } + completion.TrySetException(exception); + } + finally + { + _isRunning = false; + FinishRun(completion); + } } } } protected abstract Task ExecuteTaskAsync(CancellationToken cancellationToken); + + private Task QueueRun(bool force) + { + lock (_sync) + { + if (_pendingRun is { Task.IsCompleted: false }) + { + _pendingForce |= force; + return _pendingRun.Task; + } + + _pendingForce = force; + _pendingRun = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + _runQueue.Writer.TryWrite(0); + return _pendingRun.Task; + } + } + + private void FinishRun(TaskCompletionSource completion) + { + lock (_sync) + { + if (ReferenceEquals(_pendingRun, completion)) + { + _pendingRun = null; + _pendingForce = false; + } + } + } } diff --git a/SecondDimensionWatcherReDive.IntegrationTest/Health/HealthEndpointTests.cs b/SecondDimensionWatcherReDive.IntegrationTest/Health/HealthEndpointTests.cs new file mode 100644 index 0000000..5297e13 --- /dev/null +++ b/SecondDimensionWatcherReDive.IntegrationTest/Health/HealthEndpointTests.cs @@ -0,0 +1,47 @@ +using System.Net; +using System.Text.Json; + +namespace SecondDimensionWatcherReDive.IntegrationTest.Health; + +[TestClass] +public sealed class HealthEndpointTests +{ + private WebDavWebApplicationFactory _factory = null!; + + [TestInitialize] + public void Setup() => _factory = new WebDavWebApplicationFactory(); + + [TestCleanup] + public void Cleanup() => _factory.Dispose(); + + [TestMethod] + public async Task Liveness_HasNoExternalChecks_WhenReadinessDependencyFails() + { + using var client = _factory.CreateUnauthenticatedClient(); + + using var live = await client.GetAsync("/health/live"); + using var ready = await client.GetAsync("/health/ready"); + + Assert.AreEqual(HttpStatusCode.OK, live.StatusCode); + using var liveBody = await JsonDocument.ParseAsync( + await live.Content.ReadAsStreamAsync()); + Assert.AreEqual(0, liveBody.RootElement.GetProperty("checks").EnumerateObject().Count()); + Assert.AreEqual(HttpStatusCode.ServiceUnavailable, ready.StatusCode); + } + + [TestMethod] + public async Task Metrics_ArePublicAndDoNotExposeRawPathLabels() + { + using var client = _factory.CreateUnauthenticatedClient(); + using var _ = await client.GetAsync("/health/live?private=value"); + + using var response = await client.GetAsync("/metrics"); + var body = await response.Content.ReadAsStringAsync(); + + Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); + StringAssert.Contains(body, "target_info"); + Assert.IsFalse(body.Contains("url.path", StringComparison.OrdinalIgnoreCase)); + Assert.IsFalse(body.Contains("url_path", StringComparison.OrdinalIgnoreCase)); + Assert.IsFalse(body.Contains("private=value", StringComparison.Ordinal)); + } +} diff --git a/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/FileMappingRepositoryPostgreSqlTests.cs b/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/FileMappingRepositoryPostgreSqlTests.cs index 1a06401..3e53a88 100644 --- a/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/FileMappingRepositoryPostgreSqlTests.cs +++ b/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/FileMappingRepositoryPostgreSqlTests.cs @@ -1,4 +1,5 @@ using Microsoft.EntityFrameworkCore; +using System.Text.Json; using Testcontainers.PostgreSql; using SecondDimensionWatcherReDive.Framework.DataRepository; using SecondDimensionWatcherReDive.Repositories; @@ -88,6 +89,198 @@ static async Task WriteAsync(FileMappingRepositoryPostgreSqlTestFixture fi CollectionAssert.AreEquivalent(new long[] { 0, 1 }, versions); } + [TestMethod] + public async Task DownloadCompletion_CommitsStateAndOneDurableJobTogether() + { + var seed = await Fixture.SeedTrackedAnimationAsync(CancellationToken.None); + const string StorePath = "/store/completed"; + + await Fixture.CompleteTrackedAnimationAsync( + seed.ItemId, seed.AttemptId, StorePath, CancellationToken.None); + await Fixture.CompleteTrackedAnimationAsync( + seed.ItemId, seed.AttemptId, StorePath, CancellationToken.None); + + var state = await Fixture.GetCompletionStateAsync( + seed.ItemId, CancellationToken.None); + Assert.IsTrue(state.IsFinished); + Assert.AreEqual(1, state.JobCount); + Assert.AreEqual(seed.ItemId, state.Payload.ItemId); + Assert.AreEqual(seed.AttemptId, state.Payload.DownloadAttemptId); + Assert.AreEqual(StorePath, state.Payload.StorePath); + } + + [TestMethod] + public async Task DurableJobClaim_AllowsOnlyOneWorker() + { + var now = DateTimeOffset.UtcNow; + var job = Job(DurableJobStatus.Pending, now); + await Fixture.SeedDurableJobAsync(job, CancellationToken.None); + + var claims = await Task.WhenAll( + Fixture.ClaimDueJobsAsync("worker-a", now, CancellationToken.None), + Fixture.ClaimDueJobsAsync("worker-b", now, CancellationToken.None)); + + Assert.AreEqual(1, claims.Sum(claim => claim.Count)); + Assert.AreEqual(job.Id, claims.SelectMany(claim => claim).Single().Id); + } + + [TestMethod] + public async Task DurableJobLease_RenewalProtectsSlowConsumerAndStillExpires() + { + var now = DateTimeOffset.UtcNow; + var job = Job(DurableJobStatus.Pending, now); + await Fixture.SeedDurableJobAsync(job, CancellationToken.None); + Assert.HasCount(1, await Fixture.ClaimDueJobsAsync( + "worker-a", now, CancellationToken.None)); + Assert.IsTrue(await Fixture.RenewDurableJobLeaseAsync( + job.Id, + "worker-a", + now.AddSeconds(30), + now.AddMinutes(2), + CancellationToken.None)); + + Assert.IsEmpty(await Fixture.ClaimDueJobsAsync( + "worker-b", now.AddSeconds(61), CancellationToken.None)); + Assert.HasCount(1, await Fixture.ClaimDueJobsAsync( + "worker-b", now.AddSeconds(121), CancellationToken.None)); + } + + [TestMethod] + public async Task DurableJobLease_ExpiredOwnerCannotAdvanceStage() + { + var now = DateTimeOffset.UtcNow; + var job = Job(DurableJobStatus.Pending, now); + await Fixture.SeedDurableJobAsync(job, CancellationToken.None); + Assert.HasCount(1, await Fixture.ClaimDueJobsAsync( + "worker-a", now, CancellationToken.None)); + + Assert.IsFalse(await Fixture.AdvanceDurableJobAsync( + job.Id, + "worker-a", + DurableJobStage.MapFiles, + DurableJobStage.Notify, + now.AddMinutes(2), + CancellationToken.None)); + } + + [TestMethod] + public async Task ScheduledTaskLease_HasSingleOwnerAndExpiresForTakeover() + { + var now = DateTimeOffset.UtcNow; + var first = await Fixture.TryAcquireTaskLeaseAsync( + "SyncFeed", "instance-a", now, now.AddSeconds(30), false, CancellationToken.None); + var overlapping = await Fixture.TryAcquireTaskLeaseAsync( + "SyncFeed", "instance-b", now.AddSeconds(1), now.AddSeconds(31), false, CancellationToken.None); + var takeover = await Fixture.TryAcquireTaskLeaseAsync( + "SyncFeed", "instance-b", now.AddSeconds(31), now.AddSeconds(61), false, CancellationToken.None); + + Assert.IsTrue(first); + Assert.IsFalse(overlapping); + Assert.IsTrue(takeover); + } + + [TestMethod] + public async Task ScheduledTaskLease_NormalCompletionPreventsDuplicateUntilNextDueTime() + { + var now = DateTimeOffset.UtcNow; + Assert.IsTrue(await Fixture.TryAcquireTaskLeaseAsync( + "ScrapeSeasonBangumi", + "instance-a", + now, + now.AddSeconds(30), + false, + CancellationToken.None)); + await Fixture.CompleteTaskLeaseAsync( + "ScrapeSeasonBangumi", + "instance-a", + now.AddSeconds(5), + now.AddMinutes(10), + CancellationToken.None); + + Assert.IsFalse(await Fixture.TryAcquireTaskLeaseAsync( + "ScrapeSeasonBangumi", + "instance-b", + now.AddSeconds(31), + now.AddMinutes(1), + false, + CancellationToken.None)); + Assert.IsTrue(await Fixture.TryAcquireTaskLeaseAsync( + "ScrapeSeasonBangumi", + "instance-b", + now.AddMinutes(11), + now.AddMinutes(12), + false, + CancellationToken.None)); + } + + [TestMethod] + public async Task ScheduledTaskLease_ManualRunCanOverrideCompletedCooldown() + { + var now = DateTimeOffset.UtcNow; + Assert.IsTrue(await Fixture.TryAcquireTaskLeaseAsync( + "InferAnimationMetadata", + "instance-a", + now, + now.AddSeconds(30), + false, + CancellationToken.None)); + await Fixture.CompleteTaskLeaseAsync( + "InferAnimationMetadata", + "instance-a", + now.AddSeconds(5), + now.AddMinutes(30), + CancellationToken.None); + + Assert.IsTrue(await Fixture.TryAcquireTaskLeaseAsync( + "InferAnimationMetadata", + "instance-b", + now.AddSeconds(31), + now.AddMinutes(1), + true, + CancellationToken.None)); + } + + [TestMethod] + public async Task DeadLetterJobs_CanBeRetriedOrMarkedHandled() + { + var now = DateTimeOffset.UtcNow; + var retried = Job(DurableJobStatus.DeadLetter, now); + var resolved = Job(DurableJobStatus.DeadLetter, now); + await Fixture.SeedDurableJobAsync(retried, CancellationToken.None); + await Fixture.SeedDurableJobAsync(resolved, CancellationToken.None); + + Assert.AreEqual(1, await Fixture.RetryJobsAsync( + [retried.Id], now, CancellationToken.None)); + Assert.AreEqual(1, await Fixture.ResolveJobsAsync( + [resolved.Id], now, CancellationToken.None)); + Assert.AreEqual(DurableJobStatus.Pending, await Fixture.GetJobStatusAsync( + retried.Id, CancellationToken.None)); + Assert.AreEqual(DurableJobStatus.Resolved, await Fixture.GetJobStatusAsync( + resolved.Id, CancellationToken.None)); + } + private static FileMapping Mapping(Guid animationInfoId, string virtualPath) => new(Guid.NewGuid(), animationInfoId, virtualPath, "/physical/" + Guid.NewGuid(), "local"); + + private static DurableJob Job(DurableJobStatus status, DateTimeOffset now) + { + var id = Guid.NewGuid(); + return new DurableJob( + id, + $"test:{id:N}", + DurableJobType.DownloadCompletion, + status, + DurableJobStage.MapFiles, + JsonSerializer.Serialize(new DownloadCompletionJobPayload( + Guid.NewGuid(), "/store", "local", Guid.NewGuid())), + status == DurableJobStatus.DeadLetter ? 8 : 0, + now, + now, + now, + now, + null, + null, + null, + status == DurableJobStatus.DeadLetter ? "failed" : null); + } } diff --git a/SecondDimensionWatcherReDive.IntegrationTest/WebDavWebApplicationFactory.cs b/SecondDimensionWatcherReDive.IntegrationTest/WebDavWebApplicationFactory.cs index ae5cf79..00dde62 100644 --- a/SecondDimensionWatcherReDive.IntegrationTest/WebDavWebApplicationFactory.cs +++ b/SecondDimensionWatcherReDive.IntegrationTest/WebDavWebApplicationFactory.cs @@ -122,6 +122,7 @@ protected override void ConfigureWebHost(IWebHostBuilder builder) services.RemoveAll(); services.RemoveAll(); services.RemoveAll(); + services.RemoveAll(); services.AddSingleton(FileStoreMock.Object); services.AddSingleton(FileStoreProviderMock.Object); @@ -130,6 +131,7 @@ protected override void ConfigureWebHost(IWebHostBuilder builder) services.AddSingleton(_ => new FakeWebDavTokenRepository(TestUserName, BCrypt.Net.BCrypt.HashPassword(TestPassword))); services.AddSingleton(); + services.AddSingleton(); }); } @@ -190,6 +192,12 @@ public string GenerateScript( public bool HasPendingModelChanges() => false; } + private sealed class UnavailableReadinessRepository : IReadinessRepository + { + public Task CanConnectAsync(CancellationToken cancellationToken) => + Task.FromResult(false); + } + private sealed class FakeApplicationSettingsRepository : IApplicationSettingsRepository { private readonly object _gate = new(); diff --git a/SecondDimensionWatcherReDive.Test/CodexAppServerEngineTests.cs b/SecondDimensionWatcherReDive.Test/CodexAppServerEngineTests.cs index 887fe92..7e146e1 100644 --- a/SecondDimensionWatcherReDive.Test/CodexAppServerEngineTests.cs +++ b/SecondDimensionWatcherReDive.Test/CodexAppServerEngineTests.cs @@ -253,6 +253,32 @@ [new UserMessage("use tools")], Assert.IsNotNull(transport.SingleSent("turn/interrupt")); } + [TestMethod] + public async Task ChatAsync_BufferedUpdateOverflowFailsClosed() + { + var messages = new List + { + Response(1, "{}"), + Response(2, PermissionProfiles()), + Response(3, SafeThread()), + """{"method":"item/started","params":{"threadId":"thread-1","turnId":"turn-1","item":{"id":"message-1","type":"agentMessage","phase":"final_answer","text":""}}}""" + }; + messages.AddRange(Enumerable.Range(0, 1025).Select(_ => + """{"method":"item/agentMessage/delta","params":{"threadId":"thread-1","turnId":"turn-1","itemId":"message-1","delta":"x"}}""")); + messages.Add(Response(4, """{"turn":{"id":"turn-1"}}""")); + var transport = new ScriptedTransport(messages.ToArray()); + var (engine, _) = CreateEngine(transport); + + var exception = await Assert.ThrowsExactlyAsync(() => + CollectAsync(engine.ChatAsync( + [new UserMessage("overflow")], + null, + CancellationToken.None))); + + StringAssert.Contains(exception.Message, "buffered update limit (1024)"); + Assert.IsNotNull(transport.SingleSent("turn/interrupt")); + } + [TestMethod] public async Task GetAvailableModelsAsync_FollowsModelListPagination() { diff --git a/SecondDimensionWatcherReDive.Test/CompleteDownloadBackgroundServiceTests.cs b/SecondDimensionWatcherReDive.Test/CompleteDownloadBackgroundServiceTests.cs index 7c402db..1f5ebe2 100644 --- a/SecondDimensionWatcherReDive.Test/CompleteDownloadBackgroundServiceTests.cs +++ b/SecondDimensionWatcherReDive.Test/CompleteDownloadBackgroundServiceTests.cs @@ -1,14 +1,15 @@ +using System.Text.Json; using System.Threading.Channels; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Moq; using SecondDimensionWatcherReDive.Data; using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Framework.Notifications; using SecondDimensionWatcherReDive.Framework.PluginParams; using SecondDimensionWatcherReDive.Plugin; using SecondDimensionWatcherReDive.Services; using SecondDimensionWatcherReDive.Utils.FileStore; -using SecondDimensionWatcherReDive.Utils.Incidents; namespace SecondDimensionWatcherReDive.Test; @@ -16,120 +17,261 @@ namespace SecondDimensionWatcherReDive.Test; public sealed class CompleteDownloadBackgroundServiceTests { [TestMethod] - public async Task ProcessRequestAsync_CancelledDownload_IgnoresLateCompletion() + public async Task ProcessClaimedJobAsync_ResumesAtPersistedStage() { - var request = new DownloadCompleteRequest( - Guid.NewGuid(), - "/downloads/item", - "local", - Guid.NewGuid()); - var repository = new Mock(); - repository.Setup(candidate => candidate.TryCompleteDownloadAsync( - request.ItemId, - request.DownloadAttemptId, - request.FileStore, - request.StorePath, + var job = CreateJob(DurableJobStage.Notify); + var payload = JsonSerializer.Deserialize(job.PayloadJson)!; + var repository = new Mock(); + repository.Setup(candidate => candidate.AdvanceStageAsync( + job.Id, + It.IsAny(), + It.IsAny(), + It.IsAny(), It.IsAny(), - CancellationToken.None)) - .ReturnsAsync((AnimationInfo?)null); + It.IsAny())) + .ReturnsAsync(true); var mapper = new Mock(); + var notifier = new Mock(); var plugin = new Mock>(); - var reporter = new Mock(); - using var provider = CreateProvider(repository.Object, mapper.Object, plugin.Object); - var service = new CompleteDownloadBackgroundService( - Channel.CreateUnbounded(), - provider.GetRequiredService(), - Mock.Of>(), - reporter.Object); + using var provider = CreateProvider( + repository.Object, + mapper.Object, + notifier.Object, + plugin.Object); + var service = CreateService(provider); - await service.ProcessRequestAsync(request, CancellationToken.None); + await service.ProcessClaimedJobAsync( + provider, + repository.Object, + job, + CancellationToken.None); mapper.Verify(candidate => candidate.MapDownloadAsync( It.IsAny(), It.IsAny()), Times.Never); + notifier.Verify(candidate => candidate.NotifyAsync( + job.Id, + It.Is(value => value == payload), + It.IsAny()), Times.Once); plugin.Verify(candidate => candidate.InvokeAsync( - It.IsAny(), It.IsAny()), Times.Never); - reporter.VerifyNoOtherCalls(); + It.Is(value => + value.EventId == job.Id + && value.ItemId == payload.ItemId + && value.StorePath == payload.StorePath + && value.FileStore == payload.FileStore), + It.IsAny()), Times.Once); + repository.Verify(candidate => candidate.AdvanceStageAsync( + job.Id, + It.IsAny(), + DurableJobStage.Notify, + DurableJobStage.InvokePlugins, + It.IsAny(), + It.IsAny()), Times.Once); + repository.Verify(candidate => candidate.AdvanceStageAsync( + job.Id, + It.IsAny(), + DurableJobStage.InvokePlugins, + DurableJobStage.Done, + It.IsAny(), + It.IsAny()), Times.Once); } [TestMethod] - public async Task ProcessRequestAsync_TrackedDownload_CompletesBeforeMappingAndPlugin() + public async Task ProcessClaimedJobAsync_PluginStageDoesNotReplayPriorEffects() { - var request = new DownloadCompleteRequest( - Guid.NewGuid(), - "/downloads/item", - "local", - Guid.NewGuid()); - var info = CreateInfo(request.ItemId); - var repository = new Mock(); - repository.Setup(candidate => candidate.TryCompleteDownloadAsync( - request.ItemId, - request.DownloadAttemptId, - request.FileStore, - request.StorePath, + var job = CreateJob(DurableJobStage.InvokePlugins); + var repository = new Mock(); + repository.Setup(candidate => candidate.AdvanceStageAsync( + job.Id, + It.IsAny(), + DurableJobStage.InvokePlugins, + DurableJobStage.Done, It.IsAny(), - CancellationToken.None)) - .ReturnsAsync(info); + It.IsAny())) + .ReturnsAsync(true); + var mapper = new Mock(); + var notifier = new Mock(); + var plugin = new Mock>(); + using var provider = CreateProvider( + repository.Object, + mapper.Object, + notifier.Object, + plugin.Object); + + await CreateService(provider).ProcessClaimedJobAsync( + provider, + repository.Object, + job, + CancellationToken.None); + + mapper.VerifyNoOtherCalls(); + notifier.VerifyNoOtherCalls(); + plugin.Verify(candidate => candidate.InvokeAsync( + It.Is(value => value.EventId == job.Id), + It.IsAny()), Times.Once); + } + + [TestMethod] + public async Task ProcessClaimedJobAsync_FailureSchedulesExponentialRetry() + { + var job = CreateJob(DurableJobStage.MapFiles, attemptCount: 2); + var repository = new Mock(); var mapper = new Mock(); mapper.Setup(candidate => candidate.MapDownloadAsync( - request.ItemId, - CancellationToken.None)) - .ReturnsAsync(true); + It.IsAny(), It.IsAny())) + .ReturnsAsync(false); + using var provider = CreateProvider( + repository.Object, + mapper.Object, + Mock.Of(), + Mock.Of>()); + var service = CreateService(provider); + + await service.ProcessClaimedJobAsync( + provider, + repository.Object, + job, + CancellationToken.None); + + repository.Verify(candidate => candidate.MarkFailedAsync( + job.Id, + It.IsAny(), + 3, + It.IsAny(), + It.Is(retry => retry.HasValue), + It.Is(error => error.Contains("InvalidOperationException")), + CancellationToken.None), Times.Once); + Assert.AreEqual(TimeSpan.FromSeconds(20), + CompleteDownloadBackgroundService.RetryDelay(3)); + } + + [TestMethod] + public async Task ProcessClaimedJobAsync_LastFailureEntersDeadLetter() + { + var job = CreateJob( + DurableJobStage.InvokePlugins, + CompleteDownloadBackgroundService.MaxAttempts - 1); + var repository = new Mock(); var plugin = new Mock>(); plugin.Setup(candidate => candidate.InvokeAsync( It.IsAny(), - CancellationToken.None)) - .Returns(Task.CompletedTask); - using var provider = CreateProvider(repository.Object, mapper.Object, plugin.Object); + It.IsAny())) + .ThrowsAsync(new InvalidOperationException("plugin unavailable")); + using var provider = CreateProvider( + repository.Object, + Mock.Of(), + Mock.Of(), + plugin.Object); + var service = CreateService(provider); + + await service.ProcessClaimedJobAsync( + provider, + repository.Object, + job, + CancellationToken.None); + + repository.Verify(candidate => candidate.MarkFailedAsync( + job.Id, + It.IsAny(), + CompleteDownloadBackgroundService.MaxAttempts, + It.IsAny(), + null, + It.IsAny(), + CancellationToken.None), Times.Once); + } + + [TestMethod] + public async Task ExecuteAsync_TemporaryRepositoryFailureDoesNotStopWorker() + { + var repository = new Mock(); + repository.SetupSequence(candidate => candidate.ClaimDueAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .ThrowsAsync(new InvalidOperationException("database unavailable")) + .ReturnsAsync([]); + using var provider = CreateProvider( + repository.Object, + Mock.Of(), + Mock.Of(), + Mock.Of>()); + var channel = Channel.CreateBounded(1); var service = new CompleteDownloadBackgroundService( - Channel.CreateUnbounded(), + channel, provider.GetRequiredService(), - Mock.Of>(), - Mock.Of()); + Mock.Of>()); - await service.ProcessRequestAsync(request, CancellationToken.None); + await service.StartAsync(CancellationToken.None); + channel.Writer.TryWrite(new DownloadCompleteRequest( + Guid.NewGuid(), "/store", "local", Guid.NewGuid())); + await WaitUntilAsync( + () => repository.Invocations.Count(invocation => + invocation.Method.Name == nameof(IDurableJobRepository.ClaimDueAsync)) >= 2, + TimeSpan.FromSeconds(2)); + await service.StopAsync(CancellationToken.None); - mapper.Verify(candidate => candidate.MapDownloadAsync( - request.ItemId, CancellationToken.None), Times.Once); - plugin.Verify(candidate => candidate.InvokeAsync( - It.Is(parameter => - parameter.ItemId == request.ItemId - && parameter.StorePath == request.StorePath - && parameter.FileStore == request.FileStore), - CancellationToken.None), Times.Once); + repository.Verify(candidate => candidate.ClaimDueAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), Times.AtLeast(2)); } + private static CompleteDownloadBackgroundService CreateService( + ServiceProvider provider) => new( + Channel.CreateBounded(1), + provider.GetRequiredService(), + Mock.Of>()); + private static ServiceProvider CreateProvider( - IAnimationInfoRepository repository, + IDurableJobRepository repository, IFileMapper mapper, - IPluginEventTrigger plugin) - { - return new ServiceCollection() + IDownloadCompletionNotifier notifier, + IPluginEventTrigger plugin) => + new ServiceCollection() .AddSingleton(repository) .AddSingleton(mapper) + .AddSingleton(notifier) .AddSingleton(plugin) .BuildServiceProvider(); + + private static DurableJob CreateJob( + DurableJobStage stage, + int attemptCount = 0) + { + var now = DateTimeOffset.UtcNow; + return new DurableJob( + Guid.NewGuid(), + $"completion:{Guid.NewGuid():N}", + DurableJobType.DownloadCompletion, + DurableJobStatus.Processing, + stage, + JsonSerializer.Serialize(new DownloadCompletionJobPayload( + Guid.NewGuid(), + "/downloads/item", + "local", + Guid.NewGuid())), + attemptCount, + now, + now, + now, + null, + null, + "worker", + now.AddMinutes(1), + null); } - private static AnimationInfo CreateInfo(Guid id) => new( - id, - "Title", - "Description", - DateTimeOffset.UtcNow, - "https://example.test/item.torrent", - "torrent", - [], - "hash", - true, - DateTimeOffset.UtcNow, - DateTimeOffset.UtcNow, - true, - "local", - "/downloads/item", - 1, - 1, - null, - null, - true, - 0, - AutomationDisposition: SubscriptionAutomationDisposition.DownloadCompleted); + private static async Task WaitUntilAsync(Func condition, TimeSpan timeout) + { + var deadline = DateTimeOffset.UtcNow + timeout; + while (!condition()) + { + if (DateTimeOffset.UtcNow >= deadline) + Assert.Fail("Timed out waiting for the worker to retry."); + await Task.Delay(10); + } + } } diff --git a/SecondDimensionWatcherReDive.Test/DurableJobsControllerTests.cs b/SecondDimensionWatcherReDive.Test/DurableJobsControllerTests.cs new file mode 100644 index 0000000..952e4d9 --- /dev/null +++ b/SecondDimensionWatcherReDive.Test/DurableJobsControllerTests.cs @@ -0,0 +1,72 @@ +using Microsoft.AspNetCore.Mvc; +using Moq; +using SecondDimensionWatcherReDive.Controllers; +using SecondDimensionWatcherReDive.Controllers.External; +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Test; + +[TestClass] +public sealed class DurableJobsControllerTests +{ + [TestMethod] + public async Task GetAsync_MapsDeadLettersWithoutPayloadOrLeaseData() + { + var now = DateTimeOffset.UtcNow; + var job = new DurableJob( + Guid.NewGuid(), + "secret-deduplication-key", + DurableJobType.DownloadCompletion, + DurableJobStatus.DeadLetter, + DurableJobStage.Notify, + "{\"storePath\":\"/secret/path\"}", + 8, + now, + now, + now, + now, + null, + "private-host:worker", + now, + "InvalidOperationException"); + var repository = new Mock(); + repository.Setup(candidate => candidate.GetPageAsync( + DurableJobStatus.DeadLetter, + 0, + 50, + CancellationToken.None)) + .ReturnsAsync(new DurableJobPage([job], 1)); + var controller = new DurableJobsController(repository.Object); + + var result = await controller.GetAsync( + "deadLetter", 0, 50, CancellationToken.None); + + var response = (DurableJobListResponse)((OkObjectResult)result).Value!; + Assert.HasCount(1, response.Items); + Assert.AreEqual(job.Id, response.Items[0].Id); + Assert.AreEqual("notify", response.Items[0].Stage); + Assert.IsNull(typeof(DurableJobItem).GetProperty("PayloadJson")); + Assert.IsNull(typeof(DurableJobItem).GetProperty("LeaseOwner")); + Assert.IsNull(typeof(DurableJobItem).GetProperty("DeduplicationKey")); + } + + [TestMethod] + public async Task RetryAsync_DeduplicatesIds() + { + var id = Guid.NewGuid(); + var repository = new Mock(); + repository.Setup(candidate => candidate.RetryAsync( + It.Is>(ids => ids.Count == 1 && ids.Contains(id)), + It.IsAny(), + CancellationToken.None)) + .ReturnsAsync(1); + var controller = new DurableJobsController(repository.Object); + + var result = await controller.RetryAsync( + new DurableJobMutationRequest([id, id]), + CancellationToken.None); + + var response = (DurableJobMutationResponse)((OkObjectResult)result).Value!; + Assert.AreEqual(1, response.AffectedCount); + } +} diff --git a/SecondDimensionWatcherReDive.Test/ObservabilityTests.cs b/SecondDimensionWatcherReDive.Test/ObservabilityTests.cs new file mode 100644 index 0000000..1be8fe3 --- /dev/null +++ b/SecondDimensionWatcherReDive.Test/ObservabilityTests.cs @@ -0,0 +1,96 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; +using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Observability; + +namespace SecondDimensionWatcherReDive.Test; + +[TestClass] +public sealed class ObservabilityTests +{ + [TestMethod] + public void SensitiveTagProcessor_RemovesPathsQueriesAndStatements() + { + using var activity = new Activity("request").Start(); + activity.SetTag("url.path", "/anime/private-title"); + activity.SetTag("url.query", "token=secret"); + activity.SetTag("db.statement", "SELECT * FROM users"); + activity.SetTag("tool.arguments", "{ secret: true }"); + activity.SetTag("http.route", "/anime/{id}"); + + new SensitiveTagRedactionProcessor().OnEnd(activity); + + Assert.IsNull(activity.GetTagItem("url.path")); + Assert.IsNull(activity.GetTagItem("url.query")); + Assert.IsNull(activity.GetTagItem("db.statement")); + Assert.IsNull(activity.GetTagItem("tool.arguments")); + Assert.AreEqual("/anime/{id}", activity.GetTagItem("http.route")); + } + + [TestMethod] + public void DurableJobMetrics_UseOnlyFixedLowCardinalityTags() + { + var observedKeys = new HashSet(StringComparer.Ordinal); + using var listener = new MeterListener(); + listener.InstrumentPublished = (instrument, candidate) => + { + if (instrument.Meter.Name == RuntimeTelemetry.MeterName) + candidate.EnableMeasurementEvents(instrument); + }; + listener.SetMeasurementEventCallback((_, _, tags, _) => + { + foreach (var tag in tags) + observedKeys.Add(tag.Key); + }); + listener.SetMeasurementEventCallback((_, _, tags, _) => + { + foreach (var tag in tags) + observedKeys.Add(tag.Key); + }); + listener.Start(); + using var telemetry = new RuntimeTelemetry(); + + telemetry.RecordJobAttempt( + DurableJobType.DownloadCompletion, + DurableJobStage.MapFiles, + "retry", + TimeSpan.FromMilliseconds(20)); + + CollectionAssert.AreEquivalent( + new[] { "job.type", "job.stage", "outcome" }, + observedKeys.ToArray()); + Assert.IsFalse(observedKeys.Any(key => + key.Contains("path", StringComparison.OrdinalIgnoreCase) + || key.Contains("title", StringComparison.OrdinalIgnoreCase) + || key.Contains("argument", StringComparison.OrdinalIgnoreCase))); + } + + [TestMethod] + public void ScheduledTaskMetrics_NormalizeUnknownTaskIds() + { + var observed = new Dictionary(StringComparer.Ordinal); + using var listener = new MeterListener(); + listener.InstrumentPublished = (instrument, candidate) => + { + if (instrument.Meter.Name == RuntimeTelemetry.MeterName + && instrument.Name == "sdw.scheduled_task.runs") + candidate.EnableMeasurementEvents(instrument); + }; + listener.SetMeasurementEventCallback((_, _, tags, _) => + { + foreach (var tag in tags) + observed[tag.Key] = tag.Value?.ToString(); + }); + listener.Start(); + using var telemetry = new RuntimeTelemetry(); + + telemetry.RecordScheduledTask( + "private/title/or/tool-arguments", + "failed", + TimeSpan.FromMilliseconds(1)); + + Assert.AreEqual("other", observed["task.id"]); + Assert.AreEqual("failed", observed["outcome"]); + Assert.IsFalse(observed.Values.Contains("private/title/or/tool-arguments")); + } +} diff --git a/SecondDimensionWatcherReDive.Test/ScheduledTaskBaseTests.cs b/SecondDimensionWatcherReDive.Test/ScheduledTaskBaseTests.cs new file mode 100644 index 0000000..94b5105 --- /dev/null +++ b/SecondDimensionWatcherReDive.Test/ScheduledTaskBaseTests.cs @@ -0,0 +1,168 @@ +using SecondDimensionWatcherReDive.Framework.Tasks; +using SecondDimensionWatcherReDive.Services; + +namespace SecondDimensionWatcherReDive.Test; + +[TestClass] +public sealed class ScheduledTaskBaseTests +{ + [TestMethod] + public async Task RunNowAsync_ConcurrentRequestsAreCoalesced() + { + var task = new BlockingTask(); + var leaseManager = new FakeLeaseManager(); + using var cancellation = new CancellationTokenSource(); + var processor = task.ProcessQueueAsync(leaseManager, cancellation.Token); + + var first = task.RunNowAsync(CancellationToken.None); + await task.Started.Task.WaitAsync(TimeSpan.FromSeconds(2)); + var second = task.RunNowAsync(CancellationToken.None); + task.Release.TrySetResult(); + await Task.WhenAll(first, second).WaitAsync(TimeSpan.FromSeconds(2)); + + Assert.AreEqual(1, task.ExecutionCount); + Assert.AreEqual(1, leaseManager.AcquireCount); + Assert.AreEqual(1, leaseManager.Lease.CompletionCount); + + await cancellation.CancelAsync(); + await AssertCanceledAsync(processor); + } + + [TestMethod] + public async Task RunNowAsync_LeaseOwnedByAnotherInstance_SkipsExecution() + { + var task = new BlockingTask(); + var leaseManager = new FakeLeaseManager { Deny = true }; + using var cancellation = new CancellationTokenSource(); + var processor = task.ProcessQueueAsync(leaseManager, cancellation.Token); + + await task.RunNowAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(2)); + + Assert.AreEqual(0, task.ExecutionCount); + Assert.AreEqual(1, leaseManager.AcquireCount); + Assert.IsTrue(leaseManager.LastForce); + + await cancellation.CancelAsync(); + await AssertCanceledAsync(processor); + } + + [TestMethod] + public async Task RunScheduledAsync_ContentionReportsSkippedWithoutForcingCooldown() + { + var task = new BlockingTask(); + var leaseManager = new FakeLeaseManager { Deny = true }; + using var cancellation = new CancellationTokenSource(); + var processor = task.ProcessQueueAsync(leaseManager, cancellation.Token); + + var executed = await task.RunScheduledAsync(CancellationToken.None) + .WaitAsync(TimeSpan.FromSeconds(2)); + + Assert.IsFalse(executed); + Assert.IsFalse(leaseManager.LastForce); + await cancellation.CancelAsync(); + await AssertCanceledAsync(processor); + } + + [TestMethod] + public async Task RunScheduledAsync_TemporaryLeaseStoreFailureDoesNotStopQueue() + { + var task = new BlockingTask(); + var leaseManager = new FakeLeaseManager + { + AcquireException = new InvalidOperationException("database unavailable") + }; + using var cancellation = new CancellationTokenSource(); + var processor = task.ProcessQueueAsync(leaseManager, cancellation.Token); + + var exception = await Assert.ThrowsExactlyAsync( + () => task.RunScheduledAsync(CancellationToken.None)); + Assert.IsInstanceOfType(exception.InnerException); + + leaseManager.AcquireException = null; + leaseManager.Deny = true; + Assert.IsFalse(await task.RunScheduledAsync(CancellationToken.None)); + await cancellation.CancelAsync(); + await AssertCanceledAsync(processor); + } + + [TestMethod] + public void MediaLibraryQueue_RejectsItemsBeyondCapacity() + { + var queue = new MediaLibraryScanQueue(); + var accepted = Enumerable.Range(0, MediaLibraryScanQueue.Capacity) + .Select(_ => queue.Enqueue(Guid.NewGuid())) + .ToList(); + + Assert.IsTrue(accepted.All(value => value)); + Assert.IsFalse(queue.Enqueue(Guid.NewGuid())); + } + + private sealed class BlockingTask : ScheduledTaskBase + { + public TaskCompletionSource Started { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + public int ExecutionCount { get; private set; } + public override string Id => "test"; + public override TimeSpan Interval => TimeSpan.FromMinutes(1); + + protected override async Task ExecuteTaskAsync(CancellationToken cancellationToken) + { + ExecutionCount++; + Started.TrySetResult(); + await Release.Task.WaitAsync(cancellationToken); + } + } + + private static async Task AssertCanceledAsync(Task task) + { + try + { + await task; + Assert.Fail("Expected the queue processor to be cancelled."); + } + catch (OperationCanceledException) + { + } + } + + private sealed class FakeLeaseManager : IScheduledTaskLeaseManager + { + public FakeLease Lease { get; } = new(); + public bool Deny { get; set; } + public Exception? AcquireException { get; set; } + public int AcquireCount { get; private set; } + public bool LastForce { get; private set; } + + public Task TryAcquireAsync( + string taskId, + TimeSpan interval, + bool force, + CancellationToken cancellationToken) + { + AcquireCount++; + LastForce = force; + if (AcquireException is not null) + return Task.FromException(AcquireException); + return Task.FromResult(Deny ? null : Lease); + } + } + + private sealed class FakeLease : IScheduledTaskExecutionLease + { + public int CompletionCount { get; private set; } + public CancellationToken LeaseLostToken => CancellationToken.None; + + public Task CompleteAsync( + bool succeeded, + string? error, + CancellationToken cancellationToken) + { + CompletionCount++; + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } +} diff --git a/SecondDimensionWatcherReDive/Controllers/DurableJobsController.cs b/SecondDimensionWatcherReDive/Controllers/DurableJobsController.cs new file mode 100644 index 0000000..05f7e1f --- /dev/null +++ b/SecondDimensionWatcherReDive/Controllers/DurableJobsController.cs @@ -0,0 +1,120 @@ +using Microsoft.AspNetCore.Authentication.JwtBearer; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Controllers; + +[ApiController] +[Route("api/jobs")] +[Authorize(AuthenticationSchemes = JwtBearerDefaults.AuthenticationScheme)] +internal sealed class DurableJobsController(IDurableJobRepository repository) : ControllerBase +{ + [HttpGet] + public async Task GetAsync( + [FromQuery] string? status, + [FromQuery] int skip = 0, + [FromQuery] int take = 50, + CancellationToken cancellationToken = default) + { + if (skip < 0 || take is < 1 or > 200) + return BadRequest(new + { + message = "skip must be non-negative and take must be between 1 and 200." + }); + if (!TryParseStatus(status, out var parsedStatus)) + return BadRequest(new { message = $"Unknown job status '{status}'." }); + + var page = await repository.GetPageAsync( + parsedStatus, + skip, + take, + cancellationToken); + return Ok(new External.DurableJobListResponse( + page.Items.Select(ToExternal).ToList(), + page.TotalCount)); + } + + [HttpPost("retry")] + public async Task RetryAsync( + [FromBody] External.DurableJobMutationRequest request, + CancellationToken cancellationToken) + { + if (request.Ids is null || request.Ids.Count is < 1 or > 200) + return BadRequest(new { message = "ids must contain between 1 and 200 jobs." }); + + var affected = await repository.RetryAsync( + request.Ids.Distinct().ToList(), + DateTimeOffset.UtcNow, + cancellationToken); + return Ok(new External.DurableJobMutationResponse(affected)); + } + + [HttpPost("resolve")] + public async Task ResolveAsync( + [FromBody] External.DurableJobMutationRequest request, + CancellationToken cancellationToken) + { + if (request.Ids is null || request.Ids.Count is < 1 or > 200) + return BadRequest(new { message = "ids must contain between 1 and 200 jobs." }); + + var affected = await repository.ResolveAsync( + request.Ids.Distinct().ToList(), + DateTimeOffset.UtcNow, + cancellationToken); + return Ok(new External.DurableJobMutationResponse(affected)); + } + + private static External.DurableJobItem ToExternal(DurableJob job) => new( + job.Id, + ToApiValue(job.Type), + ToApiValue(job.Status), + ToApiValue(job.Stage), + job.AttemptCount, + job.CreatedAt, + job.UpdatedAt, + job.NextAttemptAt, + job.LastAttemptAt, + job.CompletedAt, + job.LastError); + + private static bool TryParseStatus(string? value, out DurableJobStatus? status) + { + status = value?.Trim().ToLowerInvariant() switch + { + null or "" => null, + "pending" => DurableJobStatus.Pending, + "processing" => DurableJobStatus.Processing, + "completed" => DurableJobStatus.Completed, + "deadletter" or "dead-letter" => DurableJobStatus.DeadLetter, + "resolved" => DurableJobStatus.Resolved, + _ => (DurableJobStatus?)(-1) + }; + return status != (DurableJobStatus?)(-1); + } + + private static string ToApiValue(DurableJobType type) => type switch + { + DurableJobType.DownloadCompletion => "downloadCompletion", + _ => throw new ArgumentOutOfRangeException(nameof(type), type, null) + }; + + private static string ToApiValue(DurableJobStatus status) => status switch + { + DurableJobStatus.Pending => "pending", + DurableJobStatus.Processing => "processing", + DurableJobStatus.Completed => "completed", + DurableJobStatus.DeadLetter => "deadLetter", + DurableJobStatus.Resolved => "resolved", + _ => throw new ArgumentOutOfRangeException(nameof(status), status, null) + }; + + private static string ToApiValue(DurableJobStage stage) => stage switch + { + DurableJobStage.MapFiles => "mapFiles", + DurableJobStage.Notify => "notify", + DurableJobStage.InvokePlugins => "invokePlugins", + DurableJobStage.Done => "done", + _ => throw new ArgumentOutOfRangeException(nameof(stage), stage, null) + }; +} diff --git a/SecondDimensionWatcherReDive/Controllers/External/AppJsonSerializerContext.cs b/SecondDimensionWatcherReDive/Controllers/External/AppJsonSerializerContext.cs index 0f20309..f11d162 100644 --- a/SecondDimensionWatcherReDive/Controllers/External/AppJsonSerializerContext.cs +++ b/SecondDimensionWatcherReDive/Controllers/External/AppJsonSerializerContext.cs @@ -58,6 +58,10 @@ namespace SecondDimensionWatcherReDive.Controllers.External; [JsonSerializable(typeof(IncidentListResponse))] [JsonSerializable(typeof(IncidentRetryError))] [JsonSerializable(typeof(IncidentRetryBatchResponse))] +[JsonSerializable(typeof(DurableJobItem))] +[JsonSerializable(typeof(DurableJobListResponse))] +[JsonSerializable(typeof(DurableJobMutationRequest))] +[JsonSerializable(typeof(DurableJobMutationResponse))] [JsonSerializable(typeof(MediaLibrarySourceResponse))] [JsonSerializable(typeof(List))] [JsonSerializable(typeof(CreateMediaLibrarySourceRequest))] diff --git a/SecondDimensionWatcherReDive/Controllers/External/DurableJob.cs b/SecondDimensionWatcherReDive/Controllers/External/DurableJob.cs new file mode 100644 index 0000000..32cfea3 --- /dev/null +++ b/SecondDimensionWatcherReDive/Controllers/External/DurableJob.cs @@ -0,0 +1,22 @@ +namespace SecondDimensionWatcherReDive.Controllers.External; + +public sealed record DurableJobItem( + Guid Id, + string Type, + string Status, + string Stage, + int AttemptCount, + DateTimeOffset CreatedAt, + DateTimeOffset UpdatedAt, + DateTimeOffset NextAttemptAt, + DateTimeOffset? LastAttemptAt, + DateTimeOffset? CompletedAt, + string? LastError); + +public sealed record DurableJobListResponse( + IReadOnlyList Items, + int TotalCount); + +public sealed record DurableJobMutationRequest(IReadOnlyList Ids); + +public sealed record DurableJobMutationResponse(int AffectedCount); diff --git a/SecondDimensionWatcherReDive/Migrations/20260829145227_AddDurableJobsAndTaskLeases.Designer.cs b/SecondDimensionWatcherReDive/Migrations/20260829145227_AddDurableJobsAndTaskLeases.Designer.cs new file mode 100644 index 0000000..5b9dcda --- /dev/null +++ b/SecondDimensionWatcherReDive/Migrations/20260829145227_AddDurableJobsAndTaskLeases.Designer.cs @@ -0,0 +1,1086 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using SecondDimensionWatcherReDive.Models; + +#nullable disable + +namespace SecondDimensionWatcherReDive.Migrations +{ + [DbContext(typeof(ApplicationContext))] + [Migration("20260829145227_AddDurableJobsAndTaskLeases")] + partial class AddDurableJobsAndTaskLeases + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.Animation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("OriginalName") + .IsRequired() + .HasColumnType("text"); + + b.Property("PosterPath") + .HasColumnType("text"); + + b.Property("TmdbId") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("TmdbId") + .IsUnique(); + + b.ToTable("Animations"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.AnimationGroup", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("AnimationGroups"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.AnimationInfo", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AdditionalDownloadInfo") + .IsRequired() + .HasColumnType("text"); + + b.Property("AiRetryCount") + .HasColumnType("integer"); + + b.Property("AnimationId") + .HasColumnType("uuid"); + + b.Property("AutomationDisposition") + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("AutomationExplanationJson") + .HasColumnType("text"); + + b.Property("CachedDownloadData") + .IsRequired() + .HasColumnType("bytea"); + + b.Property("CurrentMetadataReviewOperationId") + .HasColumnType("uuid"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text"); + + b.Property("DownloadAttemptId") + .HasColumnType("uuid"); + + b.Property("DownloadCancellationId") + .HasColumnType("uuid"); + + b.Property("DownloadEndTime") + .HasColumnType("timestamp with time zone"); + + b.Property("DownloadStartTime") + .HasColumnType("timestamp with time zone"); + + b.Property("DownloadType") + .IsRequired() + .HasColumnType("text"); + + b.Property("DownloadUrl") + .IsRequired() + .HasColumnType("text"); + + b.Property("Episode") + .HasColumnType("integer"); + + b.Property("FileStore") + .HasColumnType("text"); + + b.Property("GroupId") + .HasColumnType("uuid"); + + b.Property("IsAiProcessed") + .HasColumnType("boolean"); + + b.Property("IsDownloadFinished") + .HasColumnType("boolean"); + + b.Property("IsDownloadTracked") + .HasColumnType("boolean"); + + b.Property("MediaLibraryMissingSince") + .HasColumnType("timestamp with time zone"); + + b.Property("MediaLibrarySourceId") + .HasColumnType("uuid"); + + b.Property("MetadataConfidence") + .HasColumnType("double precision"); + + b.Property("MetadataLastError") + .HasMaxLength(1024) + .HasColumnType("character varying(1024)"); + + b.Property("MetadataReviewedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("MetadataStatus") + .HasColumnType("integer"); + + b.Property("PublishTime") + .HasColumnType("timestamp with time zone"); + + b.Property("ReleaseSizeBytes") + .HasColumnType("bigint"); + + b.Property("Season") + .HasColumnType("integer"); + + b.Property("SourceFeedId") + .HasColumnType("uuid"); + + b.Property("StateVersion") + .IsConcurrencyToken() + .HasColumnType("bigint"); + + b.Property("StorePath") + .HasColumnType("text"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("AnimationId"); + + b.HasIndex("CurrentMetadataReviewOperationId") + .IsUnique(); + + b.HasIndex("GroupId"); + + b.HasIndex("MediaLibrarySourceId"); + + b.HasIndex("SourceFeedId"); + + b.HasIndex("FileStore", "StorePath") + .IsUnique() + .HasFilter("\"DownloadType\" = 'http://schemas.hcgstudio.com/ws/2023/06/sdw/downloadtype/media-library-import'"); + + b.HasIndex("MetadataStatus", "PublishTime"); + + b.ToTable("AnimationInfo", t => + { + t.HasCheckConstraint("CK_AnimationInfo_MetadataConfidence_Range", "\"MetadataConfidence\" IS NULL OR (\"MetadataConfidence\" >= 0 AND \"MetadataConfidence\" <= 1)"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ApplicationSettings", b => + { + b.Property("Id") + .HasColumnType("integer"); + + b.Property("ProtectedSecrets") + .HasColumnType("text"); + + b.Property("Revision") + .IsConcurrencyToken() + .HasColumnType("bigint"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("ValuesJson") + .IsRequired() + .HasColumnType("jsonb"); + + b.HasKey("Id"); + + b.ToTable("ApplicationSettings", t => + { + t.HasCheckConstraint("CK_ApplicationSettings_Revision_Positive", "\"Revision\" > 0"); + + t.HasCheckConstraint("CK_ApplicationSettings_Singleton", "\"Id\" = 1"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.BangumiSubgroup", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("MikanSubgroupId") + .HasColumnType("integer"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("ScrapedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("SeasonBangumiId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("SeasonBangumiId", "MikanSubgroupId") + .IsUnique(); + + b.ToTable("BangumiSubgroups"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ChatConversation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Title") + .HasColumnType("text"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.ToTable("ChatConversations"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ChatMessage", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Content") + .HasColumnType("text"); + + b.Property("ConversationId") + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Order") + .HasColumnType("integer"); + + b.Property("Role") + .IsRequired() + .HasColumnType("text"); + + b.Property("ToolCallId") + .HasColumnType("text"); + + b.Property("ToolCallsJson") + .HasColumnType("text"); + + b.Property("ToolName") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("ConversationId"); + + b.ToTable("ChatMessages"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.DurableJob", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AttemptCount") + .HasColumnType("integer"); + + b.Property("CompletedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("DeduplicationKey") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("LastAttemptAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastError") + .HasMaxLength(512) + .HasColumnType("character varying(512)"); + + b.Property("LeaseExpiresAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LeaseOwner") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("NextAttemptAt") + .HasColumnType("timestamp with time zone"); + + b.Property("PayloadJson") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("Stage") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(24) + .HasColumnType("character varying(24)"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(48) + .HasColumnType("character varying(48)"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DeduplicationKey") + .IsUnique(); + + b.HasIndex("Status", "NextAttemptAt", "LeaseExpiresAt"); + + b.ToTable("DurableJobs"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.Feed", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Name") + .HasColumnType("text"); + + b.Property("Url") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("Feeds"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.FileMapping", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AnimationInfoId") + .HasColumnType("uuid"); + + b.Property("FileStore") + .IsRequired() + .HasColumnType("text"); + + b.Property("PhysicalPath") + .IsRequired() + .HasColumnType("text"); + + b.Property("VirtualPath") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("AnimationInfoId"); + + b.HasIndex("VirtualPath") + .IsUnique(); + + b.ToTable("FileMappings"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.FileNameRegexRule", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AnimationId") + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("Pattern") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("character varying(512)"); + + b.HasKey("Id"); + + b.HasIndex("AnimationId", "CreatedAt"); + + b.HasIndex("AnimationId", "Pattern") + .IsUnique(); + + b.ToTable("FileNameRegexRules"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.Incident", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Detail") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("DetectedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Fingerprint") + .IsRequired() + .HasMaxLength(96) + .HasColumnType("character varying(96)"); + + b.Property("LastRetryAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastRetryError") + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("ResolvedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("RetryCount") + .HasColumnType("integer"); + + b.Property("Severity") + .HasColumnType("integer"); + + b.Property("SourceId") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("Title") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("Type") + .HasColumnType("integer"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("Fingerprint") + .IsUnique(); + + b.HasIndex("ResolvedAt", "Type", "UpdatedAt"); + + b.ToTable("Incidents"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MediaLibrarySource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("IsMonitoring") + .HasColumnType("boolean"); + + b.Property("LastError") + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("LastImportedCount") + .HasColumnType("integer"); + + b.Property("LastRemovedCount") + .HasColumnType("integer"); + + b.Property("LastScanAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastSkippedCount") + .HasColumnType("integer"); + + b.Property("LastUpdatedCount") + .HasColumnType("integer"); + + b.Property("Path") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Path") + .IsUnique(); + + b.ToTable("MediaLibrarySources"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewMappingSnapshot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("FileStore") + .IsRequired() + .HasColumnType("text"); + + b.Property("Kind") + .HasColumnType("integer"); + + b.Property("OperationId") + .HasColumnType("uuid"); + + b.Property("PhysicalPath") + .IsRequired() + .HasColumnType("text"); + + b.Property("VirtualPath") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("OperationId", "Kind", "VirtualPath") + .IsUnique(); + + b.ToTable("MetadataReviewMappingSnapshots"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AnimationInfoId") + .HasColumnType("uuid"); + + b.Property("AppliedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("AppliedVersion") + .HasColumnType("bigint"); + + b.Property("BaseFileStore") + .HasColumnType("text"); + + b.Property("BaseIsDownloadFinished") + .HasColumnType("boolean"); + + b.Property("BaseStorePath") + .HasColumnType("text"); + + b.Property("BaseVersion") + .HasColumnType("bigint"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("ExpiresAt") + .HasColumnType("timestamp with time zone"); + + b.Property("PreviousAiRetryCount") + .HasColumnType("integer"); + + b.Property("PreviousAnimationId") + .HasColumnType("uuid"); + + b.Property("PreviousConfidence") + .HasColumnType("double precision"); + + b.Property("PreviousCurrentOperationId") + .HasColumnType("uuid"); + + b.Property("PreviousDescription") + .HasColumnType("text"); + + b.Property("PreviousEpisode") + .HasColumnType("integer"); + + b.Property("PreviousGroupId") + .HasColumnType("uuid"); + + b.Property("PreviousIsAiProcessed") + .HasColumnType("boolean"); + + b.Property("PreviousLastError") + .HasColumnType("text"); + + b.Property("PreviousMetadataStatus") + .HasColumnType("integer"); + + b.Property("PreviousReviewedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("PreviousSeason") + .HasColumnType("integer"); + + b.Property("ProposedAnimationName") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProposedAnimationOriginalName") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProposedAnimationPosterPath") + .HasColumnType("text"); + + b.Property("ProposedAnimationTmdbId") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProposedDescription") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProposedEpisode") + .HasColumnType("integer"); + + b.Property("ProposedGroupName") + .HasColumnType("text"); + + b.Property("ProposedSeason") + .HasColumnType("integer"); + + b.Property("State") + .HasColumnType("integer"); + + b.Property("UndoneAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("AnimationInfoId", "AppliedVersion") + .IsUnique(); + + b.HasIndex("AnimationInfoId", "State"); + + b.HasIndex("State", "ExpiresAt"); + + b.ToTable("MetadataReviewOperations", t => + { + t.HasCheckConstraint("CK_MetadataReviewOperations_Expiry", "\"ExpiresAt\" > \"CreatedAt\""); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MigrationMarker", b => + { + b.Property("Key") + .HasColumnType("text"); + + b.Property("AppliedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Key"); + + b.ToTable("MigrationMarkers"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.PlaybackPreference", b => + { + b.Property("UserId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AudioLanguage") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("AudioTrackLabel") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("AutoPlayNext") + .HasColumnType("boolean"); + + b.Property("SubtitleLanguage") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("SubtitleTrackLabel") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("UserId"); + + b.ToTable("PlaybackPreferences"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.PlaybackProgress", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AnimationInfoId") + .HasColumnType("uuid"); + + b.Property("DurationSeconds") + .HasColumnType("double precision"); + + b.Property("IsWatched") + .HasColumnType("boolean"); + + b.Property("PositionSeconds") + .HasColumnType("double precision"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.Property("VirtualPath") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("WatchedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("AnimationInfoId"); + + b.HasIndex("UserId", "AnimationInfoId", "VirtualPath") + .IsUnique(); + + b.HasIndex("UserId", "IsWatched", "UpdatedAt"); + + b.ToTable("PlaybackProgresses", t => + { + t.HasCheckConstraint("CK_PlaybackProgresses_Duration_NonNegative", "\"DurationSeconds\" >= 0"); + + t.HasCheckConstraint("CK_PlaybackProgresses_Position_NonNegative", "\"PositionSeconds\" >= 0"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ScheduledTaskState", b => + { + b.Property("TaskId") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("LastCompletedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastError") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("LastStartedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastSucceededAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LeaseExpiresAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LeaseOwner") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("RunCount") + .HasColumnType("bigint"); + + b.HasKey("TaskId"); + + b.ToTable("ScheduledTaskStates"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SeasonBangumi", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("DayOfWeek") + .HasColumnType("integer"); + + b.Property("ImageUrl") + .HasColumnType("text"); + + b.Property("MikanId") + .HasColumnType("integer"); + + b.Property("ScrapedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("MikanId") + .IsUnique(); + + b.ToTable("SeasonBangumis"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SubscriptionAutomationPolicy", b => + { + b.Property("FeedId") + .HasColumnType("uuid"); + + b.PrimitiveCollection("Codecs") + .IsRequired() + .HasColumnType("text[]"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.PrimitiveCollection("ExcludedKeywords") + .IsRequired() + .HasColumnType("text[]"); + + b.PrimitiveCollection("Languages") + .IsRequired() + .HasColumnType("text[]"); + + b.Property("MaxSizeBytes") + .HasColumnType("bigint"); + + b.Property("MinSizeBytes") + .HasColumnType("bigint"); + + b.Property("Mode") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.PrimitiveCollection("Resolutions") + .IsRequired() + .HasColumnType("text[]"); + + b.PrimitiveCollection("SubtitleGroups") + .IsRequired() + .HasColumnType("text[]"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("FeedId"); + + b.HasIndex("UpdatedAt"); + + b.ToTable("SubscriptionAutomationPolicies"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.WebDavToken", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("TokenHash") + .IsRequired() + .HasColumnType("text"); + + b.Property("Username") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Username") + .IsUnique(); + + b.ToTable("WebDavTokens"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.AnimationInfo", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.Animation", "Animation") + .WithMany() + .HasForeignKey("AnimationId"); + + b.HasOne("SecondDimensionWatcherReDive.Models.AnimationGroup", "Group") + .WithMany() + .HasForeignKey("GroupId"); + + b.HasOne("SecondDimensionWatcherReDive.Models.MediaLibrarySource", null) + .WithMany() + .HasForeignKey("MediaLibrarySourceId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("SecondDimensionWatcherReDive.Models.Feed", null) + .WithMany() + .HasForeignKey("SourceFeedId") + .OnDelete(DeleteBehavior.SetNull); + + b.Navigation("Animation"); + + b.Navigation("Group"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.BangumiSubgroup", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.SeasonBangumi", "SeasonBangumi") + .WithMany("Subgroups") + .HasForeignKey("SeasonBangumiId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("SeasonBangumi"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ChatMessage", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.ChatConversation", "Conversation") + .WithMany("Messages") + .HasForeignKey("ConversationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Conversation"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.FileNameRegexRule", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.Animation", null) + .WithMany() + .HasForeignKey("AnimationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewMappingSnapshot", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.MetadataReviewOperation", "Operation") + .WithMany("MappingSnapshots") + .HasForeignKey("OperationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Operation"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewOperation", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.AnimationInfo", "AnimationInfo") + .WithMany() + .HasForeignKey("AnimationInfoId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AnimationInfo"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.PlaybackProgress", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.AnimationInfo", "AnimationInfo") + .WithMany() + .HasForeignKey("AnimationInfoId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AnimationInfo"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SubscriptionAutomationPolicy", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.Feed", "Feed") + .WithOne() + .HasForeignKey("SecondDimensionWatcherReDive.Models.SubscriptionAutomationPolicy", "FeedId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Feed"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ChatConversation", b => + { + b.Navigation("Messages"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewOperation", b => + { + b.Navigation("MappingSnapshots"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SeasonBangumi", b => + { + b.Navigation("Subgroups"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/SecondDimensionWatcherReDive/Migrations/20260829145227_AddDurableJobsAndTaskLeases.cs b/SecondDimensionWatcherReDive/Migrations/20260829145227_AddDurableJobsAndTaskLeases.cs new file mode 100644 index 0000000..f23a1de --- /dev/null +++ b/SecondDimensionWatcherReDive/Migrations/20260829145227_AddDurableJobsAndTaskLeases.cs @@ -0,0 +1,79 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace SecondDimensionWatcherReDive.Migrations +{ + /// + public partial class AddDurableJobsAndTaskLeases : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "DurableJobs", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + DeduplicationKey = table.Column(type: "character varying(256)", maxLength: 256, nullable: false), + Type = table.Column(type: "character varying(48)", maxLength: 48, nullable: false), + Status = table.Column(type: "character varying(24)", maxLength: 24, nullable: false), + Stage = table.Column(type: "character varying(32)", maxLength: 32, nullable: false), + PayloadJson = table.Column(type: "jsonb", nullable: false), + AttemptCount = table.Column(type: "integer", nullable: false), + CreatedAt = table.Column(type: "timestamp with time zone", nullable: false), + UpdatedAt = table.Column(type: "timestamp with time zone", nullable: false), + NextAttemptAt = table.Column(type: "timestamp with time zone", nullable: false), + LastAttemptAt = table.Column(type: "timestamp with time zone", nullable: true), + CompletedAt = table.Column(type: "timestamp with time zone", nullable: true), + LeaseOwner = table.Column(type: "character varying(128)", maxLength: 128, nullable: true), + LeaseExpiresAt = table.Column(type: "timestamp with time zone", nullable: true), + LastError = table.Column(type: "character varying(512)", maxLength: 512, nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_DurableJobs", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "ScheduledTaskStates", + columns: table => new + { + TaskId = table.Column(type: "character varying(128)", maxLength: 128, nullable: false), + LeaseOwner = table.Column(type: "character varying(128)", maxLength: 128, nullable: true), + LeaseExpiresAt = table.Column(type: "timestamp with time zone", nullable: true), + LastStartedAt = table.Column(type: "timestamp with time zone", nullable: true), + LastCompletedAt = table.Column(type: "timestamp with time zone", nullable: true), + LastSucceededAt = table.Column(type: "timestamp with time zone", nullable: true), + RunCount = table.Column(type: "bigint", nullable: false), + LastError = table.Column(type: "character varying(256)", maxLength: 256, nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_ScheduledTaskStates", x => x.TaskId); + }); + + migrationBuilder.CreateIndex( + name: "IX_DurableJobs_DeduplicationKey", + table: "DurableJobs", + column: "DeduplicationKey", + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_DurableJobs_Status_NextAttemptAt_LeaseExpiresAt", + table: "DurableJobs", + columns: new[] { "Status", "NextAttemptAt", "LeaseExpiresAt" }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "DurableJobs"); + + migrationBuilder.DropTable( + name: "ScheduledTaskStates"); + } + } +} diff --git a/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs b/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs index 8126b9e..a3a2407 100644 --- a/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs +++ b/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs @@ -322,6 +322,75 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("ChatMessages"); }); + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.DurableJob", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AttemptCount") + .HasColumnType("integer"); + + b.Property("CompletedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("DeduplicationKey") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("LastAttemptAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastError") + .HasMaxLength(512) + .HasColumnType("character varying(512)"); + + b.Property("LeaseExpiresAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LeaseOwner") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("NextAttemptAt") + .HasColumnType("timestamp with time zone"); + + b.Property("PayloadJson") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("Stage") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(24) + .HasColumnType("character varying(24)"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(48) + .HasColumnType("character varying(48)"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DeduplicationKey") + .IsUnique(); + + b.HasIndex("Status", "NextAttemptAt", "LeaseExpiresAt"); + + b.ToTable("DurableJobs"); + }); + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.Feed", b => { b.Property("Id") @@ -753,6 +822,40 @@ protected override void BuildModel(ModelBuilder modelBuilder) }); }); + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ScheduledTaskState", b => + { + b.Property("TaskId") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("LastCompletedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastError") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("LastStartedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastSucceededAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LeaseExpiresAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LeaseOwner") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("RunCount") + .HasColumnType("bigint"); + + b.HasKey("TaskId"); + + b.ToTable("ScheduledTaskStates"); + }); + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SeasonBangumi", b => { b.Property("Id") diff --git a/SecondDimensionWatcherReDive/Models/ApplicationContext.cs b/SecondDimensionWatcherReDive/Models/ApplicationContext.cs index 59764ac..fadeb6c 100644 --- a/SecondDimensionWatcherReDive/Models/ApplicationContext.cs +++ b/SecondDimensionWatcherReDive/Models/ApplicationContext.cs @@ -33,9 +33,64 @@ public ApplicationContext(DbContextOptions options) public DbSet PlaybackPreferences { get; set; } public DbSet MediaLibrarySources { get; set; } public DbSet ApplicationSettings { get; set; } + public DbSet DurableJobs { get; set; } + public DbSet ScheduledTaskStates { get; set; } protected override void OnModelCreating(ModelBuilder modelBuilder) { + modelBuilder.Entity() + .HasIndex(job => job.DeduplicationKey) + .IsUnique(); + + modelBuilder.Entity() + .HasIndex(job => new { job.Status, job.NextAttemptAt, job.LeaseExpiresAt }); + + modelBuilder.Entity() + .Property(job => job.DeduplicationKey) + .HasMaxLength(256); + + modelBuilder.Entity() + .Property(job => job.Type) + .HasConversion() + .HasMaxLength(48); + + modelBuilder.Entity() + .Property(job => job.Status) + .HasConversion() + .HasMaxLength(24); + + modelBuilder.Entity() + .Property(job => job.Stage) + .HasConversion() + .HasMaxLength(32); + + modelBuilder.Entity() + .Property(job => job.PayloadJson) + .HasColumnType("jsonb"); + + modelBuilder.Entity() + .Property(job => job.LeaseOwner) + .HasMaxLength(128); + + modelBuilder.Entity() + .Property(job => job.LastError) + .HasMaxLength(512); + + modelBuilder.Entity() + .HasKey(state => state.TaskId); + + modelBuilder.Entity() + .Property(state => state.TaskId) + .HasMaxLength(128); + + modelBuilder.Entity() + .Property(state => state.LeaseOwner) + .HasMaxLength(128); + + modelBuilder.Entity() + .Property(state => state.LastError) + .HasMaxLength(256); + modelBuilder.Entity() .Property(settings => settings.Id) .ValueGeneratedNever(); diff --git a/SecondDimensionWatcherReDive/Models/DurableJob.cs b/SecondDimensionWatcherReDive/Models/DurableJob.cs new file mode 100644 index 0000000..b3fa530 --- /dev/null +++ b/SecondDimensionWatcherReDive/Models/DurableJob.cs @@ -0,0 +1,22 @@ +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Models; + +public sealed class DurableJob +{ + public Guid Id { get; set; } + public string DeduplicationKey { get; set; } = string.Empty; + public DurableJobType Type { get; set; } + public DurableJobStatus Status { get; set; } + public DurableJobStage Stage { get; set; } + public string PayloadJson { get; set; } = string.Empty; + public int AttemptCount { get; set; } + public DateTimeOffset CreatedAt { get; set; } + public DateTimeOffset UpdatedAt { get; set; } + public DateTimeOffset NextAttemptAt { get; set; } + public DateTimeOffset? LastAttemptAt { get; set; } + public DateTimeOffset? CompletedAt { get; set; } + public string? LeaseOwner { get; set; } + public DateTimeOffset? LeaseExpiresAt { get; set; } + public string? LastError { get; set; } +} diff --git a/SecondDimensionWatcherReDive/Models/ScheduledTaskState.cs b/SecondDimensionWatcherReDive/Models/ScheduledTaskState.cs new file mode 100644 index 0000000..257af3d --- /dev/null +++ b/SecondDimensionWatcherReDive/Models/ScheduledTaskState.cs @@ -0,0 +1,13 @@ +namespace SecondDimensionWatcherReDive.Models; + +public sealed class ScheduledTaskState +{ + public string TaskId { get; set; } = string.Empty; + public string? LeaseOwner { get; set; } + public DateTimeOffset? LeaseExpiresAt { get; set; } + public DateTimeOffset? LastStartedAt { get; set; } + public DateTimeOffset? LastCompletedAt { get; set; } + public DateTimeOffset? LastSucceededAt { get; set; } + public long RunCount { get; set; } + public string? LastError { get; set; } +} diff --git a/SecondDimensionWatcherReDive/Observability/DurableJobMetricsBackgroundService.cs b/SecondDimensionWatcherReDive/Observability/DurableJobMetricsBackgroundService.cs new file mode 100644 index 0000000..db781ea --- /dev/null +++ b/SecondDimensionWatcherReDive/Observability/DurableJobMetricsBackgroundService.cs @@ -0,0 +1,37 @@ +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Observability; + +internal sealed partial class DurableJobMetricsBackgroundService( + IServiceScopeFactory scopeFactory, + RuntimeTelemetry telemetry, + ILogger logger) : BackgroundService +{ + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + using var timer = new PeriodicTimer(TimeSpan.FromSeconds(15)); + do + { + try + { + await using var scope = scopeFactory.CreateAsyncScope(); + var repository = scope.ServiceProvider.GetRequiredService(); + telemetry.UpdateJobStatistics(await repository.GetStatisticsAsync( + DateTimeOffset.UtcNow, + stoppingToken)); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + catch (Exception exception) + { + LogCollectionFailed(logger, exception); + } + } while (await timer.WaitForNextTickAsync(stoppingToken)); + } + + [LoggerMessage(Level = LogLevel.Debug, + Message = "Durable job metric collection failed")] + private static partial void LogCollectionFailed(ILogger logger, Exception exception); +} diff --git a/SecondDimensionWatcherReDive/Observability/ReadinessHealthChecks.cs b/SecondDimensionWatcherReDive/Observability/ReadinessHealthChecks.cs new file mode 100644 index 0000000..34865e4 --- /dev/null +++ b/SecondDimensionWatcherReDive/Observability/ReadinessHealthChecks.cs @@ -0,0 +1,111 @@ +using System.Text.Json; +using Microsoft.Extensions.Caching.Distributed; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using SecondDimensionWatcherReDive.AI.Abstractions; +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Observability; + +internal static class HealthTags +{ + public const string Ready = "ready"; +} + +internal sealed class DatabaseReadinessHealthCheck(IServiceScopeFactory scopeFactory) : IHealthCheck +{ + public async Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + { + await using var scope = scopeFactory.CreateAsyncScope(); + var repository = scope.ServiceProvider.GetRequiredService(); + return await repository.CanConnectAsync(cancellationToken) + ? HealthCheckResult.Healthy() + : HealthCheckResult.Unhealthy(); + } +} + +internal sealed class DistributedCacheReadinessHealthCheck(IDistributedCache cache) : IHealthCheck +{ + public async Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + { + await cache.GetAsync("health:ready", cancellationToken); + return HealthCheckResult.Healthy(); + } +} + +internal sealed class QbittorrentReadinessHealthCheck(IHttpClientFactory httpClientFactory) : IHealthCheck +{ + public async Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + { + using var client = httpClientFactory.CreateClient("RemoteTorrentDownloadClient"); + using var response = await client.GetAsync( + "/api/v2/app/version", + HttpCompletionOption.ResponseHeadersRead, + cancellationToken); + return response.IsSuccessStatusCode + ? HealthCheckResult.Healthy() + : HealthCheckResult.Unhealthy(); + } +} + +internal sealed class LocalStorageReadinessHealthCheck(IConfiguration configuration) : IHealthCheck +{ + public Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + var path = Path.GetFullPath(configuration["FileStore:Local"] ?? "./download"); + if (!Directory.Exists(path)) + return Task.FromResult(HealthCheckResult.Unhealthy()); + + // Force a real filesystem operation without creating or deleting data. + using var enumerator = Directory.EnumerateFileSystemEntries(path).GetEnumerator(); + _ = enumerator.MoveNext(); + return Task.FromResult(HealthCheckResult.Healthy()); + } +} + +internal sealed class AiReadinessHealthCheck(IServiceScopeFactory scopeFactory) : IHealthCheck +{ + public async Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + { + await using var scope = scopeFactory.CreateAsyncScope(); + var engine = scope.ServiceProvider.GetRequiredService(); + await engine.GetAvailableModelsAsync(cancellationToken); + return HealthCheckResult.Healthy(); + } +} + +internal static class HealthResponseWriter +{ + public static async Task WriteAsync( + HttpContext context, + HealthReport report) + { + context.Response.ContentType = "application/json; charset=utf-8"; + await using var writer = new Utf8JsonWriter(context.Response.Body); + writer.WriteStartObject(); + writer.WriteString("status", report.Status.ToString().ToLowerInvariant()); + writer.WriteStartObject("checks"); + foreach (var entry in report.Entries.OrderBy(pair => pair.Key, StringComparer.Ordinal)) + { + writer.WriteStartObject(entry.Key); + writer.WriteString("status", entry.Value.Status.ToString().ToLowerInvariant()); + writer.WriteNumber("durationMs", entry.Value.Duration.TotalMilliseconds); + if (entry.Value.Exception is not null) + writer.WriteString("errorType", entry.Value.Exception.GetType().Name); + writer.WriteEndObject(); + } + writer.WriteEndObject(); + writer.WriteEndObject(); + await writer.FlushAsync(context.RequestAborted); + } +} diff --git a/SecondDimensionWatcherReDive/Observability/RuntimeTelemetry.cs b/SecondDimensionWatcherReDive/Observability/RuntimeTelemetry.cs new file mode 100644 index 0000000..fb35928 --- /dev/null +++ b/SecondDimensionWatcherReDive/Observability/RuntimeTelemetry.cs @@ -0,0 +1,143 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Observability; + +public sealed class RuntimeTelemetry : IDisposable +{ + public const string MeterName = "SecondDimensionWatcherReDive.Runtime"; + public const string ActivitySourceName = "SecondDimensionWatcherReDive.Runtime"; + + private readonly Meter _meter = new(MeterName); + private readonly Counter _jobAttempts; + private readonly Histogram _jobDuration; + private readonly Counter _scheduledTaskRuns; + private readonly Histogram _scheduledTaskDuration; + private int _pendingJobs; + private int _processingJobs; + private int _deadLetterJobs; + private double _oldestPendingAge; + + public RuntimeTelemetry() + { + _jobAttempts = _meter.CreateCounter( + "sdw.durable_job.attempts", + "{attempt}", + "Durable job execution attempts."); + _jobDuration = _meter.CreateHistogram( + "sdw.durable_job.duration", + "s", + "Durable job execution duration."); + _scheduledTaskRuns = _meter.CreateCounter( + "sdw.scheduled_task.runs", + "{run}", + "Scheduled task run outcomes."); + _scheduledTaskDuration = _meter.CreateHistogram( + "sdw.scheduled_task.duration", + "s", + "Scheduled task request duration."); + _meter.CreateObservableGauge( + "sdw.durable_jobs", + ObserveJobCounts, + "{job}", + "Current durable job counts by status."); + _meter.CreateObservableGauge( + "sdw.durable_job.oldest_pending_age", + () => Volatile.Read(ref _oldestPendingAge), + "s", + "Age of the oldest pending durable job."); + } + + public static Activity? StartDurableJob(DurableJob job) + { + var activity = TelemetryActivitySource.Instance.StartActivity( + "durable_job.process", + ActivityKind.Consumer); + activity?.SetTag("job.type", ToTag(job.Type)); + activity?.SetTag("job.stage", ToTag(job.Stage)); + return activity; + } + + public void RecordJobAttempt( + DurableJobType type, + DurableJobStage stage, + string outcome, + TimeSpan duration) + { + var tags = new TagList + { + { "job.type", ToTag(type) }, + { "job.stage", ToTag(stage) }, + { "outcome", outcome } + }; + _jobAttempts.Add(1, tags); + _jobDuration.Record(duration.TotalSeconds, tags); + } + + public void UpdateJobStatistics(DurableJobStatistics statistics) + { + Volatile.Write(ref _pendingJobs, statistics.PendingCount); + Volatile.Write(ref _processingJobs, statistics.ProcessingCount); + Volatile.Write(ref _deadLetterJobs, statistics.DeadLetterCount); + Volatile.Write(ref _oldestPendingAge, statistics.OldestPendingAgeSeconds); + } + + public void RecordScheduledTask( + string taskId, + string outcome, + TimeSpan duration) + { + var tags = new TagList + { + { "task.id", NormalizeTaskId(taskId) }, + { "outcome", outcome } + }; + _scheduledTaskRuns.Add(1, tags); + _scheduledTaskDuration.Record(duration.TotalSeconds, tags); + } + + public void Dispose() => _meter.Dispose(); + + private IEnumerable> ObserveJobCounts() + { + yield return new Measurement( + Volatile.Read(ref _pendingJobs), + new KeyValuePair("status", "pending")); + yield return new Measurement( + Volatile.Read(ref _processingJobs), + new KeyValuePair("status", "processing")); + yield return new Measurement( + Volatile.Read(ref _deadLetterJobs), + new KeyValuePair("status", "dead_letter")); + } + + private static string ToTag(DurableJobType type) => type switch + { + DurableJobType.DownloadCompletion => "download_completion", + _ => "unknown" + }; + + private static string ToTag(DurableJobStage stage) => stage switch + { + DurableJobStage.MapFiles => "map_files", + DurableJobStage.Notify => "notify", + DurableJobStage.InvokePlugins => "invoke_plugins", + DurableJobStage.Done => "done", + _ => "unknown" + }; + + private static string NormalizeTaskId(string taskId) => taskId switch + { + "SyncFeed" => "sync_feed", + "ScrapeSeasonBangumi" => "scrape_season_bangumi", + "ScanMediaLibraries" => "scan_media_libraries", + "InferAnimationMetadata" => "infer_animation_metadata", + _ => "other" + }; + + private static class TelemetryActivitySource + { + internal static readonly ActivitySource Instance = new(ActivitySourceName); + } +} diff --git a/SecondDimensionWatcherReDive/Observability/SensitiveTagRedactionProcessor.cs b/SecondDimensionWatcherReDive/Observability/SensitiveTagRedactionProcessor.cs new file mode 100644 index 0000000..be5c5e3 --- /dev/null +++ b/SecondDimensionWatcherReDive/Observability/SensitiveTagRedactionProcessor.cs @@ -0,0 +1,49 @@ +using System.Diagnostics; +using OpenTelemetry; + +namespace SecondDimensionWatcherReDive.Observability; + +internal sealed class SensitiveTagRedactionProcessor : BaseProcessor +{ + private static readonly HashSet RemovedTags = new(StringComparer.Ordinal) + { + "db.statement", + "db.query.text", + "url.full", + "url.path", + "url.query", + "http.url", + "http.target", + "tool.arguments", + "tool.result" + }; + + public override void OnEnd(Activity activity) + { + foreach (var key in activity.TagObjects + .Select(pair => pair.Key) + .Where(ShouldRemove) + .ToArray()) + activity.SetTag(key, null); + + if (activity.Source.Name.Contains("EntityFrameworkCore", StringComparison.Ordinal)) + activity.DisplayName = "database.query"; + else if (activity.Source.Name.Contains("HttpClient", StringComparison.Ordinal)) + activity.DisplayName = $"HTTP {GetTag(activity, "http.request.method") ?? "request"}"; + else if (activity.Source.Name.Contains("AspNetCore", StringComparison.Ordinal)) + { + var method = GetTag(activity, "http.request.method") ?? "request"; + var route = GetTag(activity, "http.route"); + activity.DisplayName = route is null ? $"HTTP {method}" : $"{method} {route}"; + } + } + + private static string? GetTag(Activity activity, string key) => + activity.GetTagItem(key)?.ToString(); + + private static bool ShouldRemove(string key) => + RemovedTags.Contains(key) + || key.StartsWith("db.query.parameter.", StringComparison.Ordinal) + || key.StartsWith("tool.argument", StringComparison.Ordinal) + || key.StartsWith("tool.result", StringComparison.Ordinal); +} diff --git a/SecondDimensionWatcherReDive/Program.cs b/SecondDimensionWatcherReDive/Program.cs index 80f5f19..33db7e2 100644 --- a/SecondDimensionWatcherReDive/Program.cs +++ b/SecondDimensionWatcherReDive/Program.cs @@ -4,12 +4,16 @@ using System.Text; using System.Threading.Channels; using AspSpaService; +using Microsoft.AspNetCore.Diagnostics.HealthChecks; using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.DataProtection; using Microsoft.AspNetCore.StaticFiles; using Microsoft.EntityFrameworkCore; using Microsoft.IdentityModel.Tokens; using Microsoft.Net.Http.Headers; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; using SecondDimensionWatcherReDive; using SecondDimensionWatcherReDive.Auth; using SecondDimensionWatcherReDive.Configuration; @@ -19,10 +23,12 @@ using SecondDimensionWatcherReDive.Framework.FileDownload; using SecondDimensionWatcherReDive.Framework.FileStore; using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Framework.Notifications; using SecondDimensionWatcherReDive.Framework.Tasks; using SecondDimensionWatcherReDive.Inference.AI; using SecondDimensionWatcherReDive.Models; using SecondDimensionWatcherReDive.NFS; +using SecondDimensionWatcherReDive.Observability; using SecondDimensionWatcherReDive.Repositories; using SecondDimensionWatcherReDive.Chat; using SecondDimensionWatcherReDive.Plugin; @@ -109,6 +115,116 @@ }); }); +var healthChecks = builder.Services.AddHealthChecks() + .AddCheck( + "postgresql", + tags: [HealthTags.Ready], + timeout: TimeSpan.FromSeconds(10)); +if (!string.IsNullOrEmpty(builder.Configuration["Valkey:ConnectionString"]) + && builder.Configuration.GetValue("Health:ValkeyRequired", true)) + healthChecks.AddCheck( + "valkey", + tags: [HealthTags.Ready], + timeout: TimeSpan.FromSeconds(5)); +if (builder.Configuration.GetValue("Health:QbittorrentRequired", true)) + healthChecks.AddCheck( + "qbittorrent", + tags: [HealthTags.Ready], + timeout: TimeSpan.FromSeconds(5)); +if (builder.Configuration.GetValue("Health:StorageRequired", true)) + healthChecks.AddCheck( + "storage", + tags: [HealthTags.Ready], + timeout: TimeSpan.FromSeconds(5)); +if (builder.Configuration.GetValue("Health:AIRequired", false)) + healthChecks.AddCheck( + "ai", + tags: [HealthTags.Ready], + timeout: TimeSpan.FromSeconds(10)); + +builder.Services.AddSingleton(); +var otlpEndpoint = Uri.TryCreate( + builder.Configuration["OpenTelemetry:OtlpEndpoint"], + UriKind.Absolute, + out var configuredOtlpEndpoint) + ? configuredOtlpEndpoint + : null; +builder.Services.AddOpenTelemetry() + .ConfigureResource(resource => resource.AddService("SecondDimensionWatcherReDive")) + .WithTracing(tracing => + { + tracing + .AddSource(RuntimeTelemetry.ActivitySourceName) + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + // Query parameter capture is disabled by default. The processor below + // also removes statement/query-text tags before export. + .AddEntityFrameworkCoreInstrumentation() + .AddProcessor(new SensitiveTagRedactionProcessor()); + if (otlpEndpoint is not null) + tracing.AddOtlpExporter(options => options.Endpoint = otlpEndpoint); + }) + .WithMetrics(metrics => + { + metrics + .AddMeter(RuntimeTelemetry.MeterName) + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddRuntimeInstrumentation() + .AddView( + "http.server.request.duration", + new ExplicitBucketHistogramConfiguration + { + TagKeys = + [ + "http.request.method", + "http.response.status_code", + "http.route", + "network.protocol.version" + ] + }) + .AddView( + "http.client.request.duration", + new ExplicitBucketHistogramConfiguration + { + TagKeys = + [ + "http.request.method", + "http.response.status_code", + "server.address", + "server.port", + "network.protocol.version" + ] + }) + .AddView( + "sdw.durable_job.attempts", + new MetricStreamConfiguration + { + TagKeys = ["job.type", "job.stage", "outcome"] + }) + .AddView( + "sdw.durable_job.duration", + new ExplicitBucketHistogramConfiguration + { + TagKeys = ["job.type", "job.stage", "outcome"] + }) + .AddView( + "sdw.durable_jobs", + new MetricStreamConfiguration { TagKeys = ["status"] }) + .AddView( + "sdw.scheduled_task.runs", + new MetricStreamConfiguration { TagKeys = ["task.id", "outcome"] }) + .AddView( + "sdw.scheduled_task.duration", + new ExplicitBucketHistogramConfiguration + { + TagKeys = ["task.id", "outcome"] + }) + .AddPrometheusExporter(); + if (otlpEndpoint is not null) + metrics.AddOtlpExporter(options => options.Endpoint = otlpEndpoint); + }); + //Configure JWT var key = Encoding.ASCII.GetBytes(builder.Configuration["JwtSecret"] ?? throw new ApplicationException("JwtSecret must present in the config file.")); @@ -208,10 +324,29 @@ contentTypeProvider.Mappings.Add(".mkv", "video/x-matroska"); builder.Services.AddSingleton(contentTypeProvider); -//Add channels -builder.Services.AddSingleton(Channel.CreateUnbounded()); -builder.Services.AddSingleton(Channel.CreateUnbounded()); -builder.Services.AddSingleton(Channel.CreateUnbounded()); +// In-process channels are bounded. Download completion is persisted before its +// wake hint is emitted, while high-frequency progress may safely drop old samples. +builder.Services.AddSingleton(Channel.CreateBounded( + new BoundedChannelOptions(1024) + { + SingleReader = true, + SingleWriter = false, + FullMode = BoundedChannelFullMode.Wait + })); +builder.Services.AddSingleton(Channel.CreateBounded( + new BoundedChannelOptions(1024) + { + SingleReader = true, + SingleWriter = true, + FullMode = BoundedChannelFullMode.DropOldest + })); +builder.Services.AddSingleton(Channel.CreateBounded( + new BoundedChannelOptions(128) + { + SingleReader = true, + SingleWriter = true, + FullMode = BoundedChannelFullMode.DropOldest + })); // Persistent incident inbox and health probes. builder.Services.AddSingleton(); @@ -219,6 +354,7 @@ //Add hosting services builder.Services.AddHostedService(); +builder.Services.AddHostedService(); builder.Services.AddHostedService(); builder.Services.AddHostedService(); builder.Services.AddHostedService(); @@ -226,8 +362,10 @@ builder.Services.AddSingleton(sp => sp.GetRequiredService()); builder.Services.AddHostedService(); +builder.Services.AddSingleton(); //Add scheduled tasks +builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(sp => sp.GetRequiredService()); builder.Services.AddHostedService>(); @@ -277,6 +415,9 @@ builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); builder.Services.AddSingleton(); builder.Services.AddScoped(); builder.Services.AddScoped(); @@ -320,13 +461,27 @@ app.UseRouting(); +app.MapHealthChecks("/health/live", new HealthCheckOptions +{ + Predicate = _ => false, + ResponseWriter = HealthResponseWriter.WriteAsync +}).AllowAnonymous(); +app.MapHealthChecks("/health/ready", new HealthCheckOptions +{ + Predicate = registration => registration.Tags.Contains(HealthTags.Ready), + ResponseWriter = HealthResponseWriter.WriteAsync +}).AllowAnonymous(); +app.MapPrometheusScrapingEndpoint("/metrics").AllowAnonymous(); + app.MapControllers(); if (app.Environment.IsDevelopment()) { app.UseWhen( context => !context.Request.Path.StartsWithSegments("/api") && - !context.Request.Path.StartsWithSegments("/webdav"), + !context.Request.Path.StartsWithSegments("/webdav") && + !context.Request.Path.StartsWithSegments("/health") && + !context.Request.Path.StartsWithSegments("/metrics"), then => { then.UseSpa(config => diff --git a/SecondDimensionWatcherReDive/Repositories/AnimationInfoRepository.cs b/SecondDimensionWatcherReDive/Repositories/AnimationInfoRepository.cs index d90f6b6..65b0b44 100644 --- a/SecondDimensionWatcherReDive/Repositories/AnimationInfoRepository.cs +++ b/SecondDimensionWatcherReDive/Repositories/AnimationInfoRepository.cs @@ -1,4 +1,5 @@ using System.Runtime.CompilerServices; +using System.Text.Json; using Microsoft.EntityFrameworkCore; using SecondDimensionWatcherReDive.Framework.DataRepository; using SecondDimensionWatcherReDive.Framework.FileDownload; @@ -490,9 +491,37 @@ SubscriptionAutomationDisposition.AutoDownloadQueued or if (changed) { entity.StateVersion = checked(entity.StateVersion + 1); - await writeContext.SaveChangesAsync(cancellationToken); } + // Commit the durable side-effect workflow in the same transaction as + // the download completion state. A retry sees the same unique key and + // cannot create a second workflow. + var deduplicationKey = + $"download-completion:{id:N}:{downloadAttemptId?.ToString("N") ?? "legacy"}"; + if (!await writeContext.DurableJobs.AnyAsync( + job => job.DeduplicationKey == deduplicationKey, + cancellationToken)) + { + writeContext.DurableJobs.Add(new Models.DurableJob + { + Id = Guid.NewGuid(), + DeduplicationKey = deduplicationKey, + Type = DurableJobType.DownloadCompletion, + Status = DurableJobStatus.Pending, + Stage = DurableJobStage.MapFiles, + PayloadJson = JsonSerializer.Serialize(new DownloadCompletionJobPayload( + id, + storePath, + fileStore, + downloadAttemptId)), + CreatedAt = completedAt, + UpdatedAt = completedAt, + NextAttemptAt = completedAt + }); + } + + await writeContext.SaveChangesAsync(cancellationToken); + await writeContext.Entry(entity) .Reference(info => info.Animation) .LoadAsync(cancellationToken); diff --git a/SecondDimensionWatcherReDive/Repositories/DurableJobRepository.cs b/SecondDimensionWatcherReDive/Repositories/DurableJobRepository.cs new file mode 100644 index 0000000..853e54d --- /dev/null +++ b/SecondDimensionWatcherReDive/Repositories/DurableJobRepository.cs @@ -0,0 +1,218 @@ +using Microsoft.EntityFrameworkCore; +using SecondDimensionWatcherReDive.Framework.DataRepository; +using JobEntity = SecondDimensionWatcherReDive.Models.DurableJob; + +namespace SecondDimensionWatcherReDive.Repositories; + +public sealed class DurableJobRepository(Models.ApplicationContext context) + : IDurableJobRepository +{ + public async Task> ClaimDueAsync( + string workerId, + DateTimeOffset now, + DateTimeOffset leaseUntil, + int take, + CancellationToken cancellationToken) + { + var candidateIds = await context.DurableJobs + .AsNoTracking() + .Where(job => + job.NextAttemptAt <= now + && (job.Status == DurableJobStatus.Pending + || (job.Status == DurableJobStatus.Processing + && job.LeaseExpiresAt <= now))) + .OrderBy(job => job.NextAttemptAt) + .ThenBy(job => job.CreatedAt) + .Select(job => job.Id) + .Take(take) + .ToListAsync(cancellationToken); + + var claimedIds = new List(candidateIds.Count); + foreach (var id in candidateIds) + { + var affected = await context.DurableJobs + .Where(job => job.Id == id + && job.NextAttemptAt <= now + && (job.Status == DurableJobStatus.Pending + || (job.Status == DurableJobStatus.Processing + && job.LeaseExpiresAt <= now))) + .ExecuteUpdateAsync(setters => setters + .SetProperty(job => job.Status, DurableJobStatus.Processing) + .SetProperty(job => job.LeaseOwner, workerId) + .SetProperty(job => job.LeaseExpiresAt, leaseUntil) + .SetProperty(job => job.UpdatedAt, now), cancellationToken); + if (affected == 1) + claimedIds.Add(id); + } + + if (claimedIds.Count == 0) + return []; + + return (await context.DurableJobs + .AsNoTracking() + .Where(job => claimedIds.Contains(job.Id)) + .OrderBy(job => job.CreatedAt) + .ToListAsync(cancellationToken)) + .Select(ToRecord) + .ToList(); + } + + public async Task AdvanceStageAsync( + Guid id, + string workerId, + DurableJobStage expectedStage, + DurableJobStage nextStage, + DateTimeOffset now, + CancellationToken cancellationToken) + { + var completed = nextStage == DurableJobStage.Done; + var affected = await context.DurableJobs + .Where(job => job.Id == id + && job.Status == DurableJobStatus.Processing + && job.LeaseOwner == workerId + && job.LeaseExpiresAt > now + && job.Stage == expectedStage) + .ExecuteUpdateAsync(setters => setters + .SetProperty(job => job.Stage, nextStage) + .SetProperty(job => job.Status, + completed ? DurableJobStatus.Completed : DurableJobStatus.Processing) + .SetProperty(job => job.UpdatedAt, now) + .SetProperty(job => job.LastAttemptAt, now) + .SetProperty(job => job.CompletedAt, completed ? now : null) + .SetProperty(job => job.LeaseOwner, completed ? null : workerId) + .SetProperty(job => job.LeaseExpiresAt, + job => completed ? null : job.LeaseExpiresAt) + .SetProperty(job => job.LastError, (string?)null), cancellationToken); + return affected == 1; + } + + public async Task RenewLeaseAsync( + Guid id, + string workerId, + DateTimeOffset now, + DateTimeOffset leaseUntil, + CancellationToken cancellationToken) + { + var affected = await context.DurableJobs + .Where(job => job.Id == id + && job.Status == DurableJobStatus.Processing + && job.LeaseOwner == workerId + && job.LeaseExpiresAt > now) + .ExecuteUpdateAsync(setters => setters + .SetProperty(job => job.LeaseExpiresAt, leaseUntil) + .SetProperty(job => job.UpdatedAt, now), cancellationToken); + return affected == 1; + } + + public Task MarkFailedAsync( + Guid id, + string workerId, + int attemptCount, + DateTimeOffset attemptedAt, + DateTimeOffset? nextAttemptAt, + string error, + CancellationToken cancellationToken) => + context.DurableJobs + .Where(job => job.Id == id + && job.Status == DurableJobStatus.Processing + && job.LeaseOwner == workerId) + .ExecuteUpdateAsync(setters => setters + .SetProperty(job => job.Status, + nextAttemptAt.HasValue + ? DurableJobStatus.Pending + : DurableJobStatus.DeadLetter) + .SetProperty(job => job.AttemptCount, attemptCount) + .SetProperty(job => job.LastAttemptAt, attemptedAt) + .SetProperty(job => job.UpdatedAt, attemptedAt) + .SetProperty(job => job.NextAttemptAt, nextAttemptAt ?? attemptedAt) + .SetProperty(job => job.LeaseOwner, (string?)null) + .SetProperty(job => job.LeaseExpiresAt, (DateTimeOffset?)null) + .SetProperty(job => job.LastError, error), cancellationToken); + + public async Task GetPageAsync( + DurableJobStatus? status, + int skip, + int take, + CancellationToken cancellationToken) + { + var query = context.DurableJobs.AsNoTracking().AsQueryable(); + if (status.HasValue) + query = query.Where(job => job.Status == status.Value); + + var totalCount = await query.CountAsync(cancellationToken); + var jobs = await query + .OrderByDescending(job => job.UpdatedAt) + .ThenByDescending(job => job.CreatedAt) + .Skip(skip) + .Take(take) + .ToListAsync(cancellationToken); + return new DurableJobPage(jobs.Select(ToRecord).ToList(), totalCount); + } + + public Task RetryAsync( + IReadOnlyCollection ids, + DateTimeOffset now, + CancellationToken cancellationToken) => + context.DurableJobs + .Where(job => ids.Contains(job.Id) + && job.Status == DurableJobStatus.DeadLetter) + .ExecuteUpdateAsync(setters => setters + .SetProperty(job => job.Status, DurableJobStatus.Pending) + .SetProperty(job => job.AttemptCount, 0) + .SetProperty(job => job.NextAttemptAt, now) + .SetProperty(job => job.UpdatedAt, now) + .SetProperty(job => job.LastError, (string?)null) + .SetProperty(job => job.LeaseOwner, (string?)null) + .SetProperty(job => job.LeaseExpiresAt, (DateTimeOffset?)null), cancellationToken); + + public Task ResolveAsync( + IReadOnlyCollection ids, + DateTimeOffset now, + CancellationToken cancellationToken) => + context.DurableJobs + .Where(job => ids.Contains(job.Id) + && job.Status == DurableJobStatus.DeadLetter) + .ExecuteUpdateAsync(setters => setters + .SetProperty(job => job.Status, DurableJobStatus.Resolved) + .SetProperty(job => job.CompletedAt, now) + .SetProperty(job => job.UpdatedAt, now) + .SetProperty(job => job.LeaseOwner, (string?)null) + .SetProperty(job => job.LeaseExpiresAt, (DateTimeOffset?)null), cancellationToken); + + public async Task GetStatisticsAsync( + DateTimeOffset now, + CancellationToken cancellationToken) + { + var pendingCount = await context.DurableJobs.CountAsync( + job => job.Status == DurableJobStatus.Pending, cancellationToken); + var processingCount = await context.DurableJobs.CountAsync( + job => job.Status == DurableJobStatus.Processing, cancellationToken); + var deadLetterCount = await context.DurableJobs.CountAsync( + job => job.Status == DurableJobStatus.DeadLetter, cancellationToken); + var oldest = await context.DurableJobs + .Where(job => job.Status == DurableJobStatus.Pending) + .MinAsync(job => (DateTimeOffset?)job.CreatedAt, cancellationToken); + return new DurableJobStatistics( + pendingCount, + processingCount, + deadLetterCount, + oldest.HasValue ? Math.Max(0, (now - oldest.Value).TotalSeconds) : 0); + } + + private static DurableJob ToRecord(JobEntity job) => new( + job.Id, + job.DeduplicationKey, + job.Type, + job.Status, + job.Stage, + job.PayloadJson, + job.AttemptCount, + job.CreatedAt, + job.UpdatedAt, + job.NextAttemptAt, + job.LastAttemptAt, + job.CompletedAt, + job.LeaseOwner, + job.LeaseExpiresAt, + job.LastError); +} diff --git a/SecondDimensionWatcherReDive/Repositories/FileMappingRepositoryPostgreSqlTestFixture.cs b/SecondDimensionWatcherReDive/Repositories/FileMappingRepositoryPostgreSqlTestFixture.cs index 34b17f5..4068035 100644 --- a/SecondDimensionWatcherReDive/Repositories/FileMappingRepositoryPostgreSqlTestFixture.cs +++ b/SecondDimensionWatcherReDive/Repositories/FileMappingRepositoryPostgreSqlTestFixture.cs @@ -24,7 +24,7 @@ public async Task ResetAsync(CancellationToken cancellationToken) { await using var context = new Models.ApplicationContext(_contextOptions); await context.Database.ExecuteSqlRawAsync( - "TRUNCATE TABLE \"FileMappings\", \"AnimationInfo\" RESTART IDENTITY CASCADE", + "TRUNCATE TABLE \"DurableJobs\", \"ScheduledTaskStates\", \"FileMappings\", \"AnimationInfo\" RESTART IDENTITY CASCADE", cancellationToken); } @@ -83,4 +83,200 @@ public async Task GetAnimationInfoStateVersionsAsync(CancellationToken c .Select(info => info.StateVersion) .ToArrayAsync(cancellationToken); } + + public async Task<(Guid ItemId, Guid AttemptId)> SeedTrackedAnimationAsync( + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + var attemptId = Guid.NewGuid(); + var info = new Models.AnimationInfo + { + Id = Guid.NewGuid(), + Title = "tracked integration test", + IsDownloadTracked = true, + DownloadAttemptId = attemptId, + AutomationDisposition = SubscriptionAutomationDisposition.ManualDownloadQueued + }; + context.AnimationInfo.Add(info); + await context.SaveChangesAsync(cancellationToken); + return (info.Id, attemptId); + } + + public async Task CompleteTrackedAnimationAsync( + Guid itemId, + Guid attemptId, + string storePath, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + var repository = new AnimationInfoRepository(context, _contextOptions); + var result = await repository.TryCompleteDownloadAsync( + itemId, + attemptId, + "local", + storePath, + DateTimeOffset.UtcNow, + cancellationToken); + if (result is null) + throw new InvalidOperationException("The tracked download was not completed."); + } + + public async Task<(bool IsFinished, int JobCount, DownloadCompletionJobPayload Payload)> + GetCompletionStateAsync(Guid itemId, CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + var finished = await context.AnimationInfo + .Where(info => info.Id == itemId) + .Select(info => info.IsDownloadFinished) + .SingleAsync(cancellationToken); + var jobs = await context.DurableJobs + .Where(job => job.Type == DurableJobType.DownloadCompletion) + .ToListAsync(cancellationToken); + var payload = System.Text.Json.JsonSerializer + .Deserialize(jobs.Single().PayloadJson)!; + return (finished, jobs.Count, payload); + } + + public async Task SeedDurableJobAsync( + DurableJob job, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + context.DurableJobs.Add(new Models.DurableJob + { + Id = job.Id, + DeduplicationKey = job.DeduplicationKey, + Type = job.Type, + Status = job.Status, + Stage = job.Stage, + PayloadJson = job.PayloadJson, + AttemptCount = job.AttemptCount, + CreatedAt = job.CreatedAt, + UpdatedAt = job.UpdatedAt, + NextAttemptAt = job.NextAttemptAt, + LastAttemptAt = job.LastAttemptAt, + CompletedAt = job.CompletedAt, + LeaseOwner = job.LeaseOwner, + LeaseExpiresAt = job.LeaseExpiresAt, + LastError = job.LastError + }); + await context.SaveChangesAsync(cancellationToken); + } + + public async Task> ClaimDueJobsAsync( + string ownerId, + DateTimeOffset now, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + var repository = new DurableJobRepository(context); + return await repository.ClaimDueAsync( + ownerId, + now, + now.AddMinutes(1), + 10, + cancellationToken); + } + + public async Task RenewDurableJobLeaseAsync( + Guid id, + string ownerId, + DateTimeOffset now, + DateTimeOffset leaseUntil, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + return await new DurableJobRepository(context).RenewLeaseAsync( + id, + ownerId, + now, + leaseUntil, + cancellationToken); + } + + public async Task AdvanceDurableJobAsync( + Guid id, + string ownerId, + DurableJobStage expectedStage, + DurableJobStage nextStage, + DateTimeOffset now, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + return await new DurableJobRepository(context).AdvanceStageAsync( + id, + ownerId, + expectedStage, + nextStage, + now, + cancellationToken); + } + + public async Task TryAcquireTaskLeaseAsync( + string taskId, + string ownerId, + DateTimeOffset now, + DateTimeOffset leaseUntil, + bool force, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + var repository = new ScheduledTaskLeaseRepository(context); + return await repository.TryAcquireAsync( + taskId, + ownerId, + now, + leaseUntil, + force, + cancellationToken); + } + + public async Task CompleteTaskLeaseAsync( + string taskId, + string ownerId, + DateTimeOffset completedAt, + DateTimeOffset nextRunAt, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + await new ScheduledTaskLeaseRepository(context).CompleteAsync( + taskId, + ownerId, + completedAt, + nextRunAt, + true, + null, + cancellationToken); + } + + public async Task RetryJobsAsync( + IReadOnlyCollection ids, + DateTimeOffset now, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + return await new DurableJobRepository(context) + .RetryAsync(ids, now, cancellationToken); + } + + public async Task ResolveJobsAsync( + IReadOnlyCollection ids, + DateTimeOffset now, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + return await new DurableJobRepository(context) + .ResolveAsync(ids, now, cancellationToken); + } + + public async Task GetJobStatusAsync( + Guid id, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + return await context.DurableJobs + .Where(job => job.Id == id) + .Select(job => job.Status) + .SingleAsync(cancellationToken); + } } diff --git a/SecondDimensionWatcherReDive/Repositories/ReadinessRepository.cs b/SecondDimensionWatcherReDive/Repositories/ReadinessRepository.cs new file mode 100644 index 0000000..44c7ddd --- /dev/null +++ b/SecondDimensionWatcherReDive/Repositories/ReadinessRepository.cs @@ -0,0 +1,11 @@ +using Microsoft.EntityFrameworkCore; +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Repositories; + +public sealed class ReadinessRepository(Models.ApplicationContext context) + : IReadinessRepository +{ + public Task CanConnectAsync(CancellationToken cancellationToken) => + context.Database.CanConnectAsync(cancellationToken); +} diff --git a/SecondDimensionWatcherReDive/Repositories/ScheduledTaskLeaseRepository.cs b/SecondDimensionWatcherReDive/Repositories/ScheduledTaskLeaseRepository.cs new file mode 100644 index 0000000..7e1d49f --- /dev/null +++ b/SecondDimensionWatcherReDive/Repositories/ScheduledTaskLeaseRepository.cs @@ -0,0 +1,95 @@ +using Microsoft.EntityFrameworkCore; +using Npgsql; +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Repositories; + +public sealed class ScheduledTaskLeaseRepository(Models.ApplicationContext context) + : IScheduledTaskLeaseRepository +{ + public async Task TryAcquireAsync( + string taskId, + string ownerId, + DateTimeOffset now, + DateTimeOffset leaseUntil, + bool force, + CancellationToken cancellationToken) + { + var affected = await context.ScheduledTaskStates + .Where(state => state.TaskId == taskId + && (state.LeaseOwner == null + || state.LeaseExpiresAt <= now + || state.LeaseOwner == ownerId + || (force + && state.LastCompletedAt != null + && (state.LastStartedAt == null + || state.LastCompletedAt >= state.LastStartedAt)))) + .ExecuteUpdateAsync(setters => setters + .SetProperty(state => state.LeaseOwner, ownerId) + .SetProperty(state => state.LeaseExpiresAt, leaseUntil) + .SetProperty(state => state.LastStartedAt, now) + .SetProperty(state => state.RunCount, state => state.RunCount + 1), + cancellationToken); + if (affected == 1) + return true; + + var state = new Models.ScheduledTaskState + { + TaskId = taskId, + LeaseOwner = ownerId, + LeaseExpiresAt = leaseUntil, + LastStartedAt = now, + RunCount = 1 + }; + await context.ScheduledTaskStates.AddAsync(state, cancellationToken); + try + { + await context.SaveChangesAsync(cancellationToken); + return true; + } + catch (DbUpdateException exception) when ( + exception.InnerException is PostgresException { SqlState: PostgresErrorCodes.UniqueViolation }) + { + context.ChangeTracker.Clear(); + return false; + } + } + + public async Task RenewAsync( + string taskId, + string ownerId, + DateTimeOffset now, + DateTimeOffset leaseUntil, + CancellationToken cancellationToken) + { + var affected = await context.ScheduledTaskStates + .Where(state => state.TaskId == taskId + && state.LeaseOwner == ownerId) + .ExecuteUpdateAsync(setters => setters + .SetProperty(state => state.LeaseExpiresAt, leaseUntil), cancellationToken); + return affected == 1; + } + + public Task CompleteAsync( + string taskId, + string ownerId, + DateTimeOffset completedAt, + DateTimeOffset leaseUntil, + bool succeeded, + string? error, + CancellationToken cancellationToken) => + context.ScheduledTaskStates + .Where(state => state.TaskId == taskId + && state.LeaseOwner == ownerId) + .ExecuteUpdateAsync(setters => setters + // Keep a cooldown lease until the next periodic due time. Other + // instances poll this row and can take over promptly after a crash + // without immediately duplicating a normally completed run. + .SetProperty(state => state.LeaseOwner, ownerId) + .SetProperty(state => state.LeaseExpiresAt, leaseUntil) + .SetProperty(state => state.LastCompletedAt, completedAt) + .SetProperty(state => state.LastSucceededAt, + state => succeeded ? completedAt : state.LastSucceededAt) + .SetProperty(state => state.LastError, + succeeded ? null : error), cancellationToken); +} diff --git a/SecondDimensionWatcherReDive/SecondDimensionWatcherReDive.csproj b/SecondDimensionWatcherReDive/SecondDimensionWatcherReDive.csproj index caf806d..5840222 100644 --- a/SecondDimensionWatcherReDive/SecondDimensionWatcherReDive.csproj +++ b/SecondDimensionWatcherReDive/SecondDimensionWatcherReDive.csproj @@ -31,6 +31,13 @@ + + + + + + + diff --git a/SecondDimensionWatcherReDive/Services/CompleteDownloadBackgroundService.cs b/SecondDimensionWatcherReDive/Services/CompleteDownloadBackgroundService.cs index d6f2eb9..00bafdb 100644 --- a/SecondDimensionWatcherReDive/Services/CompleteDownloadBackgroundService.cs +++ b/SecondDimensionWatcherReDive/Services/CompleteDownloadBackgroundService.cs @@ -1,206 +1,342 @@ +using System.Diagnostics; +using System.Text.Json; using System.Threading.Channels; using SecondDimensionWatcherReDive.Data; -using SecondDimensionWatcherReDive.Framework.PluginParams; using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Framework.Notifications; +using SecondDimensionWatcherReDive.Framework.PluginParams; using SecondDimensionWatcherReDive.Plugin; +using SecondDimensionWatcherReDive.Observability; using SecondDimensionWatcherReDive.Utils.FileStore; using SecondDimensionWatcherReDive.Utils.Incidents; namespace SecondDimensionWatcherReDive.Services; +/// +/// Executes persisted download-completion effects. The channel is deliberately +/// only a wake-up hint: polling and expired leases make work recoverable after a +/// process crash or a lost hint. +/// public partial class CompleteDownloadBackgroundService( Channel downloadCompleteRequest, IServiceScopeFactory scopeFactory, ILogger logger, - IIncidentReporter? incidentReporter = null) + IIncidentReporter? incidentReporter = null, + RuntimeTelemetry? telemetry = null) : BackgroundService { + internal const int MaxAttempts = 8; + internal static readonly TimeSpan PollInterval = TimeSpan.FromSeconds(10); + internal static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(2); + internal static readonly TimeSpan LeaseRenewInterval = TimeSpan.FromSeconds(30); + + private readonly string _workerId = $"{Environment.MachineName}:{Environment.ProcessId}:{Guid.NewGuid():N}"; + protected override async Task ExecuteAsync(CancellationToken cancellationToken) { - var reader = downloadCompleteRequest.Reader; - while (await reader.WaitToReadAsync(cancellationToken)) + while (!cancellationToken.IsCancellationRequested) { - var request = await reader.ReadAsync(cancellationToken); - for (var attempt = 1; attempt <= 3; attempt++) + var processed = 0; + try { - try - { - await ProcessRequestAsync(request, cancellationToken); - break; - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - throw; - } - catch (Exception ex) when (attempt < 3) - { - LogProcessingRequestRetry(logger, ex, request.ItemId, attempt); - await Task.Delay(TimeSpan.FromMilliseconds(250 * attempt), cancellationToken); - } - catch (Exception ex) - { - LogProcessingRequestFailed(logger, ex, request.ItemId); - await ReportCompletionFailureAsync(request, ex, cancellationToken); - - // The torrent tracker removes finished torrents after handing - // them to this queue. Requeue here so a transient database - // outage cannot permanently lose the completion transition. - await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken); - await downloadCompleteRequest.Writer.WriteAsync(request, cancellationToken); - LogProcessingRequestRequeued(logger, request.ItemId); - } + processed = await ProcessDueJobsAsync(cancellationToken); } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + // A temporary database outage must not terminate the hosted service. + LogPollFailed(logger, exception); + } + + if (processed > 0) + continue; + + await WaitForWakeOrPollAsync(cancellationToken); } } - internal async Task ProcessRequestAsync( - DownloadCompleteRequest request, - CancellationToken cancellationToken) + internal async Task ProcessDueJobsAsync(CancellationToken cancellationToken) { - LogProcessingRequest(logger, request.ItemId, request.StorePath, request.FileStore); - await using var scope = scopeFactory.CreateAsyncScope(); - var animationInfoRepository = scope.ServiceProvider.GetRequiredService(); - - var info = await animationInfoRepository.TryCompleteDownloadAsync( - request.ItemId, - request.DownloadAttemptId, - request.FileStore, - request.StorePath, - DateTimeOffset.Now, + var repository = scope.ServiceProvider.GetRequiredService(); + var now = DateTimeOffset.UtcNow; + var jobs = await repository.ClaimDueAsync( + _workerId, + now, + now + LeaseDuration, + 1, cancellationToken); - if (info is null) - { - LogCompletionIgnored(logger, request.ItemId); - return; - } - LogDownloadMarkedFinished(logger, request.ItemId, info.Title); + foreach (var job in jobs) + await ProcessClaimedJobAsync(scope.ServiceProvider, repository, job, cancellationToken); - if (incidentReporter is not null) - { - await incidentReporter.ResolveAsync( - IncidentType.DownloadStalled, - request.ItemId.ToString(), - cancellationToken); - } + return jobs.Count; + } - // Build virtual-fs mappings for the downloaded files. + internal async Task ProcessClaimedJobAsync( + IServiceProvider serviceProvider, + IDurableJobRepository repository, + DurableJob job, + CancellationToken cancellationToken) + { + var startedAt = Stopwatch.GetTimestamp(); + var currentStage = job.Stage; + using var activity = RuntimeTelemetry.StartDurableJob(job); + using var renewalCancellation = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken); + using var leaseLost = new CancellationTokenSource(); + using var effectCancellation = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + leaseLost.Token); + var renewalTask = RenewJobLeaseAsync( + job.Id, + leaseLost, + renewalCancellation.Token); + Guid? itemId = null; try { - var fileMapper = scope.ServiceProvider.GetRequiredService(); - if (!await fileMapper.MapDownloadAsync(request.ItemId, cancellationToken)) - throw new InvalidOperationException("No file mapping could be produced."); + if (job.Type != DurableJobType.DownloadCompletion) + throw new NotSupportedException($"Unsupported durable job type: {job.Type}"); + + var payload = JsonSerializer.Deserialize(job.PayloadJson) + ?? throw new JsonException("The durable job payload is empty."); + itemId = payload.ItemId; + var stage = job.Stage; + currentStage = stage; - if (incidentReporter is not null) + if (stage == DurableJobStage.MapFiles) { - await incidentReporter.ResolveAsync( - IncidentType.FileMappingFailure, - request.ItemId.ToString(), - cancellationToken); + var mapper = serviceProvider.GetRequiredService(); + if (!await mapper.MapDownloadAsync(payload.ItemId, effectCancellation.Token)) + throw new InvalidOperationException("No file mapping could be produced."); + + if (incidentReporter is not null) + await incidentReporter.ResolveAsync( + IncidentType.FileMappingFailure, + payload.ItemId.ToString(), + effectCancellation.Token); + + await AdvanceAsync( + repository, job.Id, stage, DurableJobStage.Notify, effectCancellation.Token); + stage = DurableJobStage.Notify; + currentStage = stage; + } + + if (stage == DurableJobStage.Notify) + { + var notifier = serviceProvider.GetRequiredService(); + await notifier.NotifyAsync(job.Id, payload, effectCancellation.Token); + await AdvanceAsync( + repository, job.Id, stage, DurableJobStage.InvokePlugins, effectCancellation.Token); + stage = DurableJobStage.InvokePlugins; + currentStage = stage; } + + if (stage == DurableJobStage.InvokePlugins) + { + var eventTrigger = serviceProvider + .GetRequiredService>(); + await eventTrigger.InvokeAsync( + new FileDownloadCompleteParam( + payload.ItemId, + payload.StorePath, + payload.FileStore, + job.Id), + effectCancellation.Token); + await AdvanceAsync( + repository, job.Id, stage, DurableJobStage.Done, effectCancellation.Token); + } + + LogJobCompleted(logger, job.Id, payload.ItemId); + activity?.SetStatus(ActivityStatusCode.Ok); + telemetry?.RecordJobAttempt( + job.Type, + currentStage, + "completed", + Stopwatch.GetElapsedTime(startedAt)); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (OperationCanceledException) when (leaseLost.IsCancellationRequested) + { + // Another worker may already own the expired lease. Do not mutate the + // job with stale ownership; its persisted stage remains resumable. + LogJobLeaseLost(logger, job.Id); } - catch (Exception ex) + catch (Exception exception) { - LogFileMappingFailed(logger, ex, request.ItemId); - if (incidentReporter is not null) + var attemptCount = job.AttemptCount + 1; + var attemptedAt = DateTimeOffset.UtcNow; + var retryAt = attemptCount >= MaxAttempts + ? (DateTimeOffset?)null + : attemptedAt + RetryDelay(attemptCount); + var error = LimitError(exception); + + await repository.MarkFailedAsync( + job.Id, + _workerId, + attemptCount, + attemptedAt, + retryAt, + error, + cancellationToken); + + if (currentStage == DurableJobStage.MapFiles && incidentReporter is not null) { await incidentReporter.ReportAsync(new IncidentReport( IncidentType.FileMappingFailure, IncidentSeverity.Error, "Downloaded files could not be mapped", - ex.Message, - request.ItemId.ToString()), + error, + (itemId ?? job.Id).ToString()), cancellationToken); } + + if (retryAt.HasValue) + LogJobRetry(logger, exception, job.Id, attemptCount, retryAt.Value); + else + LogJobDeadLettered(logger, exception, job.Id, attemptCount); + activity?.SetStatus(ActivityStatusCode.Error, exception.GetType().Name); + telemetry?.RecordJobAttempt( + job.Type, + currentStage, + retryAt.HasValue ? "retry" : "dead_letter", + Stopwatch.GetElapsedTime(startedAt)); + } + finally + { + await renewalCancellation.CancelAsync(); + await renewalTask; } + } - // Fire plugin event + private async Task RenewJobLeaseAsync( + Guid jobId, + CancellationTokenSource leaseLost, + CancellationToken cancellationToken) + { try { - LogFiringPluginEvent(logger, request.ItemId); - var eventTrigger = scope.ServiceProvider - .GetRequiredService>(); - await eventTrigger.InvokeAsync(new FileDownloadCompleteParam( - request.ItemId, request.StorePath, request.FileStore), cancellationToken); - LogPluginEventCompleted(logger, request.ItemId); + using var timer = new PeriodicTimer(LeaseRenewInterval); + while (await timer.WaitForNextTickAsync(cancellationToken)) + { + var now = DateTimeOffset.UtcNow; + await using var scope = scopeFactory.CreateAsyncScope(); + var repository = scope.ServiceProvider.GetRequiredService(); + if (await repository.RenewLeaseAsync( + jobId, + _workerId, + now, + now + LeaseDuration, + cancellationToken)) + continue; + + leaseLost.Cancel(); + return; + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { } - catch (Exception ex) + catch (Exception exception) { - LogDownloadCompletedEventFailed(logger, ex, request.ItemId); + LogJobLeaseRenewalFailed(logger, exception, jobId); + leaseLost.Cancel(); } } - private async Task ReportCompletionFailureAsync( - DownloadCompleteRequest request, - Exception exception, + private async Task AdvanceAsync( + IDurableJobRepository repository, + Guid jobId, + DurableJobStage expectedStage, + DurableJobStage nextStage, CancellationToken cancellationToken) { - if (incidentReporter is null) return; + var advanced = await repository.AdvanceStageAsync( + jobId, + _workerId, + expectedStage, + nextStage, + DateTimeOffset.UtcNow, + cancellationToken); + if (!advanced) + throw new InvalidOperationException("The durable job lease was lost."); + } + private async Task WaitForWakeOrPollAsync(CancellationToken cancellationToken) + { + var reader = downloadCompleteRequest.Reader; + using var waitCancellation = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken); + waitCancellation.CancelAfter(PollInterval); try { - await incidentReporter.ReportAsync(new IncidentReport( - IncidentType.DownloadStalled, - IncidentSeverity.Error, - "Download completion could not be persisted", - exception.Message, - request.ItemId.ToString()), - cancellationToken); + await reader.WaitToReadAsync(waitCancellation.Token); } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + catch (OperationCanceledException) when ( + !cancellationToken.IsCancellationRequested + && waitCancellation.IsCancellationRequested) { - throw; + // The bounded timeout is the polling signal. Cancelling the channel + // wait prevents an abandoned waiter from accumulating every cycle. } - catch (Exception reportException) + + // Coalesce all hints. Their payload is intentionally not processed here; + // every authoritative request is already persisted transactionally. + while (reader.TryRead(out _)) { - // Persistence may be unavailable for both the completion and its - // incident. The queued retry remains the source of eventual recovery. - LogCompletionIncidentFailed(logger, reportException, request.ItemId); } } - [LoggerMessage(Level = LogLevel.Information, Message = "Processing download complete request for {ItemId}, storePath: {StorePath}, fileStore: {FileStore}")] - private static partial void LogProcessingRequest(ILogger logger, Guid itemId, string storePath, string fileStore); + internal static TimeSpan RetryDelay(int attemptCount) + { + var seconds = Math.Min(900, 5 * Math.Pow(2, Math.Max(0, attemptCount - 1))); + return TimeSpan.FromSeconds(seconds); + } - [LoggerMessage(Level = LogLevel.Information, - Message = "Ignoring completion for {ItemId} because it was cancelled, already completed, or removed")] - private static partial void LogCompletionIgnored(ILogger logger, Guid itemId); + private static string LimitError(Exception exception) + { + var value = $"{exception.GetType().Name}: {exception.Message}"; + return value.Length <= 512 ? value : value[..512]; + } - [LoggerMessage(Level = LogLevel.Error, Message = "Failed to process completion for {ItemId}")] - private static partial void LogProcessingRequestFailed(ILogger logger, Exception exception, Guid itemId); + [LoggerMessage(Level = LogLevel.Warning, Message = "Durable completion polling failed; it will retry")] + private static partial void LogPollFailed(ILogger logger, Exception exception); + + [LoggerMessage(Level = LogLevel.Information, + Message = "Durable completion job {JobId} completed for {ItemId}")] + private static partial void LogJobCompleted(ILogger logger, Guid jobId, Guid itemId); [LoggerMessage(Level = LogLevel.Warning, - Message = "Retrying completion for {ItemId} after attempt {Attempt}")] - private static partial void LogProcessingRequestRetry( + Message = "Durable completion job {JobId} failed on attempt {Attempt}; retrying at {RetryAt}")] + private static partial void LogJobRetry( + ILogger logger, + Exception exception, + Guid jobId, + int attempt, + DateTimeOffset retryAt); + + [LoggerMessage(Level = LogLevel.Error, + Message = "Durable completion job {JobId} entered dead-letter after {Attempt} attempts")] + private static partial void LogJobDeadLettered( ILogger logger, Exception exception, - Guid itemId, + Guid jobId, int attempt); [LoggerMessage(Level = LogLevel.Warning, - Message = "Requeued failed completion for {ItemId}")] - private static partial void LogProcessingRequestRequeued(ILogger logger, Guid itemId); + Message = "Durable completion job {JobId} lost its lease; another worker will resume it")] + private static partial void LogJobLeaseLost(ILogger logger, Guid jobId); [LoggerMessage(Level = LogLevel.Warning, - Message = "Failed to record completion incident for {ItemId}")] - private static partial void LogCompletionIncidentFailed( + Message = "Durable completion job {JobId} lease renewal failed")] + private static partial void LogJobLeaseRenewalFailed( ILogger logger, Exception exception, - Guid itemId); - - [LoggerMessage(Level = LogLevel.Information, Message = "Download marked finished for {ItemId}: {Title}")] - private static partial void LogDownloadMarkedFinished(ILogger logger, Guid itemId, string title); - - [LoggerMessage(Level = LogLevel.Debug, Message = "Firing OnFileDownloadCompleted plugin event for {ItemId}")] - private static partial void LogFiringPluginEvent(ILogger logger, Guid itemId); - - [LoggerMessage(Level = LogLevel.Debug, Message = "Plugin event completed for {ItemId}")] - private static partial void LogPluginEventCompleted(ILogger logger, Guid itemId); - - [LoggerMessage(Level = LogLevel.Warning, Message = "OnFileDownloadCompleted event failed for {ItemId}")] - private static partial void LogDownloadCompletedEventFailed(ILogger logger, Exception ex, Guid itemId); - - [LoggerMessage(Level = LogLevel.Warning, Message = "File mapping failed for {ItemId}")] - private static partial void LogFileMappingFailed(ILogger logger, Exception ex, Guid itemId); + Guid jobId); } diff --git a/SecondDimensionWatcherReDive/Services/FetchRemoteTorrentBackgroundService.cs b/SecondDimensionWatcherReDive/Services/FetchRemoteTorrentBackgroundService.cs index d587b9a..cef65f0 100644 --- a/SecondDimensionWatcherReDive/Services/FetchRemoteTorrentBackgroundService.cs +++ b/SecondDimensionWatcherReDive/Services/FetchRemoteTorrentBackgroundService.cs @@ -63,13 +63,31 @@ protected override async Task ExecuteAsync(CancellationToken cancellationToken) var reader = remoteTorrentTrackRequest.Reader; var tracked = new ConcurrentDictionary(); var observations = new ConcurrentDictionary(); - - // Add unfinished to track - await foreach (var request in FetchUnfinishedTaskFromDb(cancellationToken)) - tracked[request.Hash] = request; + var nextDatabaseRefreshAt = DateTimeOffset.MinValue; while (!cancellationToken.IsCancellationRequested) { + if (DateTimeOffset.UtcNow >= nextDatabaseRefreshAt) + { + try + { + // Periodic refresh recovers requests whose initial channel + // binding happened during a temporary database outage. + await foreach (var request in FetchUnfinishedTaskFromDb(cancellationToken)) + tracked[request.Hash] = request; + nextDatabaseRefreshAt = DateTimeOffset.UtcNow.AddSeconds(30); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + nextDatabaseRefreshAt = DateTimeOffset.UtcNow.AddSeconds(5); + LogRefreshTrackedDownloadsFailed(logger, exception); + } + } + // Drain channel messages in the supervised service loop so channel // failures/cancellation cannot disappear in an unobserved Task. while (reader.TryRead(out var request)) @@ -82,9 +100,23 @@ protected override async Task ExecuteAsync(CancellationToken cancellationToken) } else { - var boundRequest = await BindCurrentAttemptAsync(request, cancellationToken); - if (boundRequest is { } currentRequest) - tracked[request.Hash] = currentRequest; + try + { + var boundRequest = await BindCurrentAttemptAsync(request, cancellationToken); + if (boundRequest is { } currentRequest) + tracked[request.Hash] = currentRequest; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + // The periodic database refresh above will recover this + // unfinished attempt without relying on an unbounded queue. + nextDatabaseRefreshAt = DateTimeOffset.MinValue; + LogBindTrackedDownloadFailed(logger, exception, request.ItemId); + } } } await Task.Delay(500, cancellationToken); @@ -145,14 +177,28 @@ await ObserveHealthAsync( if (state != FileDownloadState.Finished) continue; - //Write complete request and stop tracking. - await downloadCompleteRequest.Writer.WriteAsync( - new DownloadCompleteRequest( - request.ItemId, - torrentInfo.SavePath, - FileStores.LocalDiskStore, - request.DownloadAttemptId), - cancellationToken); + var completion = new DownloadCompleteRequest( + request.ItemId, + torrentInfo.SavePath, + FileStores.LocalDiskStore, + request.DownloadAttemptId); + try + { + // The completion transition and its durable workflow are committed + // together. The channel is only a best-effort wake-up signal; the + // durable worker also polls after restart. + await PersistCompletionAsync(completion, cancellationToken); + downloadCompleteRequest.Writer.TryWrite(completion); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + LogPersistCompletionFailed(logger, exception, request.ItemId); + continue; + } tracked.TryRemove(torrentInfo.Hash, out _); observations.TryRemove(torrentInfo.Hash, out _); } @@ -165,6 +211,21 @@ await ReportMissingAfterThresholdAsync( } } + private async Task PersistCompletionAsync( + DownloadCompleteRequest request, + CancellationToken cancellationToken) + { + await using var scope = scopeFactory.CreateAsyncScope(); + var repository = scope.ServiceProvider.GetRequiredService(); + await repository.TryCompleteDownloadAsync( + request.ItemId, + request.DownloadAttemptId, + request.FileStore, + request.StorePath, + DateTimeOffset.UtcNow, + cancellationToken); + } + private async Task ObserveHealthAsync( RemoteTorrentTrackRequest request, RemoteTorrentInfo torrentInfo, @@ -323,4 +384,24 @@ private bool ShouldResolve(DownloadObservation observation, DateTimeOffset now) [LoggerMessage(Level = LogLevel.Warning, Message = "Failed to fetch torrent status from remote client")] private static partial void LogFetchTorrentStatusFailed(ILogger logger, Exception ex); + + [LoggerMessage(Level = LogLevel.Error, + Message = "Could not persist durable download completion for {ItemId}; tracking will retry")] + private static partial void LogPersistCompletionFailed( + ILogger logger, + Exception exception, + Guid itemId); + + [LoggerMessage(Level = LogLevel.Warning, + Message = "Could not refresh unfinished downloads; retrying")] + private static partial void LogRefreshTrackedDownloadsFailed( + ILogger logger, + Exception exception); + + [LoggerMessage(Level = LogLevel.Warning, + Message = "Could not bind download attempt {ItemId}; the database refresh will retry")] + private static partial void LogBindTrackedDownloadFailed( + ILogger logger, + Exception exception, + Guid itemId); } diff --git a/SecondDimensionWatcherReDive/Services/MediaLibraryScanQueue.cs b/SecondDimensionWatcherReDive/Services/MediaLibraryScanQueue.cs index 4901b70..fc5e1f2 100644 --- a/SecondDimensionWatcherReDive/Services/MediaLibraryScanQueue.cs +++ b/SecondDimensionWatcherReDive/Services/MediaLibraryScanQueue.cs @@ -14,8 +14,14 @@ public interface IMediaLibraryScanQueue public sealed class MediaLibraryScanQueue : IMediaLibraryScanQueue { - private readonly Channel _channel = Channel.CreateUnbounded( - new UnboundedChannelOptions { SingleReader = true, SingleWriter = false }); + internal const int Capacity = 256; + private readonly Channel _channel = Channel.CreateBounded( + new BoundedChannelOptions(Capacity) + { + SingleReader = true, + SingleWriter = false, + FullMode = BoundedChannelFullMode.Wait + }); private readonly ConcurrentDictionary _pending = new(); public bool Enqueue(Guid sourceId) diff --git a/SecondDimensionWatcherReDive/Services/NullDownloadCompletionNotifier.cs b/SecondDimensionWatcherReDive/Services/NullDownloadCompletionNotifier.cs new file mode 100644 index 0000000..73c3c65 --- /dev/null +++ b/SecondDimensionWatcherReDive/Services/NullDownloadCompletionNotifier.cs @@ -0,0 +1,12 @@ +using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Framework.Notifications; + +namespace SecondDimensionWatcherReDive.Services; + +internal sealed class NullDownloadCompletionNotifier : IDownloadCompletionNotifier +{ + public Task NotifyAsync( + Guid eventId, + DownloadCompletionJobPayload payload, + CancellationToken cancellationToken) => Task.CompletedTask; +} diff --git a/SecondDimensionWatcherReDive/Services/PostgresScheduledTaskLeaseManager.cs b/SecondDimensionWatcherReDive/Services/PostgresScheduledTaskLeaseManager.cs new file mode 100644 index 0000000..c16108d --- /dev/null +++ b/SecondDimensionWatcherReDive/Services/PostgresScheduledTaskLeaseManager.cs @@ -0,0 +1,146 @@ +using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Framework.Tasks; + +namespace SecondDimensionWatcherReDive.Services; + +public sealed partial class PostgresScheduledTaskLeaseManager( + IServiceScopeFactory scopeFactory, + ILogger logger) : IScheduledTaskLeaseManager +{ + internal static readonly TimeSpan LeaseDuration = TimeSpan.FromSeconds(30); + internal static readonly TimeSpan RenewInterval = TimeSpan.FromSeconds(10); + + private readonly string _ownerId = $"{Environment.MachineName}:{Environment.ProcessId}:{Guid.NewGuid():N}"; + + public async Task TryAcquireAsync( + string taskId, + TimeSpan interval, + bool force, + CancellationToken cancellationToken) + { + var now = DateTimeOffset.UtcNow; + await using var scope = scopeFactory.CreateAsyncScope(); + var repository = scope.ServiceProvider.GetRequiredService(); + var acquired = await repository.TryAcquireAsync( + taskId, + _ownerId, + now, + now + LeaseDuration, + force, + cancellationToken); + return acquired + ? new ExecutionLease(taskId, _ownerId, interval, scopeFactory, logger) + : null; + } + + private sealed class ExecutionLease : IScheduledTaskExecutionLease + { + private readonly string _taskId; + private readonly string _ownerId; + private readonly TimeSpan _interval; + private readonly IServiceScopeFactory _scopeFactory; + private readonly ILogger _logger; + private readonly CancellationTokenSource _renewalCancellation = new(); + private readonly CancellationTokenSource _leaseLost = new(); + private readonly Task _renewalTask; + private bool _completed; + + public ExecutionLease( + string taskId, + string ownerId, + TimeSpan interval, + IServiceScopeFactory scopeFactory, + ILogger logger) + { + _taskId = taskId; + _ownerId = ownerId; + _interval = interval; + _scopeFactory = scopeFactory; + _logger = logger; + _renewalTask = RenewLoopAsync(); + } + + public CancellationToken LeaseLostToken => _leaseLost.Token; + + public async Task CompleteAsync( + bool succeeded, + string? error, + CancellationToken cancellationToken) + { + if (_completed) return; + _completed = true; + await StopRenewalAsync(); + + await using var scope = _scopeFactory.CreateAsyncScope(); + var repository = scope.ServiceProvider.GetRequiredService(); + var completedAt = DateTimeOffset.UtcNow; + await repository.CompleteAsync( + _taskId, + _ownerId, + completedAt, + completedAt + _interval, + succeeded, + error, + cancellationToken); + } + + public async ValueTask DisposeAsync() + { + await StopRenewalAsync(); + _renewalCancellation.Dispose(); + _leaseLost.Dispose(); + } + + private async Task RenewLoopAsync() + { + try + { + using var timer = new PeriodicTimer(RenewInterval); + while (await timer.WaitForNextTickAsync(_renewalCancellation.Token)) + { + var now = DateTimeOffset.UtcNow; + await using var scope = _scopeFactory.CreateAsyncScope(); + var repository = scope.ServiceProvider + .GetRequiredService(); + if (await repository.RenewAsync( + _taskId, + _ownerId, + now, + now + LeaseDuration, + _renewalCancellation.Token)) + continue; + + LogLeaseLost(_logger, _taskId); + _leaseLost.Cancel(); + return; + } + } + catch (OperationCanceledException) when (_renewalCancellation.IsCancellationRequested) + { + } + catch (Exception exception) + { + LogLeaseRenewalFailed(_logger, exception, _taskId); + _leaseLost.Cancel(); + } + } + + private async Task StopRenewalAsync() + { + if (!_renewalCancellation.IsCancellationRequested) + await _renewalCancellation.CancelAsync(); + await _renewalTask; + } + } + + [LoggerMessage(Level = LogLevel.Warning, + Message = "Scheduled task {TaskId} lost its execution lease")] + private static partial void LogLeaseLost(ILogger logger, string taskId); + + [LoggerMessage(Level = LogLevel.Warning, + Message = "Scheduled task {TaskId} lease renewal failed")] + private static partial void LogLeaseRenewalFailed( + ILogger logger, + Exception exception, + string taskId); +} diff --git a/SecondDimensionWatcherReDive/Services/ScheduledTaskBackgroundService.cs b/SecondDimensionWatcherReDive/Services/ScheduledTaskBackgroundService.cs index 595c420..4bcfb25 100644 --- a/SecondDimensionWatcherReDive/Services/ScheduledTaskBackgroundService.cs +++ b/SecondDimensionWatcherReDive/Services/ScheduledTaskBackgroundService.cs @@ -1,18 +1,24 @@ +using System.Diagnostics; using SecondDimensionWatcherReDive.Framework.Tasks; +using SecondDimensionWatcherReDive.Observability; namespace SecondDimensionWatcherReDive.Services; public partial class ScheduledTaskBackgroundService( TTask task, + IScheduledTaskLeaseManager leaseManager, + RuntimeTelemetry telemetry, ILogger> logger) : BackgroundService where TTask : ScheduledTaskBase { + private static readonly TimeSpan ContendedLeasePollInterval = TimeSpan.FromSeconds(10); + protected override async Task ExecuteAsync(CancellationToken stoppingToken) { LogStartingScheduledTask(logger, task.Id); await Task.WhenAll( - task.ProcessQueueAsync(stoppingToken), + task.ProcessQueueAsync(leaseManager, stoppingToken), RunTimerLoopAsync(stoppingToken)); } @@ -20,19 +26,49 @@ private async Task RunTimerLoopAsync(CancellationToken stoppingToken) { while (!stoppingToken.IsCancellationRequested) { + var delay = task.Interval; if (task.IsEnabled) { + var startedAt = Stopwatch.GetTimestamp(); try { - await task.RunNowAsync(stoppingToken); + var executed = await task.RunScheduledAsync(stoppingToken); + if (!executed) + delay = ContendedLeasePollInterval; + telemetry.RecordScheduledTask( + task.Id, + executed ? "completed" : "contended", + Stopwatch.GetElapsedTime(startedAt)); + } + catch (ScheduledTaskLeaseUnavailableException ex) + { + delay = ContendedLeasePollInterval; + LogScheduledTaskLeaseUnavailable(logger, ex, task.Id); + telemetry.RecordScheduledTask( + task.Id, + "lease_unavailable", + Stopwatch.GetElapsedTime(startedAt)); + } + catch (OperationCanceledException ex) when (!stoppingToken.IsCancellationRequested) + { + delay = ContendedLeasePollInterval; + LogScheduledTaskLeaseLost(logger, ex, task.Id); + telemetry.RecordScheduledTask( + task.Id, + "lease_lost", + Stopwatch.GetElapsedTime(startedAt)); } catch (Exception ex) when (ex is not OperationCanceledException) { LogScheduledTaskFailed(logger, ex, task.Id); + telemetry.RecordScheduledTask( + task.Id, + "failed", + Stopwatch.GetElapsedTime(startedAt)); } } - await Task.Delay(task.Interval, stoppingToken); + await Task.Delay(delay, stoppingToken); } } @@ -41,4 +77,18 @@ private async Task RunTimerLoopAsync(CancellationToken stoppingToken) [LoggerMessage(Level = LogLevel.Error, Message = "Scheduled task {TaskId} failed during timer execution")] private static partial void LogScheduledTaskFailed(ILogger logger, Exception ex, string taskId); + + [LoggerMessage(Level = LogLevel.Warning, + Message = "Scheduled task {TaskId} lease is unavailable; retrying soon")] + private static partial void LogScheduledTaskLeaseUnavailable( + ILogger logger, + Exception exception, + string taskId); + + [LoggerMessage(Level = LogLevel.Warning, + Message = "Scheduled task {TaskId} execution was cancelled or lost its lease; retrying soon")] + private static partial void LogScheduledTaskLeaseLost( + ILogger logger, + Exception exception, + string taskId); } diff --git a/SecondDimensionWatcherReDive/Utils/FileDownload/RemoteTorrentDownloadClient.cs b/SecondDimensionWatcherReDive/Utils/FileDownload/RemoteTorrentDownloadClient.cs index 9bb0f72..e7746a0 100644 --- a/SecondDimensionWatcherReDive/Utils/FileDownload/RemoteTorrentDownloadClient.cs +++ b/SecondDimensionWatcherReDive/Utils/FileDownload/RemoteTorrentDownloadClient.cs @@ -38,7 +38,8 @@ public override async Task SubmitDownloadTaskAsync( if (response.IsSuccessStatusCode) await remoteTorrentTrackRequest.Writer.WriteAsync( - new(itemId, additionalDownloadInfo)); + new(itemId, additionalDownloadInfo), + cancellationToken); return response.IsSuccessStatusCode; } @@ -51,7 +52,8 @@ public override async Task SubmitQueryDownloadProgressAsync( CancellationToken cancellationToken) { await remoteTorrentTrackRequest.Writer.WriteAsync( - new(itemId, additionalDownloadInfo)); + new(itemId, additionalDownloadInfo), + cancellationToken); } public override async Task PauseDownloadTaskAsync( diff --git a/SecondDimensionWatcherReDive/appsettings.example.json b/SecondDimensionWatcherReDive/appsettings.example.json index da9d5ab..a266ec7 100644 --- a/SecondDimensionWatcherReDive/appsettings.example.json +++ b/SecondDimensionWatcherReDive/appsettings.example.json @@ -92,5 +92,20 @@ "Valkey": { "ConnectionString": "", "InstanceName": "sdw-redive:" + }, + + // /health/live never probes dependencies. /health/ready always probes PostgreSQL; + // these switches control the other deployment-specific readiness dependencies. + "Health": { + "ValkeyRequired": true, + "QbittorrentRequired": true, + "StorageRequired": true, + "AIRequired": false + }, + + // Prometheus metrics are always exposed at /metrics. Set an OTLP endpoint to + // additionally export traces and metrics to an OpenTelemetry collector. + "OpenTelemetry": { + "OtlpEndpoint": "" } } diff --git a/docs/runtime-reliability.md b/docs/runtime-reliability.md new file mode 100644 index 0000000..0ac218f --- /dev/null +++ b/docs/runtime-reliability.md @@ -0,0 +1,66 @@ +# 运行可靠性与可观测性 + +## 持久下载完成流程 + +下载器确认完成后，会在同一个 PostgreSQL 事务中更新下载状态并创建唯一的 +`DownloadCompletion` 持久任务。进程内的完成 Channel 仅用于立即唤醒消费者；即使 +信号丢失或进程在任意阶段退出，消费者也会轮询数据库并在租约到期后继续。 +消费者一次只领取一个任务，2 分钟任务租约每 30 秒续期；慢速映射不会被其他实例并发 +领取，而失联消费者的任务会在租约到期后恢复。 + +每个任务按以下阶段推进，阶段成功后才持久化下一个阶段： + +1. `MapFiles`：以替换方式重建该下载的虚拟文件映射，重复执行不会新增重复映射。 +2. `Notify`：调用可选的通知扩展点。 +3. `InvokePlugins`：触发下载完成插件事件。 +4. `Done`：标记任务完成。 + +通知实现和插件处理器会收到稳定的 `EventId`（持久任务 ID），必须在执行外部副作用前 +把它作为幂等键保存。这样，即使进程恰好在外部调用成功、阶段提交前退出，重新投递也 +不会重复发送通知或执行插件副作用。 + +失败任务采用指数退避（5 秒起步，最长 15 分钟），第 8 次失败后进入死信。登录后打开 +「后台任务」可以查看失败阶段、尝试次数和错误，并选择「重试」或「标记已处理」。对应 +API 为： + +- `GET /api/jobs?status=deadLetter` +- `POST /api/jobs/retry`，请求体 `{"ids":["..."]}` +- `POST /api/jobs/resolve`，请求体 `{"ids":["..."]}` + +API 不返回任务 payload、去重键或租约所有者，避免暴露存储路径和内部拓扑。 + +## 多实例定时任务 + +每个定时任务执行前都会获取 PostgreSQL 租约，并每 10 秒续租一次；默认租约为 30 秒。 +同一时刻只有一个实例执行同名任务。未持有租约的实例每 10 秒检查一次，实例退出或失联后 +可在租约到期时接管；正常完成会把租约冷却期保留到下次应运行时间，避免其他实例立即 +重复运行。手动触发可以越过“已完成”的冷却期，但不能越过仍在执行的租约。同一实例内 +的手动与周期触发会合并为一个容量为 1 的信号，避免积压重复执行。 + +所有生产 Channel 都有明确容量与溢出策略：下载跟踪使用背压；高频进度和已持久化的 +完成唤醒信号丢弃旧值；媒体库扫描在满载时拒绝新请求；聊天 SSE 使用背压。 +Codex app-server 的请求内更新缓冲最多保留 1024 项，超限会终止该轮对话并返回明确错误， +不会继续无界占用内存。 + +## 健康探针 + +- `GET /health/live`：只说明进程可响应，不访问任何外部依赖。 +- `GET /health/ready`：始终检查 PostgreSQL，并按配置检查 Valkey、qBittorrent、下载 + 存储和可选 AI Provider。任何必需依赖失败时返回 `503`。 + +`Health:QbittorrentRequired`、`Health:StorageRequired` 和 +`Health:ValkeyRequired` 默认启用（Valkey 仅在配置连接时注册）； +`Health:AIRequired` 默认关闭。探针响应只包含检查名、状态、耗时和异常类型，不包含地址、 +凭据或异常正文。 + +## 指标与链路 + +`GET /metrics` 提供 Prometheus 格式的 ASP.NET Core、HttpClient、运行时和持久任务指标。 +设置 `OpenTelemetry:OtlpEndpoint` 后，应用也会通过 OTLP 导出指标及 ASP.NET Core、 +HttpClient、EF Core 和持久任务链路。 + +HTTP 指标使用显式标签白名单：服务端只保留方法、状态码、路由模板和协议版本；客户端 +只保留方法、状态码、服务地址/端口和协议版本。自定义任务指标只使用固定枚举值 +`job.type`、`job.stage`、`outcome` 和 `status`。原始路径、查询串、动画标题、工具参数、 +SQL 文本和查询参数不会作为指标标签或导出的链路标签。定时任务指标的 `task.id` 也只 +允许四个内置任务值，未知扩展统一归为 `other`。 diff --git a/packaging/appsettings.yml b/packaging/appsettings.yml index 4cf660f..3a1bf21 100644 --- a/packaging/appsettings.yml +++ b/packaging/appsettings.yml @@ -92,3 +92,14 @@ Nfs: # Valkey: # ConnectionString: "localhost:6379" # InstanceName: "sdw-redive:" + +# 健康探针：/health/live 不访问外部依赖；/health/ready 始终检查 PostgreSQL +Health: + ValkeyRequired: true + QbittorrentRequired: true + StorageRequired: true + AIRequired: false + +# /metrics 始终提供 Prometheus 指标；配置地址后同时向 OTLP Collector 导出 +OpenTelemetry: + OtlpEndpoint: "" From f32939d62f5c8267273f05000448aeba6ba5ebf5 Mon Sep 17 00:00:00 2001 From: mahoshojoHCG Date: Sun, 30 Aug 2026 11:15:29 +0800 Subject: [PATCH 2/2] fix: harden streaming and distributed task state --- .../ChatController.cs | 22 +-- .../Tools/ManageTasksTool.cs | 25 +++- .../IScheduledTaskLeaseRepository.cs | 4 + .../DataRepository/ScheduledTaskLeaseState.cs | 8 ++ .../Tasks/IScheduledTaskLeaseManager.cs | 8 ++ .../Tasks/ScheduledTaskBase.cs | 94 +++++++++---- .../FileMappingRepositoryPostgreSqlTests.cs | 37 +++++ .../ChatControllerStreamingTests.cs | 128 ++++++++++++++++++ .../ManageTasksToolTests.cs | 42 ++++++ .../PostgresScheduledTaskLeaseManagerTests.cs | 70 ++++++++++ .../ScheduledTaskBaseTests.cs | 64 +++++++++ .../TasksControllerTests.cs | 39 ++++++ .../Controllers/TasksController.cs | 27 ++-- ...eMappingRepositoryPostgreSqlTestFixture.cs | 10 ++ .../ScheduledTaskLeaseRepository.cs | 19 +++ .../PostgresScheduledTaskLeaseManager.cs | 35 +++++ 16 files changed, 581 insertions(+), 51 deletions(-) create mode 100644 SecondDimensionWatcherReDive.Framework/DataRepository/ScheduledTaskLeaseState.cs create mode 100644 SecondDimensionWatcherReDive.Test/ChatControllerStreamingTests.cs create mode 100644 SecondDimensionWatcherReDive.Test/ManageTasksToolTests.cs create mode 100644 SecondDimensionWatcherReDive.Test/PostgresScheduledTaskLeaseManagerTests.cs create mode 100644 SecondDimensionWatcherReDive.Test/TasksControllerTests.cs diff --git a/Plugins/SecondDimensionWatcherReDive.Chat/ChatController.cs b/Plugins/SecondDimensionWatcherReDive.Chat/ChatController.cs index 7420380..48ef392 100644 --- a/Plugins/SecondDimensionWatcherReDive.Chat/ChatController.cs +++ b/Plugins/SecondDimensionWatcherReDive.Chat/ChatController.cs @@ -179,7 +179,7 @@ public async Task SendMessage( cancellationToken)); } - private async IAsyncEnumerable> StreamChatEvents( + internal async IAsyncEnumerable> StreamChatEvents( IAIEngine aiEngine, List messages, ChatOptions chatOptions, @@ -197,6 +197,8 @@ private async IAsyncEnumerable> StreamChatEvents( SingleWriter = true, FullMode = BoundedChannelFullMode.Wait }); + using var producerCancellation = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken); // Producer: runs AI chat streaming in background, writes SSE items to channel. // Keep the task and await it during iterator disposal so a disconnected request cannot @@ -204,7 +206,7 @@ private async IAsyncEnumerable> StreamChatEvents( var producer = ProduceChatEventsAsync( aiEngine, messages, chatOptions, conversationId, messageOrder, firstUserMessage, autoTitleEligible, model, - channel.Writer, cancellationToken); + channel.Writer, producerCancellation.Token); try { @@ -214,13 +216,15 @@ private async IAsyncEnumerable> StreamChatEvents( } finally { - // Do not use the canceled request token here. The producer receives it directly, - // performs bounded engine cleanup, and persists accumulated messages before returning. + // The reader can be disposed independently of RequestAborted (for example when + // response-body I/O fails). Explicitly stop the producer, then let it persist + // accumulated messages before this request scope is released. + await producerCancellation.CancelAsync(); await producer; } } - private async Task ProduceChatEventsAsync( + internal async Task ProduceChatEventsAsync( IAIEngine aiEngine, List messages, ChatOptions chatOptions, @@ -334,12 +338,14 @@ await writer.WriteAsync( catch (Exception ex) { LogStreamingError(ex, conversationId); - await writer.WriteAsync( + // A disconnected/failed reader may leave this bounded channel full. + // Terminal error delivery is best-effort so persistence and request + // scope disposal can never wait forever for a reader that is gone. + writer.TryWrite( new SseItem( JsonSerializer.Serialize(new SseError(ex.Message), ChatJsonSerializerContext.Default.SseError), - "error"), - CancellationToken.None); + "error")); } // Save all accumulated segments to DB diff --git a/Plugins/SecondDimensionWatcherReDive.Chat/Tools/ManageTasksTool.cs b/Plugins/SecondDimensionWatcherReDive.Chat/Tools/ManageTasksTool.cs index ae9875f..5fc6330 100644 --- a/Plugins/SecondDimensionWatcherReDive.Chat/Tools/ManageTasksTool.cs +++ b/Plugins/SecondDimensionWatcherReDive.Chat/Tools/ManageTasksTool.cs @@ -9,9 +9,10 @@ namespace SecondDimensionWatcherReDive.Chat.Tools; "manage_tasks", "Manage background scheduled tasks. List all task statuses or manually trigger a specific task to run.")] internal sealed partial class ManageTasksTool( - IEnumerable scheduledTasks) : ITool + IEnumerable scheduledTasks, + IScheduledTaskLeaseManager leaseManager) : ITool { - private Task ExecuteCoreAsync( + private async Task ExecuteCoreAsync( ManageTasksParams param, CancellationToken cancellationToken) { var taskList = scheduledTasks.ToList(); @@ -20,10 +21,24 @@ private Task ExecuteCoreAsync( switch (param.Action) { case ManageTasksAction.List: + { + var statuses = await leaseManager.GetStatusesAsync( + taskList.Select(task => task.Id).ToArray(), + cancellationToken); result = new ToolSuccessResult(new TaskListResult( - taskList.Select(t => new TaskSummary( - t.Id, t.Interval.ToString(), t.IsEnabled, t.LastRunAt, t.IsRunning)))); + taskList.Select(task => + { + var status = statuses.GetValueOrDefault(task.Id) + ?? new ScheduledTaskStatus(null, false); + return new TaskSummary( + task.Id, + task.Interval.ToString(), + task.IsEnabled, + status.LastRunAt, + status.IsRunning); + }))); break; + } case ManageTasksAction.Run: { @@ -52,7 +67,7 @@ private Task ExecuteCoreAsync( break; } - return Task.FromResult(result); + return result; } } diff --git a/SecondDimensionWatcherReDive.Framework/DataRepository/IScheduledTaskLeaseRepository.cs b/SecondDimensionWatcherReDive.Framework/DataRepository/IScheduledTaskLeaseRepository.cs index 34b6424..684afa3 100644 --- a/SecondDimensionWatcherReDive.Framework/DataRepository/IScheduledTaskLeaseRepository.cs +++ b/SecondDimensionWatcherReDive.Framework/DataRepository/IScheduledTaskLeaseRepository.cs @@ -25,4 +25,8 @@ Task CompleteAsync( bool succeeded, string? error, CancellationToken cancellationToken); + + Task> GetStatesAsync( + IReadOnlyCollection taskIds, + CancellationToken cancellationToken); } diff --git a/SecondDimensionWatcherReDive.Framework/DataRepository/ScheduledTaskLeaseState.cs b/SecondDimensionWatcherReDive.Framework/DataRepository/ScheduledTaskLeaseState.cs new file mode 100644 index 0000000..c2d9f65 --- /dev/null +++ b/SecondDimensionWatcherReDive.Framework/DataRepository/ScheduledTaskLeaseState.cs @@ -0,0 +1,8 @@ +namespace SecondDimensionWatcherReDive.Framework.DataRepository; + +public sealed record ScheduledTaskLeaseState( + string TaskId, + string? LeaseOwner, + DateTimeOffset? LeaseExpiresAt, + DateTimeOffset? LastStartedAt, + DateTimeOffset? LastCompletedAt); diff --git a/SecondDimensionWatcherReDive.Framework/Tasks/IScheduledTaskLeaseManager.cs b/SecondDimensionWatcherReDive.Framework/Tasks/IScheduledTaskLeaseManager.cs index 2a0f1bd..eede259 100644 --- a/SecondDimensionWatcherReDive.Framework/Tasks/IScheduledTaskLeaseManager.cs +++ b/SecondDimensionWatcherReDive.Framework/Tasks/IScheduledTaskLeaseManager.cs @@ -3,6 +3,10 @@ namespace SecondDimensionWatcherReDive.Framework.Tasks; public sealed class ScheduledTaskLeaseUnavailableException(Exception innerException) : Exception("The scheduled-task lease store is unavailable.", innerException); +public sealed record ScheduledTaskStatus( + DateTimeOffset? LastRunAt, + bool IsRunning); + public interface IScheduledTaskExecutionLease : IAsyncDisposable { CancellationToken LeaseLostToken { get; } @@ -20,4 +24,8 @@ public interface IScheduledTaskLeaseManager TimeSpan interval, bool force, CancellationToken cancellationToken); + + Task> GetStatusesAsync( + IReadOnlyCollection taskIds, + CancellationToken cancellationToken); } diff --git a/SecondDimensionWatcherReDive.Framework/Tasks/ScheduledTaskBase.cs b/SecondDimensionWatcherReDive.Framework/Tasks/ScheduledTaskBase.cs index fa025c7..e8945e9 100644 --- a/SecondDimensionWatcherReDive.Framework/Tasks/ScheduledTaskBase.cs +++ b/SecondDimensionWatcherReDive.Framework/Tasks/ScheduledTaskBase.cs @@ -56,45 +56,49 @@ public async Task ProcessQueueAsync( await foreach (var _ in _runQueue.Reader.ReadAllAsync(cancellationToken)) { TaskCompletionSource? completion; - bool force; lock (_sync) { completion = _pendingRun; - force = _pendingForce; } if (completion is null) continue; - IScheduledTaskExecutionLease? lease; - try + IScheduledTaskExecutionLease? lease = null; + while (lease is null) { - lease = await leaseManager.TryAcquireAsync( - Id, - Interval, - force, - cancellationToken); - } - catch (OperationCanceledException exception) when (cancellationToken.IsCancellationRequested) - { - completion.TrySetCanceled(exception.CancellationToken); - FinishRun(completion); - throw; - } - catch (Exception exception) - { - completion.TrySetException( - new ScheduledTaskLeaseUnavailableException(exception)); - FinishRun(completion); - continue; - } + var force = TakePendingForce(completion); + try + { + lease = await leaseManager.TryAcquireAsync( + Id, + Interval, + force, + cancellationToken); + } + catch (OperationCanceledException exception) when (cancellationToken.IsCancellationRequested) + { + completion.TrySetCanceled(exception.CancellationToken); + FinishRun(completion); + throw; + } + catch (Exception exception) + { + completion.TrySetException( + new ScheduledTaskLeaseUnavailableException(exception)); + FinishRun(completion); + break; + } - if (lease is null) - { - // Another instance owns the same periodic task. Its local timer - // will drive the execution; this duplicate signal is complete. - completion.TrySetResult(false); - FinishRun(completion); - continue; + if (lease is not null) + break; + + // A manual request can upgrade a periodic acquisition while the + // database call is in flight. Retry that upgrade before completing + // the shared signal so a completed cooldown cannot swallow it. + if (CompleteLeaseDenialOrRetryForce(completion, force)) + continue; + break; } + if (lease is null) continue; await using (lease) { @@ -167,6 +171,36 @@ private Task QueueRun(bool force) } } + private bool TakePendingForce(TaskCompletionSource completion) + { + lock (_sync) + { + if (!ReferenceEquals(_pendingRun, completion)) + return false; + var force = _pendingForce; + _pendingForce = false; + return force; + } + } + + private bool CompleteLeaseDenialOrRetryForce( + TaskCompletionSource completion, + bool attemptedForce) + { + lock (_sync) + { + if (!ReferenceEquals(_pendingRun, completion)) + return false; + if (!attemptedForce && _pendingForce) + return true; + + _pendingRun = null; + _pendingForce = false; + completion.TrySetResult(false); + return false; + } + } + private void FinishRun(TaskCompletionSource completion) { lock (_sync) diff --git a/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/FileMappingRepositoryPostgreSqlTests.cs b/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/FileMappingRepositoryPostgreSqlTests.cs index 3e53a88..2611336 100644 --- a/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/FileMappingRepositoryPostgreSqlTests.cs +++ b/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/FileMappingRepositoryPostgreSqlTests.cs @@ -240,6 +240,43 @@ await Fixture.CompleteTaskLeaseAsync( CancellationToken.None)); } + [TestMethod] + public async Task ScheduledTaskLease_StatusProjectionReadsSharedState() + { + var now = DateTimeOffset.FromUnixTimeMilliseconds( + DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); + var leaseUntil = now.AddSeconds(30); + Assert.IsTrue(await Fixture.TryAcquireTaskLeaseAsync( + "StatusTask", + "instance-a", + now, + leaseUntil, + false, + CancellationToken.None)); + + var running = await Fixture.GetTaskLeaseStatesAsync( + ["StatusTask", "missing"], + CancellationToken.None); + + Assert.HasCount(1, running); + Assert.AreEqual("instance-a", running[0].LeaseOwner); + Assert.AreEqual(now, running[0].LastStartedAt); + Assert.AreEqual(leaseUntil, running[0].LeaseExpiresAt); + Assert.IsNull(running[0].LastCompletedAt); + + var completedAt = now.AddSeconds(5); + await Fixture.CompleteTaskLeaseAsync( + "StatusTask", + "instance-a", + completedAt, + now.AddMinutes(10), + CancellationToken.None); + var completed = await Fixture.GetTaskLeaseStatesAsync( + ["StatusTask"], + CancellationToken.None); + Assert.AreEqual(completedAt, completed.Single().LastCompletedAt); + } + [TestMethod] public async Task DeadLetterJobs_CanBeRetriedOrMarkedHandled() { diff --git a/SecondDimensionWatcherReDive.Test/ChatControllerStreamingTests.cs b/SecondDimensionWatcherReDive.Test/ChatControllerStreamingTests.cs new file mode 100644 index 0000000..a5def04 --- /dev/null +++ b/SecondDimensionWatcherReDive.Test/ChatControllerStreamingTests.cs @@ -0,0 +1,128 @@ +using System.Net.ServerSentEvents; +using System.Runtime.CompilerServices; +using System.Threading.Channels; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using SecondDimensionWatcherReDive.AI.Abstractions; +using SecondDimensionWatcherReDive.AI.Models; +using SecondDimensionWatcherReDive.Chat; +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Test; + +[TestClass] +public sealed class ChatControllerStreamingTests +{ + [TestMethod] + public async Task ProduceChatEventsAsync_ErrorWithFullChannel_DoesNotBlock() + { + var channel = Channel.CreateBounded>(1); + Assert.IsTrue(channel.Writer.TryWrite(new SseItem("buffered", "text_delta"))); + var controller = CreateController(new Mock(MockBehavior.Strict).Object); + + await controller.ProduceChatEventsAsync( + new ThrowingEngine(), + [], + new ChatOptions(), + Guid.NewGuid(), + 0, + "question", + false, + null, + channel.Writer, + CancellationToken.None) + .WaitAsync(TimeSpan.FromSeconds(2)); + + Assert.IsTrue(channel.Reader.TryRead(out var buffered)); + Assert.AreEqual("buffered", buffered.Data); + await channel.Reader.Completion.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.IsFalse(channel.Reader.TryRead(out _)); + } + + [TestMethod] + public async Task StreamChatEvents_EarlyReaderExit_CancelsAndJoinsProducer() + { + var conversationId = Guid.NewGuid(); + var repository = new Mock(MockBehavior.Strict); + repository.Setup(candidate => candidate.AddMessagesAsync( + conversationId, + It.Is>(messages => + messages.Any(message => message.Role == "assistant" + && message.Content == "partial")), + CancellationToken.None)) + .Returns(Task.CompletedTask); + var engine = new CancellationAwareEngine(); + var controller = CreateController(repository.Object); + var enumerator = controller.StreamChatEvents( + engine, + [], + new ChatOptions(), + conversationId, + 0, + "question", + false, + null, + CancellationToken.None) + .GetAsyncEnumerator(); + + Assert.IsTrue(await enumerator.MoveNextAsync()); + await enumerator.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(2)); + + await engine.CancellationObserved.Task.WaitAsync(TimeSpan.FromSeconds(2)); + repository.VerifyAll(); + } + + private static ChatController CreateController(IChatRepository repository) => + new( + repository, + Mock.Of(), + Mock.Of(), + NullLogger.Instance); + + private sealed class ThrowingEngine : IAIEngine + { + public Task> GetAvailableModelsAsync( + CancellationToken cancellationToken) => + Task.FromResult>([]); + + public async IAsyncEnumerable ChatAsync( + IReadOnlyList messages, + ChatOptions? options, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + await Task.Yield(); + if (!cancellationToken.IsCancellationRequested) + throw new InvalidOperationException("provider failed"); + cancellationToken.ThrowIfCancellationRequested(); + yield break; + } + } + + private sealed class CancellationAwareEngine : IAIEngine + { + public TaskCompletionSource CancellationObserved { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + + public Task> GetAvailableModelsAsync( + CancellationToken cancellationToken) => + Task.FromResult>([]); + + public async IAsyncEnumerable ChatAsync( + IReadOnlyList messages, + ChatOptions? options, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + yield return new TextDelta("partial"); + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + CancellationObserved.TrySetResult(); + throw; + } + } + } +} diff --git a/SecondDimensionWatcherReDive.Test/ManageTasksToolTests.cs b/SecondDimensionWatcherReDive.Test/ManageTasksToolTests.cs new file mode 100644 index 0000000..fc6702c --- /dev/null +++ b/SecondDimensionWatcherReDive.Test/ManageTasksToolTests.cs @@ -0,0 +1,42 @@ +using System.Text.Json; +using Moq; +using SecondDimensionWatcherReDive.AI.Models; +using SecondDimensionWatcherReDive.Chat.Tools; +using SecondDimensionWatcherReDive.Framework.Tasks; + +namespace SecondDimensionWatcherReDive.Test; + +[TestClass] +public sealed class ManageTasksToolTests +{ + [TestMethod] + public async Task List_UsesSharedLeaseStatuses() + { + var task = new Mock(MockBehavior.Strict); + task.SetupGet(candidate => candidate.Id).Returns("remote-task"); + task.SetupGet(candidate => candidate.Interval).Returns(TimeSpan.FromMinutes(10)); + task.SetupGet(candidate => candidate.IsEnabled).Returns(true); + var lastRunAt = DateTimeOffset.UtcNow.AddMinutes(-4); + var leaseManager = new Mock(MockBehavior.Strict); + leaseManager.Setup(candidate => candidate.GetStatusesAsync( + It.Is>(ids => ids.SequenceEqual(new[] { "remote-task" })), + CancellationToken.None)) + .ReturnsAsync(new Dictionary + { + ["remote-task"] = new(lastRunAt, true) + }); + var tool = new ManageTasksTool([task.Object], leaseManager.Object); + + var result = await tool.ExecuteAsync( + JsonSerializer.SerializeToElement( + new ManageTasksParams(ManageTasksAction.List), + ToolJsonOptions.Options), + CancellationToken.None); + + var success = result as ToolSuccessResult; + Assert.IsNotNull(success); + var status = success.Result.Tasks.Single(); + Assert.AreEqual(lastRunAt, status.LastRunAt); + Assert.IsTrue(status.IsRunning); + } +} diff --git a/SecondDimensionWatcherReDive.Test/PostgresScheduledTaskLeaseManagerTests.cs b/SecondDimensionWatcherReDive.Test/PostgresScheduledTaskLeaseManagerTests.cs new file mode 100644 index 0000000..b269f9f --- /dev/null +++ b/SecondDimensionWatcherReDive.Test/PostgresScheduledTaskLeaseManagerTests.cs @@ -0,0 +1,70 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Framework.Tasks; +using SecondDimensionWatcherReDive.Services; + +namespace SecondDimensionWatcherReDive.Test; + +[TestClass] +public sealed class PostgresScheduledTaskLeaseManagerTests +{ + [TestMethod] + public async Task GetStatusesAsync_DerivesCrossReplicaStateFromPersistedLease() + { + var now = DateTimeOffset.UtcNow; + var lastCompleted = now.AddMinutes(-2); + var repository = new Mock(MockBehavior.Strict); + repository.Setup(candidate => candidate.GetStatesAsync( + It.IsAny>(), + CancellationToken.None)) + .ReturnsAsync( + [ + new ScheduledTaskLeaseState( + "remote-running", + "instance-b", + now.AddMinutes(5), + now.AddMinutes(-1), + lastCompleted), + new ScheduledTaskLeaseState( + "cooldown", + "instance-b", + now.AddMinutes(5), + now.AddMinutes(-2), + now.AddMinutes(-1)), + new ScheduledTaskLeaseState( + "expired-incomplete", + "instance-b", + now.AddMinutes(-1), + now.AddMinutes(-2), + null), + new ScheduledTaskLeaseState( + "ownerless", + null, + now.AddMinutes(5), + now.AddMinutes(-1), + null) + ]); + using var services = new ServiceCollection() + .AddScoped(_ => repository.Object) + .BuildServiceProvider(); + var manager = new PostgresScheduledTaskLeaseManager( + services.GetRequiredService(), + NullLogger.Instance); + + var statuses = await manager.GetStatusesAsync( + ["remote-running", "cooldown", "expired-incomplete", "ownerless", "not-started"], + CancellationToken.None); + + Assert.IsTrue(statuses["remote-running"].IsRunning); + Assert.AreEqual(lastCompleted, statuses["remote-running"].LastRunAt); + Assert.IsFalse(statuses["cooldown"].IsRunning); + Assert.AreEqual(now.AddMinutes(-1), statuses["cooldown"].LastRunAt); + Assert.IsFalse(statuses["expired-incomplete"].IsRunning); + Assert.IsNull(statuses["expired-incomplete"].LastRunAt); + Assert.IsFalse(statuses["ownerless"].IsRunning); + Assert.IsFalse(statuses["not-started"].IsRunning); + Assert.IsNull(statuses["not-started"].LastRunAt); + } +} diff --git a/SecondDimensionWatcherReDive.Test/ScheduledTaskBaseTests.cs b/SecondDimensionWatcherReDive.Test/ScheduledTaskBaseTests.cs index 94b5105..9f4c715 100644 --- a/SecondDimensionWatcherReDive.Test/ScheduledTaskBaseTests.cs +++ b/SecondDimensionWatcherReDive.Test/ScheduledTaskBaseTests.cs @@ -63,6 +63,29 @@ public async Task RunScheduledAsync_ContentionReportsSkippedWithoutForcingCooldo await AssertCanceledAsync(processor); } + [TestMethod] + public async Task RunScheduledAsync_ForceArrivingDuringAcquisitionRetriesAsForced() + { + var task = new BlockingTask(); + var leaseManager = new ForceUpgradeLeaseManager(); + using var cancellation = new CancellationTokenSource(); + var processor = task.ProcessQueueAsync(leaseManager, cancellation.Token); + + var scheduled = task.RunScheduledAsync(CancellationToken.None); + await leaseManager.FirstAcquireStarted.Task.WaitAsync(TimeSpan.FromSeconds(2)); + task.Enqueue(); + leaseManager.ReleaseFirstAcquire.TrySetResult(); + await task.Started.Task.WaitAsync(TimeSpan.FromSeconds(2)); + task.Release.TrySetResult(); + + Assert.IsTrue(await scheduled.WaitAsync(TimeSpan.FromSeconds(2))); + CollectionAssert.AreEqual(new[] { false, true }, leaseManager.Forces); + Assert.AreEqual(1, task.ExecutionCount); + + await cancellation.CancelAsync(); + await AssertCanceledAsync(processor); + } + [TestMethod] public async Task RunScheduledAsync_TemporaryLeaseStoreFailureDoesNotStopQueue() { @@ -147,6 +170,47 @@ private sealed class FakeLeaseManager : IScheduledTaskLeaseManager return Task.FromException(AcquireException); return Task.FromResult(Deny ? null : Lease); } + + public Task> GetStatusesAsync( + IReadOnlyCollection taskIds, + CancellationToken cancellationToken) => + Task.FromResult>( + new Dictionary()); + } + + private sealed class ForceUpgradeLeaseManager : IScheduledTaskLeaseManager + { + public TaskCompletionSource FirstAcquireStarted { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource ReleaseFirstAcquire { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + public bool[] Forces => _forces.ToArray(); + + private readonly List _forces = []; + private readonly FakeLease _lease = new(); + + public async Task TryAcquireAsync( + string taskId, + TimeSpan interval, + bool force, + CancellationToken cancellationToken) + { + _forces.Add(force); + if (_forces.Count == 1) + { + FirstAcquireStarted.TrySetResult(); + await ReleaseFirstAcquire.Task.WaitAsync(cancellationToken); + return null; + } + + return _lease; + } + + public Task> GetStatusesAsync( + IReadOnlyCollection taskIds, + CancellationToken cancellationToken) => + Task.FromResult>( + new Dictionary()); } private sealed class FakeLease : IScheduledTaskExecutionLease diff --git a/SecondDimensionWatcherReDive.Test/TasksControllerTests.cs b/SecondDimensionWatcherReDive.Test/TasksControllerTests.cs new file mode 100644 index 0000000..89cb9c3 --- /dev/null +++ b/SecondDimensionWatcherReDive.Test/TasksControllerTests.cs @@ -0,0 +1,39 @@ +using Microsoft.AspNetCore.Mvc; +using Moq; +using SecondDimensionWatcherReDive.Controllers; +using SecondDimensionWatcherReDive.Framework.Tasks; + +namespace SecondDimensionWatcherReDive.Test; + +[TestClass] +public sealed class TasksControllerTests +{ + [TestMethod] + public async Task GetTasksAsync_UsesSharedLeaseStatuses() + { + var task = new Mock(MockBehavior.Strict); + task.SetupGet(candidate => candidate.Id).Returns("remote-task"); + task.SetupGet(candidate => candidate.Interval).Returns(TimeSpan.FromMinutes(10)); + task.SetupGet(candidate => candidate.IsEnabled).Returns(true); + var lastRunAt = DateTimeOffset.UtcNow.AddMinutes(-3); + var leaseManager = new Mock(MockBehavior.Strict); + leaseManager.Setup(candidate => candidate.GetStatusesAsync( + It.Is>(ids => ids.SequenceEqual(new[] { "remote-task" })), + CancellationToken.None)) + .ReturnsAsync(new Dictionary + { + ["remote-task"] = new(lastRunAt, true) + }); + var controller = new TasksController([task.Object], leaseManager.Object); + + var result = await controller.GetTasksAsync(CancellationToken.None); + + var ok = result as OkObjectResult; + Assert.IsNotNull(ok); + var response = ok.Value as IReadOnlyList; + Assert.IsNotNull(response); + Assert.HasCount(1, response); + Assert.AreEqual(lastRunAt, response[0].LastRunAt); + Assert.IsTrue(response[0].IsRunning); + } +} diff --git a/SecondDimensionWatcherReDive/Controllers/TasksController.cs b/SecondDimensionWatcherReDive/Controllers/TasksController.cs index 50dd947..4cec76d 100644 --- a/SecondDimensionWatcherReDive/Controllers/TasksController.cs +++ b/SecondDimensionWatcherReDive/Controllers/TasksController.cs @@ -8,17 +8,28 @@ namespace SecondDimensionWatcherReDive.Controllers; [ApiController] [Route("api/[controller]")] [Authorize(AuthenticationSchemes = JwtBearerDefaults.AuthenticationScheme)] -internal class TasksController(IEnumerable scheduledTasks) : ControllerBase +internal class TasksController( + IEnumerable scheduledTasks, + IScheduledTaskLeaseManager leaseManager) : ControllerBase { [HttpGet] - public IActionResult GetTasks() + public async Task GetTasksAsync(CancellationToken cancellationToken) { - var tasks = scheduledTasks.Select(t => new External.ScheduledTask( - t.Id, - t.Interval.ToString(), - t.IsEnabled, - t.LastRunAt, - t.IsRunning)).ToList(); + var taskList = scheduledTasks.ToList(); + var statuses = await leaseManager.GetStatusesAsync( + taskList.Select(task => task.Id).ToArray(), + cancellationToken); + var tasks = taskList.Select(task => + { + var status = statuses.GetValueOrDefault(task.Id) + ?? new ScheduledTaskStatus(null, false); + return new External.ScheduledTask( + task.Id, + task.Interval.ToString(), + task.IsEnabled, + status.LastRunAt, + status.IsRunning); + }).ToList(); return Ok(tasks); } diff --git a/SecondDimensionWatcherReDive/Repositories/FileMappingRepositoryPostgreSqlTestFixture.cs b/SecondDimensionWatcherReDive/Repositories/FileMappingRepositoryPostgreSqlTestFixture.cs index 4068035..b9b2987 100644 --- a/SecondDimensionWatcherReDive/Repositories/FileMappingRepositoryPostgreSqlTestFixture.cs +++ b/SecondDimensionWatcherReDive/Repositories/FileMappingRepositoryPostgreSqlTestFixture.cs @@ -249,6 +249,16 @@ public async Task CompleteTaskLeaseAsync( cancellationToken); } + public async Task> GetTaskLeaseStatesAsync( + IReadOnlyCollection taskIds, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + return await new ScheduledTaskLeaseRepository(context).GetStatesAsync( + taskIds, + cancellationToken); + } + public async Task RetryJobsAsync( IReadOnlyCollection ids, DateTimeOffset now, diff --git a/SecondDimensionWatcherReDive/Repositories/ScheduledTaskLeaseRepository.cs b/SecondDimensionWatcherReDive/Repositories/ScheduledTaskLeaseRepository.cs index 7e1d49f..32d6f5f 100644 --- a/SecondDimensionWatcherReDive/Repositories/ScheduledTaskLeaseRepository.cs +++ b/SecondDimensionWatcherReDive/Repositories/ScheduledTaskLeaseRepository.cs @@ -92,4 +92,23 @@ public Task CompleteAsync( state => succeeded ? completedAt : state.LastSucceededAt) .SetProperty(state => state.LastError, succeeded ? null : error), cancellationToken); + + public async Task> GetStatesAsync( + IReadOnlyCollection taskIds, + CancellationToken cancellationToken) + { + if (taskIds.Count == 0) return []; + + var ids = taskIds.Distinct(StringComparer.Ordinal).ToArray(); + return await context.ScheduledTaskStates + .AsNoTracking() + .Where(state => ids.Contains(state.TaskId)) + .Select(state => new ScheduledTaskLeaseState( + state.TaskId, + state.LeaseOwner, + state.LeaseExpiresAt, + state.LastStartedAt, + state.LastCompletedAt)) + .ToListAsync(cancellationToken); + } } diff --git a/SecondDimensionWatcherReDive/Services/PostgresScheduledTaskLeaseManager.cs b/SecondDimensionWatcherReDive/Services/PostgresScheduledTaskLeaseManager.cs index c16108d..ebafb6c 100644 --- a/SecondDimensionWatcherReDive/Services/PostgresScheduledTaskLeaseManager.cs +++ b/SecondDimensionWatcherReDive/Services/PostgresScheduledTaskLeaseManager.cs @@ -33,6 +33,41 @@ public sealed partial class PostgresScheduledTaskLeaseManager( : null; } + public async Task> GetStatusesAsync( + IReadOnlyCollection taskIds, + CancellationToken cancellationToken) + { + var ids = taskIds.Distinct(StringComparer.Ordinal).ToArray(); + if (ids.Length == 0) + return new Dictionary(StringComparer.Ordinal); + + await using var scope = scopeFactory.CreateAsyncScope(); + var repository = scope.ServiceProvider.GetRequiredService(); + var persistedStates = await repository.GetStatesAsync(ids, cancellationToken); + var statesById = persistedStates.ToDictionary( + state => state.TaskId, + StringComparer.Ordinal); + var now = DateTimeOffset.UtcNow; + return ids.ToDictionary( + taskId => taskId, + taskId => statesById.TryGetValue(taskId, out var state) + ? ToStatus(state, now) + : new ScheduledTaskStatus(null, false), + StringComparer.Ordinal); + } + + private static ScheduledTaskStatus ToStatus( + ScheduledTaskLeaseState state, + DateTimeOffset now) + { + var isRunning = state.LeaseOwner is not null + && state.LeaseExpiresAt > now + && state.LastStartedAt is { } startedAt + && (state.LastCompletedAt is null + || startedAt > state.LastCompletedAt); + return new ScheduledTaskStatus(state.LastCompletedAt, isRunning); + } + private sealed class ExecutionLease : IScheduledTaskExecutionLease { private readonly string _taskId;