From ef2780e2bef8b2e2e465cef4bf674a46decad734 Mon Sep 17 00:00:00 2001 From: mahoshojoHCG Date: Sat, 29 Aug 2026 22:46:20 +0800 Subject: [PATCH 01/45] feat: add verified backup restore and data transfer --- .github/workflows/backup-restore.yml | 165 +++++ Containerfile | 10 +- .../ILogicalDataTransferRepository.cs | 16 + .../DataRepository/LogicalDataTransfer.cs | 105 +++ .../LogicalDataTransferPostgreSqlTests.cs | 351 ++++++++++ .../LogicalDataTransferControllerTests.cs | 148 +++++ .../External/AppJsonSerializerContext.cs | 5 + .../External/LogicalDataTransfer.cs | 13 + .../LogicalDataTransferController.cs | 221 +++++++ .../ApplicationContextModelSnapshot.cs | 1 - .../Models/ApplicationContext.cs | 4 + SecondDimensionWatcherReDive/Program.cs | 1 + .../LogicalDataTransferRepository.cs | 585 +++++++++++++++++ deployments/podman-compose.yml | 3 + deployments/sdw-backup | 605 ++++++++++++++++++ deployments/tests/backup-restore-smoke.sh | 80 +++ docs/backup-restore.md | 145 +++++ docs/container-deployment.md | 2 + docs/server-deployment.md | 2 + packaging/backup.env | 17 + packaging/nfpm.yaml | 29 + packaging/postinstall.sh | 7 +- packaging/sdw-backup.service | 16 + packaging/sdw-backup.timer | 11 + 24 files changed, 2539 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/backup-restore.yml create mode 100644 SecondDimensionWatcherReDive.Framework/DataRepository/ILogicalDataTransferRepository.cs create mode 100644 SecondDimensionWatcherReDive.Framework/DataRepository/LogicalDataTransfer.cs create mode 100644 SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/LogicalDataTransferPostgreSqlTests.cs create mode 100644 SecondDimensionWatcherReDive.Test/LogicalDataTransferControllerTests.cs create mode 100644 SecondDimensionWatcherReDive/Controllers/External/LogicalDataTransfer.cs create mode 100644 SecondDimensionWatcherReDive/Controllers/LogicalDataTransferController.cs create mode 100644 SecondDimensionWatcherReDive/Repositories/LogicalDataTransferRepository.cs create mode 100755 deployments/sdw-backup create mode 100755 deployments/tests/backup-restore-smoke.sh create mode 100644 docs/backup-restore.md create mode 100644 packaging/backup.env create mode 100644 packaging/sdw-backup.service create mode 100644 packaging/sdw-backup.timer diff --git a/.github/workflows/backup-restore.yml b/.github/workflows/backup-restore.yml new file mode 100644 index 0000000..f0c664c --- /dev/null +++ b/.github/workflows/backup-restore.yml @@ -0,0 +1,165 @@ +name: Backup Restore Drill + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: backup-restore-${{ github.ref }} + cancel-in-progress: true + +jobs: + drill: + runs-on: ubuntu-latest + timeout-minutes: 15 + services: + postgres: + image: postgres:16-alpine + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: sdw_source + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres -d sdw_source" + --health-interval 5s + --health-timeout 5s + --health-retries 12 + env: + PGHOST: 127.0.0.1 + PGPORT: 5432 + PGUSER: postgres + PGPASSWORD: postgres + PGDATABASE: sdw_source + ConnectionStrings__sdw: Host=127.0.0.1;Port=5432;Username=postgres;Password=postgres;Database=sdw_source + JwtSecret: backup-restore-drill-jwt-secret-with-at-least-32-bytes + ASPNETCORE_ENVIRONMENT: Production + WORK_DIR: ${{ runner.temp }}/sdw-backup-drill + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-dotnet@v6 + with: + dotnet-version: "10.0.x" + + - name: Install PostgreSQL client + run: sudo apt-get update && sudo apt-get install -y postgresql-client + + - name: Build backend + run: >- + dotnet build SecondDimensionWatcherReDive/SecondDimensionWatcherReDive.csproj + -c Release /p:Version="$(tr -d '[:space:]' < VERSION)" + + - name: Migrate and health-check source instance + shell: bash + run: | + set -Eeuo pipefail + mkdir -p "$WORK_DIR/source/keys" "$WORK_DIR/plugins/example" "$WORK_DIR/backups" + printf '{}\n' > "$WORK_DIR/source/password.json" + printf 'DisableCors: true\n' > "$WORK_DIR/source/appsettings.yml" + printf '{"name":"example","version":"1"}\n' > "$WORK_DIR/plugins/example/manifest.json" + export PasswordFile="$WORK_DIR/source/password.json" + export DataProtection__KeyRingPath="$WORK_DIR/source/keys" + export Config="$WORK_DIR/source/appsettings.yml" + dotnet run --project SecondDimensionWatcherReDive -c Release --no-build \ + --urls http://127.0.0.1:5097 >"$WORK_DIR/source-app.log" 2>&1 & + app_pid=$! + trap 'kill "$app_pid" 2>/dev/null || true' EXIT + for attempt in {1..60}; do + if curl --silent --fail http://127.0.0.1:5097/api/auth/allowRegister >/dev/null; then + break + fi + sleep 1 + done + curl --fail http://127.0.0.1:5097/api/auth/allowRegister >/dev/null + kill "$app_pid" + wait "$app_pid" || true + trap - EXIT + psql --no-psqlrc --command \ + "INSERT INTO \"Feeds\" (\"Id\", \"Url\", \"Name\", \"CreatedAt\") VALUES ('11111111-1111-1111-1111-111111111111', 'https://example.com/feed', 'drill', now())" + + - name: Create, verify, and reject corruption + shell: bash + run: | + set -Eeuo pipefail + archive=$(deployments/sdw-backup create \ + --output "$WORK_DIR/backups" \ + --config "$WORK_DIR/source/appsettings.yml" \ + --password-file "$WORK_DIR/source/password.json" \ + --key-ring "$WORK_DIR/source/keys" \ + --plugin-dir "$WORK_DIR/plugins" \ + --retention-days 7 \ + --app-version "$(tr -d '[:space:]' < VERSION)") + deployments/sdw-backup verify "$archive" + cp "$archive" "$WORK_DIR/corrupt.tar.gz" + archive_size=$(stat --format=%s "$WORK_DIR/corrupt.tar.gz") + printf 'CORRUPT' | dd of="$WORK_DIR/corrupt.tar.gz" bs=1 \ + seek=$((archive_size / 2)) conv=notrunc status=none + if deployments/sdw-backup verify "$WORK_DIR/corrupt.tar.gz"; then + echo "corrupted archive unexpectedly verified" >&2 + exit 1 + fi + printf '%s\n' "$archive" > "$WORK_DIR/archive-path" + + - name: Restore into a fresh database and health-check it + shell: bash + run: | + set -Eeuo pipefail + createdb sdw_restore + export PGDATABASE=sdw_restore + export ConnectionStrings__sdw='Host=127.0.0.1;Port=5432;Username=postgres;Password=postgres;Database=sdw_restore' + archive=$(cat "$WORK_DIR/archive-path") + mkdir -p "$WORK_DIR/restored" + psql --no-psqlrc --command \ + 'CREATE TABLE restore_guard (value integer NOT NULL); INSERT INTO restore_guard VALUES (1)' + if deployments/sdw-backup restore "$WORK_DIR/corrupt.tar.gz" \ + --confirm-replace --expected-version "$(tr -d '[:space:]' < VERSION)"; then + echo "corrupted restore unexpectedly started" >&2 + exit 1 + fi + test "$(psql --no-psqlrc --tuples-only --no-align --command \ + 'SELECT value FROM restore_guard')" = "1" + if deployments/sdw-backup restore "$archive" \ + --confirm-replace --expected-version 999.0.0; then + echo "incompatible restore unexpectedly started" >&2 + exit 1 + fi + test "$(psql --no-psqlrc --tuples-only --no-align --command \ + 'SELECT value FROM restore_guard')" = "1" + deployments/sdw-backup restore "$archive" \ + --confirm-replace \ + --expected-version "$(tr -d '[:space:]' < VERSION)" \ + --config-destination "$WORK_DIR/restored/appsettings.yml" \ + --password-destination "$WORK_DIR/restored/password.json" \ + --key-ring-destination "$WORK_DIR/restored/keys" \ + --plugin-destination "$WORK_DIR/restored/plugins" \ + --safety-directory "$WORK_DIR/backups" + test "$(psql --no-psqlrc --tuples-only --no-align --command 'SELECT count(*) FROM "Feeds"')" = "1" + cmp "$WORK_DIR/source/password.json" "$WORK_DIR/restored/password.json" + cmp "$WORK_DIR/source/appsettings.yml" "$WORK_DIR/restored/appsettings.yml" + cmp "$WORK_DIR/plugins/example/manifest.json" \ + "$WORK_DIR/restored/plugins/example/manifest.json" + export PasswordFile="$WORK_DIR/restored/password.json" + export DataProtection__KeyRingPath="$WORK_DIR/restored/keys" + export Config="$WORK_DIR/restored/appsettings.yml" + dotnet run --project SecondDimensionWatcherReDive -c Release --no-build \ + --urls http://127.0.0.1:5098 >"$WORK_DIR/restored-app.log" 2>&1 & + app_pid=$! + trap 'kill "$app_pid" 2>/dev/null || true' EXIT + for attempt in {1..60}; do + if curl --silent --fail http://127.0.0.1:5098/api/auth/allowRegister >/dev/null; then + break + fi + sleep 1 + done + curl --fail http://127.0.0.1:5098/api/auth/allowRegister >/dev/null + kill "$app_pid" + wait "$app_pid" || true + trap - EXIT diff --git a/Containerfile b/Containerfile index 72b49a9..4bc08c8 100644 --- a/Containerfile +++ b/Containerfile @@ -10,18 +10,26 @@ RUN yarn build FROM mcr.microsoft.com/dotnet/sdk:10.0 AS backend-build WORKDIR /src COPY SecondDimensionWatcherReDive.slnx . +COPY VERSION . COPY SecondDimensionWatcherReDive.Framework/ SecondDimensionWatcherReDive.Framework/ COPY SecondDimensionWatcherReDive/ SecondDimensionWatcherReDive/ COPY Plugins/ Plugins/ COPY Share/ Share/ COPY --from=frontend-build /app/dist SecondDimensionWatcherReDive/wwwroot/ RUN dotnet restore SecondDimensionWatcherReDive/SecondDimensionWatcherReDive.csproj -RUN dotnet publish SecondDimensionWatcherReDive/SecondDimensionWatcherReDive.csproj -c Release -o /app --no-restore +RUN dotnet publish SecondDimensionWatcherReDive/SecondDimensionWatcherReDive.csproj \ + -c Release -o /app --no-restore \ + /p:Version="$(tr -d '[:space:]' < VERSION)" # Stage 3: Runtime FROM mcr.microsoft.com/dotnet/aspnet:10.0 WORKDIR /app +RUN apt-get update \ + && apt-get install -y --no-install-recommends postgresql-client \ + && rm -rf /var/lib/apt/lists/* COPY --from=backend-build /app . +COPY deployments/sdw-backup /usr/local/bin/sdw-backup +COPY VERSION /usr/lib/sdw-redive/VERSION EXPOSE 8080 # Optional: read-only NFSv4 export (set Nfs:Enabled=true to activate; publish port at run time). EXPOSE 2049 diff --git a/SecondDimensionWatcherReDive.Framework/DataRepository/ILogicalDataTransferRepository.cs b/SecondDimensionWatcherReDive.Framework/DataRepository/ILogicalDataTransferRepository.cs new file mode 100644 index 0000000..2186b9d --- /dev/null +++ b/SecondDimensionWatcherReDive.Framework/DataRepository/ILogicalDataTransferRepository.cs @@ -0,0 +1,16 @@ +namespace SecondDimensionWatcherReDive.Framework.DataRepository; + +public interface ILogicalDataTransferRepository +{ + Task ExportAsync( + LogicalDataCategory categories, + Guid userId, + string applicationVersion, + CancellationToken cancellationToken); + + Task ImportAsync( + LogicalDataBundle bundle, + LogicalImportConflictStrategy conflictStrategy, + Guid userId, + CancellationToken cancellationToken); +} diff --git a/SecondDimensionWatcherReDive.Framework/DataRepository/LogicalDataTransfer.cs b/SecondDimensionWatcherReDive.Framework/DataRepository/LogicalDataTransfer.cs new file mode 100644 index 0000000..d8d4028 --- /dev/null +++ b/SecondDimensionWatcherReDive.Framework/DataRepository/LogicalDataTransfer.cs @@ -0,0 +1,105 @@ +using System.Text.Json.Serialization; + +namespace SecondDimensionWatcherReDive.Framework.DataRepository; + +[Flags] +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum LogicalDataCategory +{ + None = 0, + Feeds = 1, + AutomationPolicies = 2, + FileNameRules = 4, + MetadataCorrections = 8, + Playback = 16, + All = Feeds | AutomationPolicies | FileNameRules | MetadataCorrections | Playback +} + +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum LogicalImportConflictStrategy +{ + Skip, + Overwrite, + Fail +} + +public sealed record LogicalFeed( + Guid Id, + string Url, + string? Name, + DateTimeOffset CreatedAt); + +public sealed record LogicalAutomationPolicy( + string FeedUrl, + IReadOnlyList SubtitleGroups, + IReadOnlyList Resolutions, + IReadOnlyList Codecs, + IReadOnlyList Languages, + long? MinSizeBytes, + long? MaxSizeBytes, + IReadOnlyList ExcludedKeywords, + SubscriptionAutomationMode Mode, + DateTimeOffset CreatedAt, + DateTimeOffset UpdatedAt); + +public sealed record LogicalFileNameRule( + Guid Id, + string AnimationTmdbId, + string AnimationName, + string AnimationOriginalName, + string? AnimationPosterPath, + string Pattern, + string? Description, + DateTimeOffset CreatedAt); + +public sealed record LogicalMetadataCorrection( + Guid OperationId, + string ReleaseDownloadUrl, + string ReleaseTitle, + DateTimeOffset ReleasePublishTime, + string AnimationTmdbId, + string AnimationName, + string AnimationOriginalName, + string? AnimationPosterPath, + string Description, + int? Season, + int? Episode, + string? GroupName, + DateTimeOffset AppliedAt); + +public sealed record LogicalPlaybackProgress( + string VirtualPath, + double PositionSeconds, + double DurationSeconds, + bool IsWatched, + DateTimeOffset UpdatedAt, + DateTimeOffset? WatchedAt); + +public sealed record LogicalPlaybackPreferences( + string? SubtitleLanguage, + string? SubtitleTrackLabel, + string? AudioLanguage, + string? AudioTrackLabel, + bool AutoPlayNext, + DateTimeOffset UpdatedAt); + +public sealed record LogicalDataBundle( + int FormatVersion, + DateTimeOffset ExportedAtUtc, + string ApplicationVersion, + LogicalDataCategory Categories, + IReadOnlyList Feeds, + IReadOnlyList AutomationPolicies, + IReadOnlyList FileNameRules, + IReadOnlyList MetadataCorrections, + IReadOnlyList PlaybackProgress, + LogicalPlaybackPreferences? PlaybackPreferences); + +public sealed record LogicalImportResult( + int Added, + int Updated, + int Skipped, + int Conflicts, + IReadOnlyList Messages); + +public sealed class LogicalDataImportConflictException(string message) : InvalidOperationException(message); diff --git a/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/LogicalDataTransferPostgreSqlTests.cs b/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/LogicalDataTransferPostgreSqlTests.cs new file mode 100644 index 0000000..d39c863 --- /dev/null +++ b/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/LogicalDataTransferPostgreSqlTests.cs @@ -0,0 +1,351 @@ +using Microsoft.EntityFrameworkCore; +using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Repositories; +using Testcontainers.PostgreSql; +using Models = SecondDimensionWatcherReDive.Models; + +namespace SecondDimensionWatcherReDive.IntegrationTest.PostgreSql; + +[TestClass] +public sealed class LogicalDataTransferPostgreSqlTests +{ + private static readonly PostgreSqlContainer Database = new PostgreSqlBuilder("postgres:17-alpine") + .WithDatabase("sdw_transfer_tests") + .WithUsername("postgres") + .WithPassword("postgres") + .Build(); + + private static DbContextOptions Options = null!; + + [ClassInitialize] + public static async Task InitializeAsync(TestContext _) + { + await Database.StartAsync(); + Options = new DbContextOptionsBuilder() + .UseNpgsql(Database.GetConnectionString()) + .Options; + } + + [ClassCleanup] + public static async Task CleanupAsync() => await Database.DisposeAsync(); + + [TestInitialize] + public async Task ResetAsync() + { + await using var context = new Models.ApplicationContext(Options); + await context.Database.EnsureDeletedAsync(); + await context.Database.MigrateAsync(); + } + + [TestMethod] + public async Task FeedsPoliciesAndRulesRoundTripWithExplicitConflictStrategies() + { + LogicalDataBundle bundle; + await using (var source = new Models.ApplicationContext(Options)) + { + var feed = new Models.Feed + { + Id = Guid.NewGuid(), + Url = "https://example.com/feed.xml", + Name = "Example", + CreatedAt = DateTimeOffset.UtcNow.AddDays(-2) + }; + var animation = new Models.Animation + { + Id = Guid.NewGuid(), + TmdbId = "tv:123", + Name = "Example Show", + OriginalName = "Example Show", + PosterPath = "/poster.jpg" + }; + source.AddRange( + feed, + animation, + new Models.SubscriptionAutomationPolicy + { + FeedId = feed.Id, + Feed = feed, + SubtitleGroups = ["Group"], + Resolutions = ["1080p"], + Codecs = ["HEVC"], + Languages = ["zh-Hans"], + ExcludedKeywords = ["batch"], + Mode = SubscriptionAutomationMode.ManualConfirm, + CreatedAt = feed.CreatedAt, + UpdatedAt = DateTimeOffset.UtcNow + }, + new Models.FileNameRegexRule + { + Id = Guid.NewGuid(), + AnimationId = animation.Id, + Pattern = @"E(?\d+)", + Description = "episode", + CreatedAt = DateTimeOffset.UtcNow + }); + await source.SaveChangesAsync(); + bundle = await new LogicalDataTransferRepository(source).ExportAsync( + LogicalDataCategory.Feeds | + LogicalDataCategory.AutomationPolicies | + LogicalDataCategory.FileNameRules, + Guid.Empty, + "1.0.0", + CancellationToken.None); + + await source.SubscriptionAutomationPolicies.ExecuteDeleteAsync(); + await source.FileNameRegexRules.ExecuteDeleteAsync(); + await source.Feeds.ExecuteDeleteAsync(); + await source.Animations.ExecuteDeleteAsync(); + } + + await using var target = new Models.ApplicationContext(Options); + var repository = new LogicalDataTransferRepository(target); + var first = await repository.ImportAsync( + bundle, + LogicalImportConflictStrategy.Skip, + Guid.Empty, + CancellationToken.None); + Assert.AreEqual(3, first.Added); + Assert.AreEqual(1, await target.Feeds.CountAsync()); + Assert.AreEqual(1, await target.SubscriptionAutomationPolicies.CountAsync()); + Assert.AreEqual(1, await target.FileNameRegexRules.CountAsync()); + + var repeated = await repository.ImportAsync( + bundle, + LogicalImportConflictStrategy.Skip, + Guid.Empty, + CancellationToken.None); + Assert.AreEqual(3, repeated.Skipped); + Assert.AreEqual(1, repeated.Conflicts); + + var changed = bundle with + { + Feeds = [bundle.Feeds[0] with { Name = "Renamed" }], + AutomationPolicies = + [bundle.AutomationPolicies[0] with { Mode = SubscriptionAutomationMode.AutoDownload }], + FileNameRules = [bundle.FileNameRules[0] with { Description = "updated" }] + }; + var overwritten = await repository.ImportAsync( + changed, + LogicalImportConflictStrategy.Overwrite, + Guid.Empty, + CancellationToken.None); + Assert.AreEqual(3, overwritten.Updated); + Assert.AreEqual("Renamed", (await target.Feeds.SingleAsync()).Name); + Assert.AreEqual(SubscriptionAutomationMode.AutoDownload, + (await target.SubscriptionAutomationPolicies.SingleAsync()).Mode); + Assert.AreEqual("updated", (await target.FileNameRegexRules.SingleAsync()).Description); + } + + [TestMethod] + public async Task MetadataAndPlaybackUseStableReleaseAndVirtualPathKeys() + { + var publishedAt = DateTimeOffset.UtcNow.AddDays(-1); + const string DownloadUrl = "https://example.com/release.torrent"; + const string VirtualPath = "/Example/Group/Example S01E02.mkv"; + LogicalDataBundle bundle; + + await using (var source = new Models.ApplicationContext(Options)) + { + var animation = Animation("tv:456", "Correct Show"); + var group = new Models.AnimationGroup { Id = Guid.NewGuid(), Name = "Correct Group" }; + var info = Release(Guid.NewGuid(), DownloadUrl, publishedAt); + info.Animation = animation; + info.Group = group; + info.Description = "corrected description"; + info.Season = 1; + info.Episode = 2; + info.MetadataStatus = MetadataReviewStatus.Reviewed; + info.MetadataReviewedAt = DateTimeOffset.UtcNow; + info.StateVersion = 1; + var mapping = Mapping(info.Id, VirtualPath); + source.AddRange(animation, group, info, mapping); + await source.SaveChangesAsync(); + + var operation = new Models.MetadataReviewOperation + { + Id = Guid.NewGuid(), + AnimationInfoId = info.Id, + AnimationInfo = info, + State = MetadataReviewOperationState.Applied, + CreatedAt = DateTimeOffset.UtcNow.AddMinutes(-2), + ExpiresAt = DateTimeOffset.UtcNow.AddDays(1), + BaseVersion = 0, + ProposedAnimationTmdbId = animation.TmdbId, + ProposedAnimationName = animation.Name, + ProposedAnimationOriginalName = animation.OriginalName, + ProposedAnimationPosterPath = animation.PosterPath, + ProposedDescription = info.Description, + ProposedSeason = info.Season, + ProposedEpisode = info.Episode, + ProposedGroupName = group.Name, + AppliedAt = DateTimeOffset.UtcNow.AddMinutes(-1), + AppliedVersion = 1, + PreviousDescription = "uncorrected", + PreviousMetadataStatus = MetadataReviewStatus.LowConfidence, + PreviousIsAiProcessed = true, + PreviousAiRetryCount = 0 + }; + info.CurrentMetadataReviewOperationId = operation.Id; + source.AddRange( + operation, + new Models.MetadataReviewOperation + { + Id = Guid.NewGuid(), + AnimationInfoId = info.Id, + AnimationInfo = info, + State = MetadataReviewOperationState.Applied, + CreatedAt = DateTimeOffset.UtcNow.AddDays(-2), + ExpiresAt = DateTimeOffset.UtcNow.AddDays(-1), + BaseVersion = 0, + AppliedAt = DateTimeOffset.UtcNow.AddDays(-1), + AppliedVersion = 0 + }, + new Models.PlaybackProgress + { + Id = Guid.NewGuid(), + UserId = Guid.Empty, + AnimationInfoId = info.Id, + VirtualPath = VirtualPath, + PositionSeconds = 600, + DurationSeconds = 1_440, + UpdatedAt = DateTimeOffset.UtcNow + }); + await source.SaveChangesAsync(); + await source.Database.ExecuteSqlInterpolatedAsync( + $""" + INSERT INTO "PlaybackPreferences" + ("UserId", "SubtitleLanguage", "AutoPlayNext", "UpdatedAt") + VALUES ({Guid.Empty}, {"zh-Hans"}, {false}, {DateTimeOffset.UtcNow}) + """); + Assert.AreEqual(1, await source.PlaybackPreferences.AsNoTracking().CountAsync()); + + bundle = await new LogicalDataTransferRepository(source).ExportAsync( + LogicalDataCategory.MetadataCorrections | LogicalDataCategory.Playback, + Guid.Empty, + "1.0.0", + CancellationToken.None); + Assert.IsNotNull(bundle.PlaybackPreferences); + Assert.AreEqual(1, bundle.MetadataCorrections.Count); + Assert.AreEqual(operation.Id, bundle.MetadataCorrections[0].OperationId); + + await source.MetadataReviewOperations.ExecuteDeleteAsync(); + await source.PlaybackProgresses.ExecuteDeleteAsync(); + await source.PlaybackPreferences.ExecuteDeleteAsync(); + await source.FileMappings.ExecuteDeleteAsync(); + await source.AnimationInfo.ExecuteDeleteAsync(); + await source.Animations.ExecuteDeleteAsync(); + await source.AnimationGroups.ExecuteDeleteAsync(); + } + + var targetInfoId = Guid.NewGuid(); + await using var target = new Models.ApplicationContext(Options); + var targetInfo = Release(targetInfoId, DownloadUrl, publishedAt); + targetInfo.Description = "uncorrected"; + target.AddRange(targetInfo, Mapping(targetInfoId, VirtualPath)); + await target.SaveChangesAsync(); + + var result = await new LogicalDataTransferRepository(target).ImportAsync( + bundle, + LogicalImportConflictStrategy.Skip, + Guid.Empty, + CancellationToken.None); + + Assert.AreEqual(0, result.Skipped, string.Join(Environment.NewLine, result.Messages)); + Assert.AreEqual(3, result.Added); + var importedInfo = await target.AnimationInfo + .Include(info => info.Animation) + .Include(info => info.Group) + .SingleAsync(); + Assert.AreEqual("corrected description", importedInfo.Description); + Assert.AreEqual("tv:456", importedInfo.Animation!.TmdbId); + Assert.AreEqual("Correct Group", importedInfo.Group!.Name); + Assert.IsNotNull(importedInfo.CurrentMetadataReviewOperationId); + var progress = await target.PlaybackProgresses.SingleAsync(); + Assert.AreEqual(targetInfoId, progress.AnimationInfoId); + Assert.AreEqual(600, progress.PositionSeconds); + var preferences = await target.PlaybackPreferences.SingleAsync(); + Assert.AreEqual(Guid.Empty, preferences.UserId); + Assert.IsFalse(preferences.AutoPlayNext); + } + + [TestMethod] + public async Task FailConflictStrategyRollsBackEarlierItems() + { + await using (var seed = new Models.ApplicationContext(Options)) + { + seed.Feeds.Add(new Models.Feed + { + Id = Guid.NewGuid(), + Url = "https://example.com/existing.xml", + Name = "Existing", + CreatedAt = DateTimeOffset.UtcNow + }); + await seed.SaveChangesAsync(); + } + + var bundle = new LogicalDataBundle( + 1, + DateTimeOffset.UtcNow, + "1.0.0", + LogicalDataCategory.Feeds, + [ + new LogicalFeed(Guid.NewGuid(), "https://example.com/new.xml", "New", DateTimeOffset.UtcNow), + new LogicalFeed(Guid.NewGuid(), "https://example.com/existing.xml", "Changed", DateTimeOffset.UtcNow) + ], + [], + [], + [], + [], + null); + + await using (var importing = new Models.ApplicationContext(Options)) + { + await Assert.ThrowsAsync(() => + new LogicalDataTransferRepository(importing).ImportAsync( + bundle, + LogicalImportConflictStrategy.Fail, + Guid.Empty, + CancellationToken.None)); + } + + await using var verification = new Models.ApplicationContext(Options); + Assert.AreEqual(1, await verification.Feeds.CountAsync()); + Assert.AreEqual("Existing", (await verification.Feeds.SingleAsync()).Name); + } + + private static Models.Animation Animation(string tmdbId, string name) => + new() + { + Id = Guid.NewGuid(), + TmdbId = tmdbId, + Name = name, + OriginalName = name, + PosterPath = "/poster.jpg" + }; + + private static Models.AnimationInfo Release( + Guid id, + string downloadUrl, + DateTimeOffset publishedAt) => + new() + { + Id = id, + Title = "[Group] Example - 02", + Description = "uncorrected", + PublishTime = publishedAt, + DownloadUrl = downloadUrl, + DownloadType = "torrent", + IsAiProcessed = true, + MetadataStatus = MetadataReviewStatus.LowConfidence + }; + + private static Models.FileMapping Mapping(Guid animationInfoId, string virtualPath) => + new() + { + Id = Guid.NewGuid(), + AnimationInfoId = animationInfoId, + VirtualPath = virtualPath, + PhysicalPath = "/media/example.mkv", + FileStore = "local" + }; +} diff --git a/SecondDimensionWatcherReDive.Test/LogicalDataTransferControllerTests.cs b/SecondDimensionWatcherReDive.Test/LogicalDataTransferControllerTests.cs new file mode 100644 index 0000000..56c5744 --- /dev/null +++ b/SecondDimensionWatcherReDive.Test/LogicalDataTransferControllerTests.cs @@ -0,0 +1,148 @@ +using System.Security.Cryptography; +using System.Text.Json; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Moq; +using SecondDimensionWatcherReDive.Controllers; +using SecondDimensionWatcherReDive.Controllers.External; +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Test; + +[TestClass] +public sealed class LogicalDataTransferControllerTests +{ + [TestMethod] + public async Task ExportSelectsCategoriesAndReturnsChecksummedEnvelope() + { + var repository = new Mock(); + repository.Setup(item => item.ExportAsync( + LogicalDataCategory.Feeds | LogicalDataCategory.Playback, + Guid.Empty, + It.IsAny(), + It.IsAny())) + .ReturnsAsync((LogicalDataCategory categories, Guid _, string version, CancellationToken _) => + Bundle(version, categories)); + var controller = Controller(repository.Object); + + var action = await controller.ExportAsync("feeds,playback", CancellationToken.None); + + var file = (FileContentResult)action; + Assert.AreEqual("application/json", file.ContentType); + StringAssert.StartsWith(file.FileDownloadName!, "sdw-logical-export-"); + var envelope = JsonSerializer.Deserialize( + file.FileContents, + AppJsonSerializerContext.Default.LogicalDataExportEnvelope)!; + Assert.AreEqual(LogicalDataCategory.Feeds | LogicalDataCategory.Playback, + envelope.Data.Categories); + Assert.AreEqual(Digest(envelope.Data), envelope.Sha256); + } + + [TestMethod] + public async Task ImportRejectsChecksumMismatchBeforeRepositoryWrite() + { + var repository = new Mock(); + var controller = Controller(repository.Object); + var request = new LogicalDataImportRequest( + Bundle("1.0.0", LogicalDataCategory.Feeds), + new string('0', 64), + LogicalImportConflictStrategy.Skip); + + var action = await controller.ImportAsync(request, CancellationToken.None); + + Assert.IsInstanceOfType(action); + repository.Verify(item => item.ImportAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), Times.Never); + } + + [TestMethod] + public async Task ImportRejectsUnknownConflictStrategyBeforeRepositoryWrite() + { + var repository = new Mock(); + var controller = Controller(repository.Object); + var bundle = Bundle("1.0.0", LogicalDataCategory.Feeds); + var request = new LogicalDataImportRequest( + bundle, + Digest(bundle), + (LogicalImportConflictStrategy)999); + + var action = await controller.ImportAsync(request, CancellationToken.None); + + Assert.IsInstanceOfType(action); + repository.Verify(item => item.ImportAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), Times.Never); + } + + [TestMethod] + public async Task ImportPassesValidatedBundleAndConflictStrategy() + { + var repository = new Mock(); + repository.Setup(item => item.ExportAsync( + It.IsAny(), Guid.Empty, It.IsAny(), + It.IsAny())) + .ReturnsAsync((LogicalDataCategory categories, Guid _, string version, CancellationToken _) => + Bundle(version, categories)); + repository.Setup(item => item.ImportAsync( + It.IsAny(), + LogicalImportConflictStrategy.Overwrite, + Guid.Empty, + It.IsAny())) + .ReturnsAsync(new LogicalImportResult(1, 0, 0, 0, [])); + var controller = Controller(repository.Object); + var export = (FileContentResult)await controller.ExportAsync("feeds", CancellationToken.None); + var envelope = JsonSerializer.Deserialize( + export.FileContents, + AppJsonSerializerContext.Default.LogicalDataExportEnvelope)!; + + var action = await controller.ImportAsync( + new LogicalDataImportRequest( + envelope.Data, + envelope.Sha256, + LogicalImportConflictStrategy.Overwrite), + CancellationToken.None); + + var result = (LogicalImportResult)((OkObjectResult)action).Value!; + Assert.AreEqual(1, result.Added); + } + + private static LogicalDataTransferController Controller( + ILogicalDataTransferRepository repository) => + new(repository) + { + ControllerContext = new ControllerContext + { + HttpContext = new DefaultHttpContext() + } + }; + + private static LogicalDataBundle Bundle( + string version, + LogicalDataCategory categories) => + new( + 1, + DateTimeOffset.UtcNow, + version, + categories, + categories.HasFlag(LogicalDataCategory.Feeds) + ? [new LogicalFeed(Guid.NewGuid(), "https://example.com/feed", "Example", DateTimeOffset.UtcNow)] + : [], + [], + [], + [], + [], + null); + + private static string Digest(LogicalDataBundle bundle) + { + var bytes = JsonSerializer.SerializeToUtf8Bytes( + bundle, + AppJsonSerializerContext.Default.LogicalDataBundle); + return Convert.ToHexString(SHA256.HashData(bytes)); + } +} diff --git a/SecondDimensionWatcherReDive/Controllers/External/AppJsonSerializerContext.cs b/SecondDimensionWatcherReDive/Controllers/External/AppJsonSerializerContext.cs index 0f20309..b61931c 100644 --- a/SecondDimensionWatcherReDive/Controllers/External/AppJsonSerializerContext.cs +++ b/SecondDimensionWatcherReDive/Controllers/External/AppJsonSerializerContext.cs @@ -1,4 +1,5 @@ using System.Text.Json.Serialization; +using SecondDimensionWatcherReDive.Framework.DataRepository; namespace SecondDimensionWatcherReDive.Controllers.External; @@ -65,4 +66,8 @@ namespace SecondDimensionWatcherReDive.Controllers.External; [JsonSerializable(typeof(QueueMediaLibraryScanResponse))] [JsonSerializable(typeof(ApplicationSettingsResponse))] [JsonSerializable(typeof(PatchApplicationSettingsRequest))] +[JsonSerializable(typeof(LogicalDataBundle))] +[JsonSerializable(typeof(LogicalDataExportEnvelope))] +[JsonSerializable(typeof(LogicalDataImportRequest))] +[JsonSerializable(typeof(LogicalImportResult))] internal partial class AppJsonSerializerContext : JsonSerializerContext; diff --git a/SecondDimensionWatcherReDive/Controllers/External/LogicalDataTransfer.cs b/SecondDimensionWatcherReDive/Controllers/External/LogicalDataTransfer.cs new file mode 100644 index 0000000..00e8f5f --- /dev/null +++ b/SecondDimensionWatcherReDive/Controllers/External/LogicalDataTransfer.cs @@ -0,0 +1,13 @@ +using System.ComponentModel.DataAnnotations; +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Controllers.External; + +internal sealed record LogicalDataExportEnvelope( + LogicalDataBundle Data, + string Sha256); + +internal sealed record LogicalDataImportRequest( + [property: Required] LogicalDataBundle? Data, + [property: Required] string? Sha256, + [property: Required] LogicalImportConflictStrategy? ConflictStrategy); diff --git a/SecondDimensionWatcherReDive/Controllers/LogicalDataTransferController.cs b/SecondDimensionWatcherReDive/Controllers/LogicalDataTransferController.cs new file mode 100644 index 0000000..3a6f7fc --- /dev/null +++ b/SecondDimensionWatcherReDive/Controllers/LogicalDataTransferController.cs @@ -0,0 +1,221 @@ +using System.Globalization; +using System.Reflection; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using Microsoft.AspNetCore.Authentication.JwtBearer; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Framework.Inference; + +namespace SecondDimensionWatcherReDive.Controllers; + +[ApiController] +[Route("api/data-transfer")] +[Authorize(AuthenticationSchemes = JwtBearerDefaults.AuthenticationScheme)] +internal sealed class LogicalDataTransferController( + ILogicalDataTransferRepository repository) : ControllerBase +{ + private const int SupportedFormatVersion = 1; + private const int MaximumItemsPerCategory = 10_000; + private static readonly Guid CurrentUserId = Guid.Empty; + private static readonly string ApplicationVersion = + typeof(LogicalDataTransferController).Assembly + .GetCustomAttribute()?.InformationalVersion + ?? typeof(LogicalDataTransferController).Assembly.GetName().Version?.ToString() + ?? "0.0.0"; + + [HttpGet("export")] + public async Task ExportAsync( + [FromQuery] string categories = "all", + CancellationToken cancellationToken = default) + { + if (!TryParseCategories(categories, out var selected)) + return BadRequest(new { error = "Unknown data category." }); + + var bundle = await repository.ExportAsync( + selected, + CurrentUserId, + ApplicationVersion, + cancellationToken); + var envelope = new External.LogicalDataExportEnvelope(bundle, Digest(bundle)); + var bytes = JsonSerializer.SerializeToUtf8Bytes( + envelope, + External.AppJsonSerializerContext.Default.LogicalDataExportEnvelope); + var timestamp = bundle.ExportedAtUtc.UtcDateTime.ToString("yyyyMMddTHHmmssZ", CultureInfo.InvariantCulture); + Response.Headers.CacheControl = "private,no-store"; + return File(bytes, "application/json", $"sdw-logical-export-{timestamp}.json"); + } + + [HttpPost("import")] + [RequestSizeLimit(10 * 1024 * 1024)] + public async Task ImportAsync( + [FromBody] External.LogicalDataImportRequest request, + CancellationToken cancellationToken) + { + if (request.Data is null || request.ConflictStrategy is null || + string.IsNullOrWhiteSpace(request.Sha256)) + return BadRequest(new { error = "Data, sha256 and conflictStrategy are required." }); + if (!Enum.IsDefined(request.ConflictStrategy.Value)) + return BadRequest(new { error = "Unknown import conflict strategy." }); + + var actualDigest = Encoding.ASCII.GetBytes(Digest(request.Data)); + var expectedDigest = Encoding.ASCII.GetBytes(request.Sha256.Trim().ToUpperInvariant()); + if (actualDigest.Length != expectedDigest.Length || + !CryptographicOperations.FixedTimeEquals(actualDigest, expectedDigest)) + return BadRequest(new { error = "Logical export checksum mismatch." }); + if (!IsCompatible(request.Data, out var error)) + return BadRequest(new { error }); + + try + { + var result = await repository.ImportAsync( + request.Data, + request.ConflictStrategy.Value, + CurrentUserId, + cancellationToken); + return Ok(result); + } + catch (LogicalDataImportConflictException exception) + { + return Conflict(new { error = exception.Message }); + } + } + + private static bool TryParseCategories(string value, out LogicalDataCategory categories) + { + categories = LogicalDataCategory.None; + foreach (var token in value.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries)) + { + var category = token.ToLowerInvariant() switch + { + "all" => LogicalDataCategory.All, + "feeds" => LogicalDataCategory.Feeds, + "automation" or "automation-policies" => LogicalDataCategory.AutomationPolicies, + "rules" or "filename-rules" => LogicalDataCategory.FileNameRules, + "metadata" or "metadata-corrections" => LogicalDataCategory.MetadataCorrections, + "playback" => LogicalDataCategory.Playback, + _ => LogicalDataCategory.None + }; + if (category == LogicalDataCategory.None) + return false; + categories |= category; + } + return categories != LogicalDataCategory.None; + } + + private static bool IsCompatible(LogicalDataBundle bundle, out string error) + { + if (string.IsNullOrWhiteSpace(bundle.ApplicationVersion) || + bundle.Feeds is null || bundle.AutomationPolicies is null || + bundle.FileNameRules is null || bundle.MetadataCorrections is null || + bundle.PlaybackProgress is null) + { + error = "Logical export is incomplete."; + return false; + } + if (bundle.FormatVersion != SupportedFormatVersion) + { + error = $"Unsupported logical export format {bundle.FormatVersion}."; + return false; + } + if (bundle.Categories == LogicalDataCategory.None || + (bundle.Categories & ~LogicalDataCategory.All) != 0) + { + error = "Logical export contains unknown categories."; + return false; + } + var importedMajor = Major(bundle.ApplicationVersion); + var currentMajor = Major(ApplicationVersion); + if (importedMajor < 0 || currentMajor < 0 || importedMajor != currentMajor) + { + error = "Logical export was created by an incompatible application major version."; + return false; + } + if (bundle.Feeds.Count > MaximumItemsPerCategory || + bundle.AutomationPolicies.Count > MaximumItemsPerCategory || + bundle.FileNameRules.Count > MaximumItemsPerCategory || + bundle.MetadataCorrections.Count > MaximumItemsPerCategory || + bundle.PlaybackProgress.Count > MaximumItemsPerCategory) + { + error = $"A logical export category exceeds {MaximumItemsPerCategory} items."; + return false; + } + if ((!bundle.Categories.HasFlag(LogicalDataCategory.Feeds) && bundle.Feeds.Count > 0) || + (!bundle.Categories.HasFlag(LogicalDataCategory.AutomationPolicies) && bundle.AutomationPolicies.Count > 0) || + (!bundle.Categories.HasFlag(LogicalDataCategory.FileNameRules) && bundle.FileNameRules.Count > 0) || + (!bundle.Categories.HasFlag(LogicalDataCategory.MetadataCorrections) && bundle.MetadataCorrections.Count > 0) || + (!bundle.Categories.HasFlag(LogicalDataCategory.Playback) && + (bundle.PlaybackProgress.Count > 0 || bundle.PlaybackPreferences is not null))) + { + error = "Logical export data does not match its declared categories."; + return false; + } + if (bundle.PlaybackProgress.Any(item => + item.PositionSeconds < 0 || item.DurationSeconds < 0 || + !double.IsFinite(item.PositionSeconds) || !double.IsFinite(item.DurationSeconds))) + { + error = "Logical export contains invalid playback values."; + return false; + } + if (bundle.Feeds.Any(item => item.Id == Guid.Empty || !IsSafeHttpUrl(item.Url)) || + bundle.AutomationPolicies.Any(item => + !IsSafeHttpUrl(item.FeedUrl) || + item.SubtitleGroups is null || item.Resolutions is null || item.Codecs is null || + item.Languages is null || item.ExcludedKeywords is null || + !Enum.IsDefined(item.Mode) || item.MinSizeBytes < 0 || item.MaxSizeBytes < 0 || + (item.MinSizeBytes is not null && item.MaxSizeBytes is not null && + item.MinSizeBytes > item.MaxSizeBytes)) || + bundle.FileNameRules.Any(item => + item.Id == Guid.Empty || + string.IsNullOrWhiteSpace(item.AnimationTmdbId) || + string.IsNullOrWhiteSpace(item.AnimationName) || + string.IsNullOrWhiteSpace(item.AnimationOriginalName) || + !FileNameRegexMatcher.TryCreateRegex(item.Pattern, out _, out _)) || + bundle.FileNameRules + .GroupBy(item => item.AnimationTmdbId, StringComparer.Ordinal) + .Any(group => group.Count() > FileNameRegexMatcher.MaxRulesPerAnimation) || + bundle.MetadataCorrections.Any(item => + item.OperationId == Guid.Empty || + string.IsNullOrWhiteSpace(item.ReleaseDownloadUrl) || + string.IsNullOrWhiteSpace(item.ReleaseTitle) || + string.IsNullOrWhiteSpace(item.AnimationTmdbId) || + string.IsNullOrWhiteSpace(item.AnimationName) || + string.IsNullOrWhiteSpace(item.AnimationOriginalName) || + item.Description is null || item.Season < 0 || item.Episode < 0) || + bundle.PlaybackProgress.Any(item => + string.IsNullOrWhiteSpace(item.VirtualPath) || + !item.VirtualPath.StartsWith("/", StringComparison.Ordinal)) || + bundle.PlaybackPreferences is { } preferences && + (preferences.SubtitleLanguage?.Length > 64 || + preferences.AudioLanguage?.Length > 64 || + preferences.SubtitleTrackLabel?.Length > 128 || + preferences.AudioTrackLabel?.Length > 128)) + { + error = "Logical export contains invalid identifiers or paths."; + return false; + } + error = string.Empty; + return true; + } + + private static bool IsSafeHttpUrl(string? value) => + Uri.TryCreate(value, UriKind.Absolute, out var uri) && + (uri.Scheme == Uri.UriSchemeHttp || uri.Scheme == Uri.UriSchemeHttps) && + string.IsNullOrEmpty(uri.UserInfo); + + private static int Major(string version) + { + var value = version.Split('+', 2)[0].Split('-', 2)[0]; + return Version.TryParse(value, out var parsed) ? parsed.Major : -1; + } + + private static string Digest(LogicalDataBundle bundle) + { + var bytes = JsonSerializer.SerializeToUtf8Bytes( + bundle, + External.AppJsonSerializerContext.Default.LogicalDataBundle); + return Convert.ToHexString(SHA256.HashData(bytes)); + } +} diff --git a/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs b/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs index 8126b9e..7364757 100644 --- a/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs +++ b/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs @@ -674,7 +674,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) modelBuilder.Entity("SecondDimensionWatcherReDive.Models.PlaybackPreference", b => { b.Property("UserId") - .ValueGeneratedOnAdd() .HasColumnType("uuid"); b.Property("AudioLanguage") diff --git a/SecondDimensionWatcherReDive/Models/ApplicationContext.cs b/SecondDimensionWatcherReDive/Models/ApplicationContext.cs index 59764ac..4e0ac4f 100644 --- a/SecondDimensionWatcherReDive/Models/ApplicationContext.cs +++ b/SecondDimensionWatcherReDive/Models/ApplicationContext.cs @@ -207,6 +207,10 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) modelBuilder.Entity() .HasKey(preference => preference.UserId); + modelBuilder.Entity() + .Property(preference => preference.UserId) + .ValueGeneratedNever(); + modelBuilder.Entity() .Property(preference => preference.SubtitleLanguage) .HasMaxLength(64); diff --git a/SecondDimensionWatcherReDive/Program.cs b/SecondDimensionWatcherReDive/Program.cs index 80f5f19..00c04c7 100644 --- a/SecondDimensionWatcherReDive/Program.cs +++ b/SecondDimensionWatcherReDive/Program.cs @@ -277,6 +277,7 @@ builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); +builder.Services.AddScoped(); builder.Services.AddSingleton(); builder.Services.AddScoped(); builder.Services.AddScoped(); diff --git a/SecondDimensionWatcherReDive/Repositories/LogicalDataTransferRepository.cs b/SecondDimensionWatcherReDive/Repositories/LogicalDataTransferRepository.cs new file mode 100644 index 0000000..d9ef62b --- /dev/null +++ b/SecondDimensionWatcherReDive/Repositories/LogicalDataTransferRepository.cs @@ -0,0 +1,585 @@ +using Microsoft.EntityFrameworkCore; +using System.Security.Cryptography; +using System.Text; +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Repositories; + +public sealed class LogicalDataTransferRepository(Models.ApplicationContext context) + : ILogicalDataTransferRepository +{ + private const int FormatVersion = 1; + + public async Task ExportAsync( + LogicalDataCategory categories, + Guid userId, + string applicationVersion, + CancellationToken cancellationToken) + { + var feeds = categories.HasFlag(LogicalDataCategory.Feeds) + ? await context.Feeds.AsNoTracking() + .OrderBy(feed => feed.CreatedAt) + .Select(feed => new LogicalFeed(feed.Id, feed.Url, feed.Name, feed.CreatedAt)) + .ToListAsync(cancellationToken) + : []; + + var policies = categories.HasFlag(LogicalDataCategory.AutomationPolicies) + ? await context.SubscriptionAutomationPolicies.AsNoTracking() + .OrderBy(policy => policy.Feed.Url) + .Select(policy => new LogicalAutomationPolicy( + policy.Feed.Url, + policy.SubtitleGroups, + policy.Resolutions, + policy.Codecs, + policy.Languages, + policy.MinSizeBytes, + policy.MaxSizeBytes, + policy.ExcludedKeywords, + policy.Mode, + policy.CreatedAt, + policy.UpdatedAt)) + .ToListAsync(cancellationToken) + : []; + + var rules = categories.HasFlag(LogicalDataCategory.FileNameRules) + ? await (from rule in context.FileNameRegexRules.AsNoTracking() + join animation in context.Animations.AsNoTracking() + on rule.AnimationId equals animation.Id + orderby animation.TmdbId, rule.CreatedAt + select new LogicalFileNameRule( + rule.Id, + animation.TmdbId, + animation.Name, + animation.OriginalName, + animation.PosterPath, + rule.Pattern, + rule.Description, + rule.CreatedAt)) + .ToListAsync(cancellationToken) + : []; + + var corrections = categories.HasFlag(LogicalDataCategory.MetadataCorrections) + ? await context.MetadataReviewOperations.AsNoTracking() + .Where(operation => operation.State == MetadataReviewOperationState.Applied + && operation.AppliedAt != null + && operation.AnimationInfo.CurrentMetadataReviewOperationId == operation.Id) + .OrderBy(operation => operation.AppliedAt) + .Select(operation => new LogicalMetadataCorrection( + operation.Id, + operation.AnimationInfo.DownloadUrl, + operation.AnimationInfo.Title, + operation.AnimationInfo.PublishTime, + operation.ProposedAnimationTmdbId, + operation.ProposedAnimationName, + operation.ProposedAnimationOriginalName, + operation.ProposedAnimationPosterPath, + operation.ProposedDescription, + operation.ProposedSeason, + operation.ProposedEpisode, + operation.ProposedGroupName, + operation.AppliedAt!.Value)) + .ToListAsync(cancellationToken) + : []; + + var progress = categories.HasFlag(LogicalDataCategory.Playback) + ? await context.PlaybackProgresses.AsNoTracking() + .Where(item => item.UserId == userId) + .OrderBy(item => item.VirtualPath) + .Select(item => new LogicalPlaybackProgress( + item.VirtualPath, + item.PositionSeconds, + item.DurationSeconds, + item.IsWatched, + item.UpdatedAt, + item.WatchedAt)) + .ToListAsync(cancellationToken) + : []; + + LogicalPlaybackPreferences? preferences = null; + if (categories.HasFlag(LogicalDataCategory.Playback)) + { + preferences = await context.PlaybackPreferences.AsNoTracking() + .Where(item => item.UserId == userId) + .Select(item => new LogicalPlaybackPreferences( + item.SubtitleLanguage, + item.SubtitleTrackLabel, + item.AudioLanguage, + item.AudioTrackLabel, + item.AutoPlayNext, + item.UpdatedAt)) + .FirstOrDefaultAsync(cancellationToken); + } + + return new LogicalDataBundle( + FormatVersion, + DateTimeOffset.UtcNow, + applicationVersion, + categories, + feeds, + policies, + rules, + corrections, + progress, + preferences); + } + + public async Task ImportAsync( + LogicalDataBundle bundle, + LogicalImportConflictStrategy conflictStrategy, + Guid userId, + CancellationToken cancellationToken) + { + if (bundle.FormatVersion != FormatVersion) + throw new ArgumentException($"Unsupported logical data format {bundle.FormatVersion}.", nameof(bundle)); + + var statistics = new ImportStatistics(); + await using var transaction = await context.Database.BeginTransactionAsync(cancellationToken); + + var feedsByUrl = await context.Feeds + .ToDictionaryAsync(feed => feed.Url, StringComparer.Ordinal, cancellationToken); + var usedFeedIds = await context.Feeds.AsNoTracking() + .Select(feed => feed.Id) + .ToHashSetAsync(cancellationToken); + await ImportFeedsAsync(bundle, conflictStrategy, feedsByUrl, usedFeedIds, statistics, cancellationToken); + await ImportPoliciesAsync(bundle, conflictStrategy, feedsByUrl, statistics, cancellationToken); + await ImportRulesAsync(bundle, conflictStrategy, statistics, cancellationToken); + await ImportMetadataCorrectionsAsync(bundle, conflictStrategy, statistics, cancellationToken); + await ImportPlaybackAsync(bundle, conflictStrategy, userId, statistics, cancellationToken); + + await context.SaveChangesAsync(cancellationToken); + await transaction.CommitAsync(cancellationToken); + return statistics.ToResult(); + } + + private async Task ImportFeedsAsync( + LogicalDataBundle bundle, + LogicalImportConflictStrategy strategy, + IDictionary feedsByUrl, + ISet usedIds, + ImportStatistics statistics, + CancellationToken cancellationToken) + { + if (!bundle.Categories.HasFlag(LogicalDataCategory.Feeds)) + return; + + foreach (var imported in bundle.Feeds) + { + cancellationToken.ThrowIfCancellationRequested(); + if (feedsByUrl.TryGetValue(imported.Url, out var existing)) + { + if (existing.Name == imported.Name) + { + statistics.Skip(); + continue; + } + if (!HandleConflict(strategy, Identifier("feed", imported.Url), statistics)) + continue; + existing.Name = imported.Name; + statistics.Update(); + continue; + } + + var id = usedIds.Add(imported.Id) ? imported.Id : Guid.NewGuid(); + var entity = new Models.Feed + { + Id = id, + Url = imported.Url, + Name = imported.Name, + CreatedAt = imported.CreatedAt + }; + context.Feeds.Add(entity); + feedsByUrl.Add(entity.Url, entity); + statistics.Add(); + } + await Task.CompletedTask; + } + + private async Task ImportPoliciesAsync( + LogicalDataBundle bundle, + LogicalImportConflictStrategy strategy, + IReadOnlyDictionary feedsByUrl, + ImportStatistics statistics, + CancellationToken cancellationToken) + { + if (!bundle.Categories.HasFlag(LogicalDataCategory.AutomationPolicies)) + return; + + var existing = await context.SubscriptionAutomationPolicies + .ToDictionaryAsync(policy => policy.FeedId, cancellationToken); + foreach (var imported in bundle.AutomationPolicies) + { + if (!feedsByUrl.TryGetValue(imported.FeedUrl, out var feed)) + { + statistics.Skip($"policy feed is missing:{Identifier("feed", imported.FeedUrl)}"); + continue; + } + + if (existing.TryGetValue(feed.Id, out var entity)) + { + if (!HandleConflict(strategy, Identifier("policy", imported.FeedUrl), statistics)) + continue; + ApplyPolicy(imported, entity); + statistics.Update(); + continue; + } + + entity = new Models.SubscriptionAutomationPolicy { FeedId = feed.Id, Feed = feed }; + ApplyPolicy(imported, entity); + context.SubscriptionAutomationPolicies.Add(entity); + existing.Add(feed.Id, entity); + statistics.Add(); + } + } + + private async Task ImportRulesAsync( + LogicalDataBundle bundle, + LogicalImportConflictStrategy strategy, + ImportStatistics statistics, + CancellationToken cancellationToken) + { + if (!bundle.Categories.HasFlag(LogicalDataCategory.FileNameRules)) + return; + + var animations = await context.Animations + .ToDictionaryAsync(animation => animation.TmdbId, StringComparer.Ordinal, cancellationToken); + var rules = await context.FileNameRegexRules.ToListAsync(cancellationToken); + var ruleByKey = rules.ToDictionary(rule => (rule.AnimationId, rule.Pattern)); + var usedRuleIds = rules.Select(rule => rule.Id).ToHashSet(); + + foreach (var imported in bundle.FileNameRules) + { + if (!animations.TryGetValue(imported.AnimationTmdbId, out var animation)) + { + animation = new Models.Animation + { + Id = Guid.NewGuid(), + TmdbId = imported.AnimationTmdbId, + Name = imported.AnimationName, + OriginalName = imported.AnimationOriginalName, + PosterPath = imported.AnimationPosterPath + }; + context.Animations.Add(animation); + animations.Add(animation.TmdbId, animation); + } + + if (ruleByKey.TryGetValue((animation.Id, imported.Pattern), out var existing)) + { + if (existing.Description == imported.Description) + { + statistics.Skip(); + continue; + } + if (!HandleConflict(strategy, + $"filename-rule:{imported.AnimationTmdbId}:{imported.Pattern}", statistics)) + continue; + existing.Description = imported.Description; + statistics.Update(); + continue; + } + + var id = usedRuleIds.Add(imported.Id) ? imported.Id : Guid.NewGuid(); + var entity = new Models.FileNameRegexRule + { + Id = id, + AnimationId = animation.Id, + Pattern = imported.Pattern, + Description = imported.Description, + CreatedAt = imported.CreatedAt + }; + context.FileNameRegexRules.Add(entity); + ruleByKey.Add((animation.Id, entity.Pattern), entity); + statistics.Add(); + } + } + + private async Task ImportMetadataCorrectionsAsync( + LogicalDataBundle bundle, + LogicalImportConflictStrategy strategy, + ImportStatistics statistics, + CancellationToken cancellationToken) + { + if (!bundle.Categories.HasFlag(LogicalDataCategory.MetadataCorrections) || + bundle.MetadataCorrections.Count == 0) + return; + + var downloadUrls = bundle.MetadataCorrections.Select(item => item.ReleaseDownloadUrl).Distinct().ToArray(); + var candidates = await context.AnimationInfo + .Include(info => info.Animation) + .Include(info => info.Group) + .Where(info => downloadUrls.Contains(info.DownloadUrl)) + .ToListAsync(cancellationToken); + var byKey = candidates + .GroupBy(info => (info.DownloadUrl, info.Title, info.PublishTime.ToUnixTimeSeconds())) + .ToDictionary(group => group.Key, group => group.ToList()); + var operationIds = bundle.MetadataCorrections.Select(item => item.OperationId).ToArray(); + var existingOperationIds = await context.MetadataReviewOperations.AsNoTracking() + .Where(operation => operationIds.Contains(operation.Id)) + .Select(operation => operation.Id) + .ToHashSetAsync(cancellationToken); + var animations = await context.Animations + .ToDictionaryAsync(animation => animation.TmdbId, StringComparer.Ordinal, cancellationToken); + var groups = await context.AnimationGroups + .ToDictionaryAsync(group => group.Name, StringComparer.Ordinal, cancellationToken); + + foreach (var imported in bundle.MetadataCorrections) + { + if (existingOperationIds.Contains(imported.OperationId)) + { + statistics.Skip(); + continue; + } + if (!byKey.TryGetValue( + (imported.ReleaseDownloadUrl, imported.ReleaseTitle, + imported.ReleasePublishTime.ToUnixTimeSeconds()), + out var matches) || matches.Count != 1) + { + statistics.Skip($"metadata release is missing or ambiguous:{imported.ReleaseTitle}"); + continue; + } + + var info = matches[0]; + if (info.CurrentMetadataReviewOperationId is not null && + !HandleConflict(strategy, $"metadata:{imported.ReleaseTitle}", statistics)) + continue; + + if (!animations.TryGetValue(imported.AnimationTmdbId, out var animation)) + { + animation = new Models.Animation + { + Id = Guid.NewGuid(), + TmdbId = imported.AnimationTmdbId + }; + context.Animations.Add(animation); + animations.Add(animation.TmdbId, animation); + } + animation.Name = imported.AnimationName; + animation.OriginalName = imported.AnimationOriginalName; + animation.PosterPath = imported.AnimationPosterPath; + + Models.AnimationGroup? group = null; + if (!string.IsNullOrWhiteSpace(imported.GroupName) && + !groups.TryGetValue(imported.GroupName, out group)) + { + group = new Models.AnimationGroup { Id = Guid.NewGuid(), Name = imported.GroupName }; + context.AnimationGroups.Add(group); + groups.Add(group.Name, group); + } + + var nextVersion = checked(info.StateVersion + 1); + var operation = new Models.MetadataReviewOperation + { + Id = imported.OperationId, + AnimationInfoId = info.Id, + AnimationInfo = info, + State = MetadataReviewOperationState.Applied, + CreatedAt = imported.AppliedAt, + ExpiresAt = imported.AppliedAt.AddDays(1), + BaseVersion = info.StateVersion, + BaseFileStore = info.FileStore, + BaseStorePath = info.StorePath, + BaseIsDownloadFinished = info.IsDownloadFinished, + ProposedAnimationTmdbId = imported.AnimationTmdbId, + ProposedAnimationName = imported.AnimationName, + ProposedAnimationOriginalName = imported.AnimationOriginalName, + ProposedAnimationPosterPath = imported.AnimationPosterPath, + ProposedDescription = imported.Description, + ProposedSeason = imported.Season, + ProposedEpisode = imported.Episode, + ProposedGroupName = imported.GroupName, + AppliedAt = imported.AppliedAt, + AppliedVersion = nextVersion, + PreviousDescription = info.Description, + PreviousAnimationId = info.Animation?.Id, + PreviousGroupId = info.Group?.Id, + PreviousSeason = info.Season, + PreviousEpisode = info.Episode, + PreviousMetadataStatus = info.MetadataStatus, + PreviousConfidence = info.MetadataConfidence, + PreviousLastError = info.MetadataLastError, + PreviousIsAiProcessed = info.IsAiProcessed, + PreviousAiRetryCount = info.AiRetryCount, + PreviousReviewedAt = info.MetadataReviewedAt, + PreviousCurrentOperationId = info.CurrentMetadataReviewOperationId + }; + context.MetadataReviewOperations.Add(operation); + info.Animation = animation; + info.Group = group; + info.Description = imported.Description; + info.Season = imported.Season; + info.Episode = imported.Episode; + info.MetadataStatus = MetadataReviewStatus.Reviewed; + info.MetadataConfidence = 1; + info.MetadataLastError = null; + info.MetadataReviewedAt = imported.AppliedAt; + info.IsAiProcessed = true; + info.AiRetryCount = 0; + info.StateVersion = nextVersion; + info.CurrentMetadataReviewOperationId = operation.Id; + existingOperationIds.Add(operation.Id); + statistics.Add(); + } + } + + private async Task ImportPlaybackAsync( + LogicalDataBundle bundle, + LogicalImportConflictStrategy strategy, + Guid userId, + ImportStatistics statistics, + CancellationToken cancellationToken) + { + if (!bundle.Categories.HasFlag(LogicalDataCategory.Playback)) + return; + + var paths = bundle.PlaybackProgress.Select(item => item.VirtualPath).Distinct().ToArray(); + var mappings = await context.FileMappings.AsNoTracking() + .Where(mapping => paths.Contains(mapping.VirtualPath)) + .ToDictionaryAsync(mapping => mapping.VirtualPath, StringComparer.Ordinal, cancellationToken); + var existing = await context.PlaybackProgresses + .Where(item => item.UserId == userId && paths.Contains(item.VirtualPath)) + .ToDictionaryAsync(item => item.VirtualPath, StringComparer.Ordinal, cancellationToken); + + foreach (var imported in bundle.PlaybackProgress + .GroupBy(item => item.VirtualPath, StringComparer.Ordinal) + .Select(group => group.OrderByDescending(item => item.UpdatedAt).First())) + { + if (!mappings.TryGetValue(imported.VirtualPath, out var mapping)) + { + statistics.Skip($"playback path is missing:{imported.VirtualPath}"); + continue; + } + if (existing.TryGetValue(imported.VirtualPath, out var progress)) + { + if (!HandleConflict(strategy, $"playback:{imported.VirtualPath}", statistics)) + continue; + ApplyProgress(imported, mapping.AnimationInfoId, userId, progress); + statistics.Update(); + continue; + } + + progress = new Models.PlaybackProgress { Id = Guid.NewGuid() }; + ApplyProgress(imported, mapping.AnimationInfoId, userId, progress); + context.PlaybackProgresses.Add(progress); + existing.Add(progress.VirtualPath, progress); + statistics.Add(); + } + + if (bundle.PlaybackPreferences is not { } importedPreferences) + return; + var preferences = await context.PlaybackPreferences + .FirstOrDefaultAsync(item => item.UserId == userId, cancellationToken); + if (preferences is not null) + { + if (!HandleConflict(strategy, "playback-preferences", statistics)) + return; + ApplyPreferences(importedPreferences, userId, preferences); + statistics.Update(); + return; + } + + preferences = new Models.PlaybackPreference(); + ApplyPreferences(importedPreferences, userId, preferences); + context.PlaybackPreferences.Add(preferences); + statistics.Add(); + } + + private static bool HandleConflict( + LogicalImportConflictStrategy strategy, + string identifier, + ImportStatistics statistics) + { + if (strategy == LogicalImportConflictStrategy.Fail) + throw new LogicalDataImportConflictException($"Import conflict at {identifier}."); + if (strategy == LogicalImportConflictStrategy.Skip) + { + statistics.Conflict(identifier); + return false; + } + return true; + } + + private static string Identifier(string kind, string value) + { + var digest = SHA256.HashData(Encoding.UTF8.GetBytes(value)); + return $"{kind}:{Convert.ToHexString(digest.AsSpan(0, 8))}"; + } + + private static void ApplyPolicy( + LogicalAutomationPolicy source, + Models.SubscriptionAutomationPolicy target) + { + target.SubtitleGroups = source.SubtitleGroups.ToArray(); + target.Resolutions = source.Resolutions.ToArray(); + target.Codecs = source.Codecs.ToArray(); + target.Languages = source.Languages.ToArray(); + target.MinSizeBytes = source.MinSizeBytes; + target.MaxSizeBytes = source.MaxSizeBytes; + target.ExcludedKeywords = source.ExcludedKeywords.ToArray(); + target.Mode = source.Mode; + target.CreatedAt = source.CreatedAt; + target.UpdatedAt = source.UpdatedAt; + } + + private static void ApplyProgress( + LogicalPlaybackProgress source, + Guid animationInfoId, + Guid userId, + Models.PlaybackProgress target) + { + target.UserId = userId; + target.AnimationInfoId = animationInfoId; + target.VirtualPath = source.VirtualPath; + target.PositionSeconds = source.PositionSeconds; + target.DurationSeconds = source.DurationSeconds; + target.IsWatched = source.IsWatched; + target.UpdatedAt = source.UpdatedAt; + target.WatchedAt = source.WatchedAt; + } + + private static void ApplyPreferences( + LogicalPlaybackPreferences source, + Guid userId, + Models.PlaybackPreference target) + { + target.UserId = userId; + target.SubtitleLanguage = source.SubtitleLanguage; + target.SubtitleTrackLabel = source.SubtitleTrackLabel; + target.AudioLanguage = source.AudioLanguage; + target.AudioTrackLabel = source.AudioTrackLabel; + target.AutoPlayNext = source.AutoPlayNext; + target.UpdatedAt = source.UpdatedAt; + } + + private sealed class ImportStatistics + { + private readonly List _messages = []; + + public int Added { get; private set; } + public int Updated { get; private set; } + public int Skipped { get; private set; } + public int Conflicts { get; private set; } + + public void Add() => Added++; + public void Update() => Updated++; + public void Skip(string? message = null) + { + Skipped++; + Message(message); + } + + public void Conflict(string identifier) + { + Conflicts++; + Skipped++; + Message($"conflict skipped:{identifier}"); + } + + public LogicalImportResult ToResult() => + new(Added, Updated, Skipped, Conflicts, _messages); + + private void Message(string? value) + { + if (!string.IsNullOrEmpty(value) && _messages.Count < 100) + _messages.Add(value); + } + } +} diff --git a/deployments/podman-compose.yml b/deployments/podman-compose.yml index 7a4f463..47944af 100644 --- a/deployments/podman-compose.yml +++ b/deployments/podman-compose.yml @@ -65,6 +65,9 @@ services: volumes: - downloads:/downloads - appdata:/app/data + - ./backups:/app/backups + # The backup CLI archives this deployment configuration without printing it. + - ./podman-compose.yml:/app/deployment/podman-compose.yml:ro # Existing libraries can be imported in place from Settings. Replace the # host path and keep the mount read-only so originals cannot be modified. # - /path/to/anime:/media/anime:ro diff --git a/deployments/sdw-backup b/deployments/sdw-backup new file mode 100755 index 0000000..1670cad --- /dev/null +++ b/deployments/sdw-backup @@ -0,0 +1,605 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +umask 077 + +readonly BACKUP_FORMAT_VERSION=1 +backup_temp_dir="" +backup_partial_archive="" +backup_partial_checksum="" + +cleanup() { + case "${backup_temp_dir}" in + "${TMPDIR:-/tmp}"/sdw-backup.*) + [[ -d "${backup_temp_dir}" ]] && rm -rf -- "${backup_temp_dir}" + ;; + esac + if [[ "${backup_partial_archive}" == *sdw-backup-*.partial ]]; then + rm -f -- "${backup_partial_archive}" + fi + if [[ "${backup_partial_checksum}" == *sdw-backup-*.sha256.partial ]]; then + rm -f -- "${backup_partial_checksum}" + fi +} + +notify_failure() { + local status=$? + if [[ ${status} -ne 0 && -n "${SDW_BACKUP_FAILURE_WEBHOOK:-}" ]] && command -v curl >/dev/null 2>&1; then + curl --silent --show-error --fail --max-time 10 \ + --header 'Content-Type: application/json' \ + --data '{"event":"sdw_backup_failed"}' \ + "${SDW_BACKUP_FAILURE_WEBHOOK}" >/dev/null 2>&1 || true + fi + return "${status}" +} + +trap notify_failure ERR +trap cleanup EXIT + +usage() { + cat <<'EOF' +Usage: + sdw-backup create [options] + sdw-backup list [--output DIR] + sdw-backup verify ARCHIVE [--age-identity FILE] + sdw-backup restore ARCHIVE --confirm-replace [options] + +Create options: + --output DIR Local backup target + --config FILE Deployment config to include + --password-file FILE password.json to include + --key-ring DIR Data Protection key ring to include + --plugin-dir DIR Plugin directory; manifest files are included + --retention-days DAYS Delete completed backups older than DAYS + --app-version VERSION Application version written to non-secret metadata + --age-recipient RECIPIENT Encrypt the final archive with age + +Restore options: + --config-destination FILE + --password-destination FILE + --key-ring-destination DIR + --plugin-destination DIR + --expected-version VERSION + --expected-schema MIGRATION + --age-identity FILE + --safety-directory DIR Write a pre-restore database dump here + +PostgreSQL is read from PGHOST/PGPORT/PGUSER/PGPASSWORD/PGDATABASE. If those +are absent, ConnectionStrings__sdw may contain an ASP.NET semicolon connection +string. Secrets are never printed. +EOF +} + +die() { + printf 'sdw-backup: %s\n' "$1" >&2 + exit 1 +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || die "required command is missing: $1" +} + +new_temp_dir() { + backup_temp_dir=$(mktemp -d "${TMPDIR:-/tmp}/sdw-backup.XXXXXX") +} + +parse_connection_string() { + if [[ -n "${PGHOST:-}" && -n "${PGUSER:-}" && -n "${PGDATABASE:-}" ]]; then + export PGHOST PGUSER PGDATABASE + export PGPORT="${PGPORT:-5432}" + return + fi + + local connection="${ConnectionStrings__sdw:-}" + [[ -n "${connection}" ]] || die "PostgreSQL connection environment is not configured" + local pair key value + local -a pairs + IFS=';' read -r -a pairs <<<"${connection}" + for pair in "${pairs[@]}"; do + key=${pair%%=*} + value=${pair#*=} + key=${key//[[:space:]]/} + case "${key,,}" in + host) PGHOST=${value} ;; + port) PGPORT=${value} ;; + username|userid) PGUSER=${value} ;; + password) PGPASSWORD=${value} ;; + database|initial\ catalog) PGDATABASE=${value} ;; + esac + done + [[ -n "${PGHOST:-}" && -n "${PGUSER:-}" && -n "${PGDATABASE:-}" ]] || + die "the ASP.NET connection string is missing Host, Username, or Database" + export PGHOST PGPORT="${PGPORT:-5432}" PGUSER PGPASSWORD="${PGPASSWORD:-}" PGDATABASE +} + +validate_output_directory() { + local directory=$1 + [[ -n "${directory}" && "${directory}" != "/" ]] || die "refusing unsafe backup output directory" + mkdir -p -- "${directory}" + chmod 0700 "${directory}" +} + +validate_file_destination() { + local destination=$1 + [[ -n "${destination}" && "${destination}" != "/" && + ! -d "${destination}" && ! -L "${destination}" ]] || + die "refusing unsafe restore file destination" +} + +validate_directory_destination() { + local destination=$1 normalized + [[ -n "${destination}" ]] || die "refusing empty restore directory destination" + [[ ! -L "${destination}" ]] || die "refusing symlink restore directory destination" + normalized=$(realpath --canonicalize-missing -- "${destination}") + case "${normalized}" in + /|/etc|/usr|/var|/var/lib|/home|/root|/app) + die "refusing unsafe restore directory destination" + ;; + esac +} + +safe_token() { + [[ "$1" =~ ^[A-Za-z0-9._:+/@-]+$ ]] +} + +database_schema_version() { + local schema + schema=$(psql --no-psqlrc --tuples-only --no-align --command \ + 'SELECT "MigrationId" FROM "__EFMigrationsHistory" ORDER BY "MigrationId" DESC LIMIT 1' \ + 2>/dev/null || true) + schema=${schema//$'\n'/} + if [[ -z "${schema}" ]]; then + printf 'uninitialized' + else + safe_token "${schema}" || die "database returned an unsafe schema version" + printf '%s' "${schema}" + fi +} + +copy_plugin_manifests() { + local plugin_dir=$1 destination=$2 + mkdir -p "${destination}" + if [[ ! -d "${plugin_dir}" ]]; then + : >"${destination}/none" + return + fi + + local source relative target count=0 + while IFS= read -r -d '' source; do + relative=${source#"${plugin_dir}"/} + target="${destination}/${relative}" + mkdir -p -- "$(dirname "${target}")" + install -m 0600 -- "${source}" "${target}" + count=$((count + 1)) + done < <(find "${plugin_dir}" -type f \( -name 'plugin.json' -o -name 'manifest.json' \) -print0) + if [[ ${count} -eq 0 ]]; then + : >"${destination}/none" + fi +} + +validate_stage_paths() { + local stage=$1 relative + if find "${stage}" -mindepth 1 ! -type f ! -type d -print -quit | grep -q .; then + die "backup source contains a link or special file" + fi + while IFS= read -r -d '' relative; do + relative=${relative#"${stage}"/} + [[ "${relative}" =~ ^[A-Za-z0-9._+@:/{}-]+$ ]] || + die "backup source contains an unsupported file name" + [[ "${relative}" != */../* && "${relative}" != ../* ]] || + die "backup source contains an unsafe path" + done < <(find "${stage}" -mindepth 1 -print0) +} + +create_backup() { + local output="${SDW_BACKUP_DIRECTORY:-/var/lib/sdw-redive/backups}" + local config="${SDW_CONFIG_PATH:-/etc/sdw-redive/appsettings.yml}" + local password_file="${SDW_PASSWORD_FILE:-/var/lib/sdw-redive/password.json}" + local key_ring="${SDW_KEY_RING_PATH:-/var/lib/sdw-redive/data-protection-keys}" + local plugin_dir="${SDW_PLUGIN_DIRECTORY:-/var/lib/sdw-redive/plugins}" + local retention_days="${SDW_BACKUP_RETENTION_DAYS:-14}" + local app_version="${SDW_APP_VERSION:-}" + local age_recipient="${SDW_BACKUP_AGE_RECIPIENT:-}" + + while [[ $# -gt 0 ]]; do + case "$1" in + --output) output=$2; shift 2 ;; + --config) config=$2; shift 2 ;; + --password-file) password_file=$2; shift 2 ;; + --key-ring) key_ring=$2; shift 2 ;; + --plugin-dir) plugin_dir=$2; shift 2 ;; + --retention-days) retention_days=$2; shift 2 ;; + --app-version) app_version=$2; shift 2 ;; + --age-recipient) age_recipient=$2; shift 2 ;; + *) die "unknown create option: $1" ;; + esac + done + + if [[ -z "${app_version}" && -f /usr/lib/sdw-redive/VERSION ]]; then + app_version=$(tr -d '[:space:]' /dev/null + local created schema stage_bytes required_bytes + created=$(date -u +%Y-%m-%dT%H:%M:%SZ) + schema=$(database_schema_version) + stage_bytes=$(du -sb "${stage}" | awk '{print $1}') + required_bytes=$((stage_bytes * 2 + 67108864)) + { + printf 'format_version=%s\n' "${BACKUP_FORMAT_VERSION}" + printf 'created_utc=%s\n' "${created}" + printf 'application_version=%s\n' "${app_version}" + printf 'schema_version=%s\n' "${schema}" + printf 'required_free_bytes=%s\n' "${required_bytes}" + printf 'media_files_included=false\n' + } >"${stage}/manifest.env" + ( + cd "${stage}" + find . -type f ! -path ./checksums.sha256 -print0 | + LC_ALL=C sort -z | + xargs -0 sha256sum >checksums.sha256 + ) + + local timestamp archive_name plain_archive final_archive + timestamp=$(date -u +%Y%m%dT%H%M%SZ) + archive_name="sdw-backup-${timestamp}-$$.tar.gz" + plain_archive="${backup_temp_dir}/${archive_name}" + tar --create --gzip --file "${plain_archive}" --directory "${stage}" . + gzip --test "${plain_archive}" + if [[ -n "${age_recipient}" ]]; then + require_command age + final_archive="${output}/${archive_name}.age" + else + final_archive="${output}/${archive_name}" + fi + [[ ! -e "${final_archive}" && ! -e "${final_archive}.sha256" ]] || + die "refusing to overwrite an existing backup" + backup_partial_archive="${final_archive}.partial" + backup_partial_checksum="${final_archive}.sha256.partial" + if [[ -n "${age_recipient}" ]]; then + age --recipient "${age_recipient}" --output "${backup_partial_archive}" "${plain_archive}" + else + cp -- "${plain_archive}" "${backup_partial_archive}" + printf 'sdw-backup: warning: archive is not encrypted; protect it as a secret\n' >&2 + fi + chmod 0600 "${backup_partial_archive}" + local archive_digest + archive_digest=$(sha256sum "${backup_partial_archive}" | awk '{print $1}') + printf '%s %s\n' "${archive_digest}" "$(basename "${final_archive}")" \ + >"${backup_partial_checksum}" + chmod 0600 "${backup_partial_checksum}" + mv -- "${backup_partial_archive}" "${final_archive}" + backup_partial_archive="" + mv -- "${backup_partial_checksum}" "${final_archive}.sha256" + backup_partial_checksum="" + + if [[ ${retention_days} -gt 0 ]]; then + find "${output}" -maxdepth 1 -type f \ + \( -name 'sdw-backup-*.tar.gz' -o -name 'sdw-backup-*.tar.gz.age' -o \ + -name 'sdw-backup-*.tar.gz.sha256' -o -name 'sdw-backup-*.tar.gz.age.sha256' \) \ + -mtime "+${retention_days}" -delete + fi + printf '%s\n' "${final_archive}" +} + +list_backups() { + local output="${SDW_BACKUP_DIRECTORY:-/var/lib/sdw-redive/backups}" + while [[ $# -gt 0 ]]; do + case "$1" in + --output) output=$2; shift 2 ;; + *) die "unknown list option: $1" ;; + esac + done + [[ -d "${output}" ]] || return 0 + TZ=UTC find "${output}" -maxdepth 1 -type f \ + \( -name 'sdw-backup-*.tar.gz' -o -name 'sdw-backup-*.tar.gz.age' \) \ + -printf '%TY-%Tm-%TdT%TH:%TM:%TSZ %s %f\n' | LC_ALL=C sort -r +} + +validate_archive_members() { + local archive=$1 + if LC_ALL=C tar --list --gzip --file "${archive}" | + grep -Eq '(^/|(^|/)\.\.(/|$))'; then + die "archive contains an unsafe path" + fi + if LC_ALL=C tar --list --gzip --file "${archive}" | + grep -Eqv '^(\./)?([A-Za-z0-9._+@:{}-]+/)*[A-Za-z0-9._+@:{}-]*/?$'; then + die "archive contains an unsupported path" + fi + if LC_ALL=C tar --list --verbose --gzip --file "${archive}" | + awk 'substr($1, 1, 1) !~ /^[-d]$/ { found=1 } END { exit(found ? 0 : 1) }'; then + die "archive contains a link or special file" + fi +} + +archive_unpacked_bytes() { + LC_ALL=C tar --list --verbose --numeric-owner --full-time --gzip --file "$1" | + awk ' + $3 !~ /^[0-9]+$/ { exit 2 } + { total += $3; if (total > 900000000000000000) exit 2 } + END { if (NR > 0) printf "%.0f\n", total; else exit 2 } + ' +} + +available_bytes() { + df -Pk "$1" | awk 'NR==2 {printf "%.0f\n", $4 * 1024}' +} + +manifest_value() { + local manifest=$1 key=$2 + awk -F= -v wanted="${key}" '$1 == wanted { sub(/^[^=]*=/, ""); print; exit }' "${manifest}" +} + +verify_outer_checksum() { + local archive=$1 sidecar="${1}.sha256" line expected expected_name actual + [[ -f "${sidecar}" ]] || return 0 + IFS= read -r line <"${sidecar}" || die "backup archive checksum is unreadable" + expected=${line:0:64} + expected_name=${line:66} + [[ "${expected}" =~ ^[0-9a-f]{64}$ && "${line:64:2}" == " " && + "${expected_name}" == "$(basename "${archive}")" ]] || + die "backup archive checksum is malformed" + actual=$(sha256sum "${archive}" | awk '{print $1}') + [[ "${actual}" == "${expected}" ]] || die "backup archive checksum verification failed" +} + +validate_checksum_manifest() { + local root=$1 manifest="${1}/checksums.sha256" line digest path + local listed=() + while IFS= read -r line; do + digest=${line:0:64} + path=${line:66} + [[ "${digest}" =~ ^[0-9a-f]{64}$ && "${line:64:2}" == " " ]] || + die "backup checksum manifest is malformed" + [[ "${path}" == ./* && "${path}" != */../* && "${path}" != ../* ]] || + die "backup checksum manifest contains an unsafe path" + [[ "${path}" =~ ^\./[A-Za-z0-9._+@:/{}-]+$ ]] || + die "backup checksum manifest contains an unsupported path" + listed+=("${path}") + done <"${manifest}" + + local actual + actual=$(find "${root}" -type f ! -path "${root}/checksums.sha256" \ + -printf './%P\n' | LC_ALL=C sort) + [[ "$(printf '%s\n' "${listed[@]}" | LC_ALL=C sort)" == "${actual}" ]] || + die "backup checksum manifest does not cover every file" +} + +prepare_archive() { + local archive=$1 identity=$2 + [[ -f "${archive}" ]] || die "backup archive does not exist" + require_command tar + require_command sha256sum + require_command pg_restore + verify_outer_checksum "${archive}" + new_temp_dir + local plain="${archive}" + if [[ "${archive}" == *.age ]]; then + [[ -n "${identity}" ]] || die "an age identity is required for this archive" + require_command age + plain="${backup_temp_dir}/backup.tar.gz" + local encrypted_bytes decrypt_free_bytes + encrypted_bytes=$(wc -c <"${archive}") + decrypt_free_bytes=$(available_bytes "${backup_temp_dir}") + [[ ${decrypt_free_bytes%.*} -gt $((encrypted_bytes * 3 + 67108864)) ]] || + die "insufficient temporary disk space to decrypt backup" + age --decrypt --identity "${identity}" --output "${plain}" "${archive}" + fi + + validate_archive_members "${plain}" + local archive_bytes unpacked_bytes free_bytes + archive_bytes=$(wc -c <"${plain}") + unpacked_bytes=$(archive_unpacked_bytes "${plain}") || die "archive has invalid size metadata" + free_bytes=$(available_bytes "${backup_temp_dir}") + [[ ${free_bytes%.*} -gt $((archive_bytes + unpacked_bytes + 67108864)) ]] || + die "insufficient temporary disk space to validate backup" + mkdir "${backup_temp_dir}/extracted" + tar --extract --gzip --no-same-owner --no-same-permissions \ + --file "${plain}" --directory "${backup_temp_dir}/extracted" + [[ -f "${backup_temp_dir}/extracted/manifest.env" && + -f "${backup_temp_dir}/extracted/checksums.sha256" && + -f "${backup_temp_dir}/extracted/database.dump" ]] || + die "backup is missing required files" + validate_checksum_manifest "${backup_temp_dir}/extracted" + ( + cd "${backup_temp_dir}/extracted" + sha256sum --check --strict checksums.sha256 >/dev/null + ) || die "backup checksum verification failed" + pg_restore --list "${backup_temp_dir}/extracted/database.dump" >/dev/null || + die "PostgreSQL dump verification failed" + local format required created application schema media_scope + format=$(manifest_value "${backup_temp_dir}/extracted/manifest.env" format_version) + [[ "${format}" == "${BACKUP_FORMAT_VERSION}" ]] || die "unsupported backup format" + required=$(manifest_value "${backup_temp_dir}/extracted/manifest.env" required_free_bytes) + created=$(manifest_value "${backup_temp_dir}/extracted/manifest.env" created_utc) + application=$(manifest_value "${backup_temp_dir}/extracted/manifest.env" application_version) + schema=$(manifest_value "${backup_temp_dir}/extracted/manifest.env" schema_version) + media_scope=$(manifest_value "${backup_temp_dir}/extracted/manifest.env" media_files_included) + [[ "${required}" =~ ^[0-9]+$ && ${#required} -le 18 ]] || + die "backup has invalid space metadata" + [[ "${created}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || + die "backup has invalid timestamp metadata" + safe_token "${application}" && safe_token "${schema}" || + die "backup has invalid version metadata" + [[ "${media_scope}" == false ]] || die "backup has an unsupported media scope" + free_bytes=$(available_bytes "${backup_temp_dir}") + [[ ${free_bytes%.*} -gt ${required} ]] || die "insufficient disk space for restore" +} + +verify_backup() { + [[ $# -ge 1 ]] || die "verify requires an archive" + local archive=$1 identity="${SDW_BACKUP_AGE_IDENTITY:-}" + shift + while [[ $# -gt 0 ]]; do + case "$1" in + --age-identity) identity=$2; shift 2 ;; + *) die "unknown verify option: $1" ;; + esac + done + prepare_archive "${archive}" "${identity}" + local manifest="${backup_temp_dir}/extracted/manifest.env" + printf 'verified format=%s created=%s application=%s schema=%s media_files_included=false\n' \ + "$(manifest_value "${manifest}" format_version)" \ + "$(manifest_value "${manifest}" created_utc)" \ + "$(manifest_value "${manifest}" application_version)" \ + "$(manifest_value "${manifest}" schema_version)" +} + +major_version() { + local value=${1%%+*} + value=${value%%-*} + [[ "${value}" =~ ^[0-9]+(\.[0-9]+){0,3}$ ]] || return 1 + printf '%s' "${value%%.*}" +} + +restore_backup() { + [[ $# -ge 1 ]] || die "restore requires an archive" + local archive=$1 + shift + local config_destination="${SDW_CONFIG_PATH:-/etc/sdw-redive/appsettings.yml}" + local password_destination="${SDW_PASSWORD_FILE:-/var/lib/sdw-redive/password.json}" + local key_ring_destination="${SDW_KEY_RING_PATH:-/var/lib/sdw-redive/data-protection-keys}" + local plugin_destination="${SDW_PLUGIN_DIRECTORY:-/var/lib/sdw-redive/plugins}" + local expected_version="${SDW_APP_VERSION:-}" + local expected_schema="${SDW_EXPECTED_SCHEMA_VERSION:-}" + local identity="${SDW_BACKUP_AGE_IDENTITY:-}" + local safety_directory="${SDW_BACKUP_DIRECTORY:-/var/lib/sdw-redive/backups}" + local confirmed=false allow_version_mismatch=false + + while [[ $# -gt 0 ]]; do + case "$1" in + --config-destination) config_destination=$2; shift 2 ;; + --password-destination) password_destination=$2; shift 2 ;; + --key-ring-destination) key_ring_destination=$2; shift 2 ;; + --plugin-destination) plugin_destination=$2; shift 2 ;; + --expected-version) expected_version=$2; shift 2 ;; + --expected-schema) expected_schema=$2; shift 2 ;; + --age-identity) identity=$2; shift 2 ;; + --safety-directory) safety_directory=$2; shift 2 ;; + --allow-version-mismatch) allow_version_mismatch=true; shift ;; + --confirm-replace) confirmed=true; shift ;; + *) die "unknown restore option: $1" ;; + esac + done + [[ "${confirmed}" == true ]] || die "restore requires --confirm-replace" + require_command realpath + validate_file_destination "${config_destination}" + validate_file_destination "${password_destination}" + validate_directory_destination "${key_ring_destination}" + validate_directory_destination "${plugin_destination}" + parse_connection_string + require_command psql + require_command pg_dump + prepare_archive "${archive}" "${identity}" + + local extracted="${backup_temp_dir}/extracted" + local manifest="${extracted}/manifest.env" + local backup_version backup_schema + backup_version=$(manifest_value "${manifest}" application_version) + backup_schema=$(manifest_value "${manifest}" schema_version) + if [[ "${allow_version_mismatch}" != true ]]; then + [[ -n "${expected_version}" ]] || + die "--expected-version is required unless --allow-version-mismatch is explicit" + [[ "$(major_version "${backup_version}")" == "$(major_version "${expected_version}")" ]] || + die "backup application major version is incompatible" + fi + if [[ -n "${expected_schema}" && "${backup_schema}" != "${expected_schema}" ]]; then + die "backup schema is incompatible with the requested schema" + fi + + local restored_config required_bytes destination_parent probe + restored_config=$(find "${extracted}/state/config" -maxdepth 1 -type f -print -quit) + [[ -n "${restored_config}" && -f "${extracted}/state/password.json" && + -d "${extracted}/state/data-protection-keys" ]] || + die "backup state files are incomplete" + required_bytes=$(manifest_value "${manifest}" required_free_bytes) + for destination_parent in \ + "$(dirname "${config_destination}")" \ + "$(dirname "${password_destination}")" \ + "$(dirname "${key_ring_destination}")" \ + "$(dirname "${plugin_destination}")"; do + mkdir -p -- "${destination_parent}" + [[ $(available_bytes "${destination_parent}") -gt ${required_bytes} ]] || + die "insufficient destination disk space for restore" + probe="${destination_parent}/.sdw-restore-write-test-$$" + : >"${probe}" || die "restore destination is not writable" + rm -f -- "${probe}" + done + + validate_output_directory "${safety_directory}" + local timestamp safety_dump + timestamp=$(date -u +%Y%m%dT%H%M%SZ) + safety_dump="${safety_directory}/pre-restore-${timestamp}-$$.dump" + pg_dump --format=custom --compress=6 --no-owner --no-acl --file "${safety_dump}.partial" + chmod 0600 "${safety_dump}.partial" + mv "${safety_dump}.partial" "${safety_dump}" + + pg_restore --clean --if-exists --exit-on-error --no-owner --no-acl \ + --dbname "${PGDATABASE}" "${extracted}/database.dump" + [[ "$(database_schema_version)" == "${backup_schema}" ]] || + die "restored database schema does not match the backup" + + if [[ -f "${config_destination}" ]]; then + install -m 0600 "${config_destination}" "${config_destination}.pre-restore-${timestamp}" + fi + if [[ -f "${password_destination}" ]]; then + install -m 0600 "${password_destination}" "${password_destination}.pre-restore-${timestamp}" + fi + install -m 0600 "${restored_config}" "${config_destination}.partial" + mv "${config_destination}.partial" "${config_destination}" + install -m 0600 "${extracted}/state/password.json" "${password_destination}.partial" + mv "${password_destination}.partial" "${password_destination}" + + if [[ -d "${key_ring_destination}" ]]; then + mv "${key_ring_destination}" "${key_ring_destination}.pre-restore-${timestamp}" + fi + mkdir -p "${key_ring_destination}" + chmod 0700 "${key_ring_destination}" + cp -a "${extracted}/state/data-protection-keys/." "${key_ring_destination}/" + find "${key_ring_destination}" -type f -exec chmod 0600 {} + + + if [[ -d "${extracted}/state/plugin-manifests" && + ! -f "${extracted}/state/plugin-manifests/none" ]]; then + mkdir -p "${plugin_destination}" + cp -a "${extracted}/state/plugin-manifests/." "${plugin_destination}/" + fi + printf 'restore completed; restart the application and run its health check\n' +} + +command=${1:-} +[[ -n "${command}" ]] || { usage; exit 1; } +shift +case "${command}" in + create) create_backup "$@" ;; + list) list_backups "$@" ;; + verify) verify_backup "$@" ;; + restore) restore_backup "$@" ;; + help|-h|--help) usage ;; + *) usage; die "unknown command: ${command}" ;; +esac diff --git a/deployments/tests/backup-restore-smoke.sh b/deployments/tests/backup-restore-smoke.sh new file mode 100755 index 0000000..38f4875 --- /dev/null +++ b/deployments/tests/backup-restore-smoke.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +drill_root=$(mktemp -d "${TMPDIR:-/tmp}/sdw-backup-drill.XXXXXX") +container_name="sdw-backup-drill-$$" + +cleanup() { + podman stop "${container_name}" >/dev/null 2>&1 || true + case "${drill_root}" in + "${TMPDIR:-/tmp}"/sdw-backup-drill.*) rm -rf -- "${drill_root}" ;; + esac +} +trap cleanup EXIT + +mkdir -p \ + "${drill_root}/keys" \ + "${drill_root}/plugins" \ + "${drill_root}/backups" \ + "${drill_root}/restored" +install -m 0600 "${repo_root}/VERSION" "${drill_root}/appsettings.yml" +install -m 0600 "${repo_root}/VERSION" "${drill_root}/password.json" +app_version=$(tr -d '[:space:]' <"${repo_root}/VERSION") + +podman run --rm --detach --name "${container_name}" \ + --env POSTGRES_PASSWORD=postgres \ + --env POSTGRES_USER=postgres \ + --env POSTGRES_DB=sdw_source \ + --publish 127.0.0.1::5432 \ + postgres:17-alpine >/dev/null + +port=$(podman port "${container_name}" 5432/tcp | sed -E 's/.*:([0-9]+)$/\1/') +export PGHOST=127.0.0.1 PGPORT="${port}" PGUSER=postgres PGPASSWORD=postgres PGDATABASE=sdw_source +for _ in {1..30}; do + pg_isready --quiet && break + sleep 1 +done +pg_isready --quiet +psql --no-psqlrc --set=ON_ERROR_STOP=1 --command \ + "CREATE TABLE drill_items (id integer PRIMARY KEY, value text NOT NULL); INSERT INTO drill_items VALUES (1, 'round-trip');" \ + >/dev/null + +export ConnectionStrings__sdw="Host=127.0.0.1;Port=${port};User ID=postgres;Password=postgres;Database=sdw_source" +unset PGHOST PGPORT PGUSER PGPASSWORD PGDATABASE +archive=$("${repo_root}/deployments/sdw-backup" create \ + --output "${drill_root}/backups" \ + --config "${drill_root}/appsettings.yml" \ + --password-file "${drill_root}/password.json" \ + --key-ring "${drill_root}/keys" \ + --plugin-dir "${drill_root}/plugins" \ + --retention-days 7 \ + --app-version "${app_version}") +unset ConnectionStrings__sdw +export PGHOST=127.0.0.1 PGPORT="${port}" PGUSER=postgres PGPASSWORD=postgres PGDATABASE=sdw_source +"${repo_root}/deployments/sdw-backup" verify "${archive}" + +cp -- "${archive}" "${drill_root}/corrupt.tar.gz" +archive_size=$(stat --format=%s "${drill_root}/corrupt.tar.gz") +printf CORRUPT | dd of="${drill_root}/corrupt.tar.gz" bs=1 \ + seek=$((archive_size / 2)) conv=notrunc status=none +if "${repo_root}/deployments/sdw-backup" verify "${drill_root}/corrupt.tar.gz" >/dev/null 2>&1; then + printf 'corrupt archive unexpectedly verified\n' >&2 + exit 1 +fi + +createdb sdw_restore +export PGDATABASE=sdw_restore +"${repo_root}/deployments/sdw-backup" restore "${archive}" \ + --confirm-replace \ + --expected-version "${app_version}" \ + --config-destination "${drill_root}/restored/appsettings.yml" \ + --password-destination "${drill_root}/restored/password.json" \ + --key-ring-destination "${drill_root}/restored/keys" \ + --plugin-destination "${drill_root}/restored/plugins" \ + --safety-directory "${drill_root}/backups" + +test "$(psql --no-psqlrc --tuples-only --no-align --command \ + 'SELECT value FROM drill_items WHERE id = 1')" = round-trip +cmp "${drill_root}/password.json" "${drill_root}/restored/password.json" +printf 'backup restore smoke test passed\n' diff --git a/docs/backup-restore.md b/docs/backup-restore.md new file mode 100644 index 0000000..49e2952 --- /dev/null +++ b/docs/backup-restore.md @@ -0,0 +1,145 @@ +# 备份、恢复与逻辑数据迁移 + +`sdw-backup` 提供可自动化的本地备份目标；系统包和应用容器均携带该命令。备份由 PostgreSQL custom-format dump、部署配置、`password.json`、Data Protection 密钥环和插件 manifest 组成。下载媒体、导入的原始媒体、qBittorrent 数据与 Valkey 缓存不在其中。 + +## 恢复目标与范围 + +- 建议 RPO：每日一次，重要升级前额外一次。默认 systemd timer 示例为每天 03:30，并随机延迟最多 30 分钟。 +- 典型 RTO：数据库小于 10 GiB 时约 15–60 分钟；实际取决于 PostgreSQL、存储速度和重新核对媒体映射的时间。 +- `downloads` 与外部媒体目录必须使用文件系统快照、NAS 快照或独立备份工具。数据库备份不能还原媒体内容。 +- Data Protection 密钥环是恢复数据库内加密运行时凭据的必要条件，必须与数据库来自同一个恢复点。 +- Valkey 中的会话和短期状态不恢复；恢复后用户可能需要重新登录。 + +每个归档包含时间戳、应用版本、最新 EF schema 版本、最低临时空间估算和 SHA-256 文件清单；旁边的同名 `.sha256` 文件校验整个压缩或加密归档。manifest 和旁路校验文件均不包含数据库口令、JWT、上游 API key 或连接字符串。未加密归档仍包含配置和密钥文件，因此必须按秘密处理。 + +## 创建、列出和验证 + +命令读取标准 `PGHOST`、`PGPORT`、`PGUSER`、`PGPASSWORD`、`PGDATABASE`；在容器内也可直接解析已有的 `ConnectionStrings__sdw` 环境变量。 + +```bash +export PGHOST=localhost PGPORT=5432 PGUSER=sdw PGDATABASE=sdw +read -rsp 'PostgreSQL password: ' PGPASSWORD && export PGPASSWORD + +sudo -u sdw-redive --preserve-env=PGHOST,PGPORT,PGUSER,PGPASSWORD,PGDATABASE \ + sdw-backup create \ + --output /var/lib/sdw-redive/backups \ + --config /etc/sdw-redive/appsettings.yml \ + --password-file /var/lib/sdw-redive/password.json \ + --key-ring /var/lib/sdw-redive/data-protection-keys \ + --retention-days 14 + +sdw-backup list --output /var/lib/sdw-redive/backups +sdw-backup verify /var/lib/sdw-redive/backups/sdw-backup-YYYYMMDDTHHMMSSZ-PID.tar.gz +``` + +命令只输出最终归档路径或非敏感验证结果。最终文件及其 `.sha256` 以 `0600` 发布；失败的 `.partial` 不会被当作备份。保留清理只匹配目标目录中的 `sdw-backup-*.tar.gz[.age]` 及对应校验文件。 + +### age 加密 + +```bash +sdw-backup create ... --age-recipient 'age1...' +sdw-backup verify backup.tar.gz.age --age-identity /secure/backup-key.txt +``` + +私钥不写入归档、manifest 或日志。若使用 webhook，只会发送固定的 `sdw_backup_failed` 事件，不发送错误文本、路径或凭据： + +```bash +export SDW_BACKUP_FAILURE_WEBHOOK=https://monitor.example/hooks/opaque-token +``` + +本地目录是当前内置 target driver。向对象存储扩展时，应在归档完成并通过本地 `verify` 后上传不可变文件；远端上传器不得读取或重新生成 manifest，也不得把 age identity 与备份存放在同一目标。 + +## systemd 定时执行 + +系统包安装 `/etc/sdw-redive/backup.env`、`sdw-backup.service` 和 `sdw-backup.timer`，但不会在口令仍为占位符时自动启用。安装 PostgreSQL client，编辑并保护环境文件后启用： + +```bash +sudoedit /etc/sdw-redive/backup.env +sudo chown root:sdw-redive /etc/sdw-redive/backup.env +sudo chmod 0640 /etc/sdw-redive/backup.env +sudo systemctl enable --now sdw-backup.timer +sudo systemctl start sdw-backup.service +sudo journalctl -u sdw-backup.service +``` + +升级前可执行 `sudo systemctl start sdw-backup.service`，验证新归档后再升级。这样自动迁移数据库前已有明确恢复点。 + +## 容器部署 + +模板把 `./backups` 挂载到 `/app/backups`，并以只读方式挂载 Compose 配置。创建与验证： + +```bash +mkdir -p backups && chmod 0700 backups +podman-compose exec sdw-redive sdw-backup create \ + --output /app/backups \ + --config /app/deployment/podman-compose.yml \ + --password-file /app/data/password.json \ + --key-ring /app/data/data-protection-keys +podman-compose exec sdw-redive sdw-backup verify /app/backups/sdw-backup-....tar.gz +``` + +生产环境可用宿主机 systemd timer 或 cron 调用上述命令。不要把数据库口令作为命令行参数；脚本使用容器已有的连接字符串环境变量。 + +## 灾难恢复 + +恢复是替换操作。先停止所有应用副本和后台任务；仅停止应用，不要停止 PostgreSQL。 + +1. 把目标应用安装为与备份相同的 major 版本，并准备空数据库。 +2. 预先挂载足够空间；脚本在任何数据库写入前验证路径、链接、所有 SHA-256、`pg_restore --list`、格式、major 版本、可选 schema 版本、临时空间和目标目录可写性。 +3. 执行恢复。命令先在 safety directory 创建现有数据库 dump，旧配置、密码和密钥环也会重命名为 `.pre-restore-*`，可人工回退。 +4. 修正文件所有者与权限，启动应用，检查 `/api/auth/allowRegister`、登录、订阅和文件浏览。 + +```bash +sudo systemctl stop sdw-redive +sudo -u sdw-redive --preserve-env=PGHOST,PGPORT,PGUSER,PGPASSWORD,PGDATABASE \ + sdw-backup restore /var/lib/sdw-redive/backups/sdw-backup-....tar.gz \ + --confirm-replace \ + --expected-version 2.3.0 \ + --expected-schema 20260801000000_ExpectedMigration \ + --config-destination /etc/sdw-redive/appsettings.yml \ + --password-destination /var/lib/sdw-redive/password.json \ + --key-ring-destination /var/lib/sdw-redive/data-protection-keys +sudo chown root:sdw-redive /etc/sdw-redive/appsettings.yml +sudo chmod 0640 /etc/sdw-redive/appsettings.yml +sudo chown -R sdw-redive:sdw-redive /var/lib/sdw-redive/data-protection-keys \ + /var/lib/sdw-redive/password.json +sudo systemctl start sdw-redive +curl --fail http://127.0.0.1:5097/api/auth/allowRegister +``` + +容器恢复时停止应用，然后用临时容器运行 backup entrypoint。Compose 文件在容器内只读，所以先把恢复出的配置放到宿主机可写的 backups 目录，再由管理员检查并替换： + +```bash +podman-compose stop sdw-redive +podman-compose run --rm --no-deps --entrypoint sdw-backup sdw-redive \ + restore /app/backups/sdw-backup-....tar.gz \ + --confirm-replace --expected-version 2.3.0 \ + --config-destination /app/backups/restored/podman-compose.yml \ + --password-destination /app/data/password.json \ + --key-ring-destination /app/data/data-protection-keys \ + --safety-directory /app/backups +# 检查并在宿主机替换 podman-compose.yml，然后： +podman-compose up -d +curl --fail http://127.0.0.1:5097/api/auth/allowRegister +``` + +如果版本、schema 或空间检查失败，数据库不会被写入。只有在经过人工评估后才能使用 `--allow-version-mismatch`；该开关仍不会跳过格式、校验和、dump 与空间验证。 + +## 逻辑 JSON 导出与导入 + +JWT 管理员可按类别迁移非秘密业务数据：`feeds`、`automation-policies`、`filename-rules`、`metadata-corrections`、`playback`，或 `all`。 + +```bash +curl --fail -H "Authorization: Bearer $TOKEN" \ + 'https://sdw.example/api/data-transfer/export?categories=feeds,automation-policies,playback' \ + -o logical-export.json + +jq '. + {conflictStrategy:"skip"}' logical-export.json > logical-import.json +curl --fail -H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json' \ + --data-binary @logical-import.json \ + https://sdw.example/api/data-transfer/import +``` + +导出 envelope 内的 SHA-256 在任何写入前验证。`skip` 可安全重复导入；`overwrite` 更新稳定键冲突项；`fail` 在首个冲突处返回 409，事务不会提交。订阅以 URL、规则以 TMDB id + pattern、人工修正以 release URL + title + publish time、播放进度以虚拟路径匹配。目标实例缺少对应 release 或虚拟文件时会明确计入 skipped，不会制造指向不存在媒体的记录。人工修正迁移当前元数据和审计操作，但不会覆盖物理路径；映射仍由目标实例的文件映射流程负责。 + +逻辑导出不含 JWT、登录密码、WebDAV token、Data Protection key、AI/qBittorrent 凭据、聊天内容或媒体文件。跨 major 格式、不匹配校验和、未知类别、非法数值与超大类别会在事务开始前拒绝。 diff --git a/docs/container-deployment.md b/docs/container-deployment.md index f14757e..18ec411 100644 --- a/docs/container-deployment.md +++ b/docs/container-deployment.md @@ -215,6 +215,8 @@ podman-compose down -v ## 更新 +更新前建议先执行并验证一次快照；完整的定时、加密和灾难恢复流程见 [备份、恢复与逻辑数据迁移](backup-restore.md)。数据库备份不包含 `downloads` 或外部媒体目录，这些卷需要独立快照。 + ```bash # 拉取最新镜像 podman-compose pull sdw-redive diff --git a/docs/server-deployment.md b/docs/server-deployment.md index 5814636..35fecd0 100644 --- a/docs/server-deployment.md +++ b/docs/server-deployment.md @@ -218,6 +218,8 @@ Environment=ASPNETCORE_URLS=http://0.0.0.0:8080 ## 服务管理 +正式升级或修改数据库前，建议先运行 `sdw-backup create` 并执行 `sdw-backup verify`。系统包还提供可选的每日 systemd timer；RPO/RTO、加密、恢复演练和逻辑 JSON 迁移详见 [备份、恢复与逻辑数据迁移](backup-restore.md)。媒体文件不包含在数据库备份中，必须单独保护。 + ```bash # 启动服务 sudo systemctl start sdw-redive diff --git a/packaging/backup.env b/packaging/backup.env new file mode 100644 index 0000000..4b8684b --- /dev/null +++ b/packaging/backup.env @@ -0,0 +1,17 @@ +# Protect this file as a secret (root:sdw-redive, mode 0640). +PGHOST=localhost +PGPORT=5432 +PGUSER=sdw +PGDATABASE=sdw +PGPASSWORD=CHANGE_ME + +SDW_BACKUP_DIRECTORY=/var/lib/sdw-redive/backups +SDW_BACKUP_RETENTION_DAYS=14 +SDW_CONFIG_PATH=/etc/sdw-redive/appsettings.yml +SDW_PASSWORD_FILE=/var/lib/sdw-redive/password.json +SDW_KEY_RING_PATH=/var/lib/sdw-redive/data-protection-keys +SDW_PLUGIN_DIRECTORY=/var/lib/sdw-redive/plugins + +# Optional age encryption and generic failure webhook. Neither value is logged. +# SDW_BACKUP_AGE_RECIPIENT=age1... +# SDW_BACKUP_FAILURE_WEBHOOK=https://monitor.example/hooks/... diff --git a/packaging/nfpm.yaml b/packaging/nfpm.yaml index 3dcd289..d3ce403 100644 --- a/packaging/nfpm.yaml +++ b/packaging/nfpm.yaml @@ -43,6 +43,28 @@ contents: file_info: mode: 0755 + - src: ./deployments/sdw-backup + dst: /usr/bin/sdw-backup + file_info: + mode: 0755 + + - src: ./packaging/sdw-backup.service + dst: /usr/lib/systemd/system/sdw-backup.service + + - src: ./packaging/sdw-backup.timer + dst: /usr/lib/systemd/system/sdw-backup.timer + + - src: ./packaging/backup.env + dst: /etc/sdw-redive/backup.env + type: config|noreplace + file_info: + mode: 0640 + owner: root + group: sdw-redive + + - src: ./VERSION + dst: /usr/lib/sdw-redive/VERSION + - src: ./LICENSE dst: /usr/share/doc/sdw-redive/LICENSE @@ -77,6 +99,13 @@ contents: owner: sdw-redive group: sdw-redive + - dst: /var/lib/sdw-redive/backups + type: dir + file_info: + mode: 0700 + owner: sdw-redive + group: sdw-redive + scripts: postinstall: ./packaging/postinstall.sh preremove: ./packaging/preremove.sh diff --git a/packaging/postinstall.sh b/packaging/postinstall.sh index 1530a32..10cd22a 100755 --- a/packaging/postinstall.sh +++ b/packaging/postinstall.sh @@ -19,6 +19,10 @@ if [ -f "$CONFIG" ]; then chown root:sdw-redive "$CONFIG" chmod 0640 "$CONFIG" fi +if [ -f /etc/sdw-redive/backup.env ]; then + chown root:sdw-redive /etc/sdw-redive/backup.env + chmod 0640 /etc/sdw-redive/backup.env +fi # Add sdw-redive to valkey group if it exists (for Unix socket access) if getent group valkey >/dev/null 2>&1; then @@ -43,7 +47,8 @@ chown -R sdw-redive:sdw-redive /var/lib/sdw-redive # Data Protection keys and the password hash are service-owned secrets. The # private directory also protects keys created by future application runs. install -d -m 0700 -o sdw-redive -g sdw-redive \ - /var/lib/sdw-redive/data-protection-keys + /var/lib/sdw-redive/data-protection-keys \ + /var/lib/sdw-redive/backups if [ -f /var/lib/sdw-redive/password.json ]; then chown sdw-redive:sdw-redive /var/lib/sdw-redive/password.json chmod 0600 /var/lib/sdw-redive/password.json diff --git a/packaging/sdw-backup.service b/packaging/sdw-backup.service new file mode 100644 index 0000000..cd14bd4 --- /dev/null +++ b/packaging/sdw-backup.service @@ -0,0 +1,16 @@ +[Unit] +Description=Create a verified SecondDimensionWatcher Re:Dive backup +After=network-online.target postgresql.service +Wants=network-online.target + +[Service] +Type=oneshot +User=sdw-redive +Group=sdw-redive +EnvironmentFile=/etc/sdw-redive/backup.env +ExecStart=/usr/bin/sdw-backup create +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/var/lib/sdw-redive/backups diff --git a/packaging/sdw-backup.timer b/packaging/sdw-backup.timer new file mode 100644 index 0000000..4af178b --- /dev/null +++ b/packaging/sdw-backup.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Daily SecondDimensionWatcher Re:Dive backup + +[Timer] +OnCalendar=*-*-* 03:30:00 +Persistent=true +RandomizedDelaySec=30m +Unit=sdw-backup.service + +[Install] +WantedBy=timers.target From 30192c9d34a52924dce8692212d9af198a36bb17 Mon Sep 17 00:00:00 2001 From: mahoshojoHCG Date: Sat, 29 Aug 2026 23:20:06 +0800 Subject: [PATCH 02/45] feat: add server-side HLS transcoding --- .github/workflows/build.yml | 5 + Containerfile | 3 + ...ffmpeg-ffmpeg-npm-0.12.15-ef17f09f1b.patch | 47 - .../mock-server.mjs | 98 +- .../package.json | 3 +- .../src/i18n/locales/en/player.json | 24 +- .../src/i18n/locales/ja/player.json | 22 +- .../src/i18n/locales/zh-CN/player.json | 22 +- .../src/pages/PlayerPage.tsx | 237 ++++- .../playback/mkv/serverTranscoding.test.ts | 84 ++ .../src/playback/mkv/transcoder.ts | 397 -------- .../src/playback/serverTranscoding.ts | 121 +++ .../src/types/parcel-assets.d.ts | 10 - SecondDimensionWatcherReDive.Client/yarn.lock | 42 +- .../Helpers/FakeTranscodingService.cs | 103 ++ .../Transcoding/TranscodingApiTests.cs | 72 ++ .../WebDavWebApplicationFactory.cs | 5 + .../FfmpegProcessRunnerTests.cs | 126 +++ .../HlsTranscodingServiceTests.cs | 480 +++++++++ .../TranscodingControllerTests.cs | 178 ++++ .../TranscodingPlannerTests.cs | 145 +++ .../External/AppJsonSerializerContext.cs | 5 + .../Controllers/External/Transcoding.cs | 50 + .../Controllers/FileController.cs | 28 +- .../Controllers/TranscodingController.cs | 237 +++++ SecondDimensionWatcherReDive/Program.cs | 36 + .../Transcoding/FfmpegProcessRunner.cs | 545 ++++++++++ .../Transcoding/HlsTranscodingService.cs | 945 ++++++++++++++++++ .../Transcoding/IHlsTranscodingService.cs | 44 + .../Services/Transcoding/ScopeOwnedStream.cs | 97 ++ .../Transcoding/TranscodingMetrics.cs | 107 ++ .../Services/Transcoding/TranscodingModels.cs | 161 +++ .../Transcoding/TranscodingOptions.cs | 27 + .../Transcoding/TranscodingPlanner.cs | 126 +++ .../Utils/FileStore/PlaybackPathResolver.cs | 29 + .../appsettings.example.json | 23 + THIRD_PARTY_NOTICES.md | 31 +- deployments/podman-compose.yml | 1 + docs/container-deployment.md | 12 +- docs/server-deployment.md | 53 + packaging/appsettings.yml | 21 + packaging/nfpm.yaml | 9 + packaging/postinstall.sh | 2 + 43 files changed, 4196 insertions(+), 617 deletions(-) delete mode 100644 SecondDimensionWatcherReDive.Client/.yarn/patches/@ffmpeg-ffmpeg-npm-0.12.15-ef17f09f1b.patch create mode 100644 SecondDimensionWatcherReDive.Client/src/playback/mkv/serverTranscoding.test.ts delete mode 100644 SecondDimensionWatcherReDive.Client/src/playback/mkv/transcoder.ts create mode 100644 SecondDimensionWatcherReDive.Client/src/playback/serverTranscoding.ts create mode 100644 SecondDimensionWatcherReDive.IntegrationTest/Helpers/FakeTranscodingService.cs create mode 100644 SecondDimensionWatcherReDive.IntegrationTest/Transcoding/TranscodingApiTests.cs create mode 100644 SecondDimensionWatcherReDive.Test/FfmpegProcessRunnerTests.cs create mode 100644 SecondDimensionWatcherReDive.Test/HlsTranscodingServiceTests.cs create mode 100644 SecondDimensionWatcherReDive.Test/TranscodingControllerTests.cs create mode 100644 SecondDimensionWatcherReDive.Test/TranscodingPlannerTests.cs create mode 100644 SecondDimensionWatcherReDive/Controllers/External/Transcoding.cs create mode 100644 SecondDimensionWatcherReDive/Controllers/TranscodingController.cs create mode 100644 SecondDimensionWatcherReDive/Services/Transcoding/FfmpegProcessRunner.cs create mode 100644 SecondDimensionWatcherReDive/Services/Transcoding/HlsTranscodingService.cs create mode 100644 SecondDimensionWatcherReDive/Services/Transcoding/IHlsTranscodingService.cs create mode 100644 SecondDimensionWatcherReDive/Services/Transcoding/ScopeOwnedStream.cs create mode 100644 SecondDimensionWatcherReDive/Services/Transcoding/TranscodingMetrics.cs create mode 100644 SecondDimensionWatcherReDive/Services/Transcoding/TranscodingModels.cs create mode 100644 SecondDimensionWatcherReDive/Services/Transcoding/TranscodingOptions.cs create mode 100644 SecondDimensionWatcherReDive/Services/Transcoding/TranscodingPlanner.cs create mode 100644 SecondDimensionWatcherReDive/Utils/FileStore/PlaybackPathResolver.cs diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 98cb3e1..6b635da 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,6 +26,11 @@ jobs: with: dotnet-version: '10.0.x' + - name: Install FFmpeg + run: | + sudo apt-get update + sudo apt-get install -y ffmpeg + - name: Run tests run: >- dotnet test SecondDimensionWatcherReDive.slnx -c Release diff --git a/Containerfile b/Containerfile index 72b49a9..5468f19 100644 --- a/Containerfile +++ b/Containerfile @@ -21,6 +21,9 @@ RUN dotnet publish SecondDimensionWatcherReDive/SecondDimensionWatcherReDive.csp # Stage 3: Runtime FROM mcr.microsoft.com/dotnet/aspnet:10.0 WORKDIR /app +RUN apt-get update \ + && apt-get install -y --no-install-recommends ffmpeg \ + && rm -rf /var/lib/apt/lists/* COPY --from=backend-build /app . EXPOSE 8080 # Optional: read-only NFSv4 export (set Nfs:Enabled=true to activate; publish port at run time). diff --git a/SecondDimensionWatcherReDive.Client/.yarn/patches/@ffmpeg-ffmpeg-npm-0.12.15-ef17f09f1b.patch b/SecondDimensionWatcherReDive.Client/.yarn/patches/@ffmpeg-ffmpeg-npm-0.12.15-ef17f09f1b.patch deleted file mode 100644 index 94376e5..0000000 --- a/SecondDimensionWatcherReDive.Client/.yarn/patches/@ffmpeg-ffmpeg-npm-0.12.15-ef17f09f1b.patch +++ /dev/null @@ -1,47 +0,0 @@ -diff --git a/dist/esm/worker.js b/dist/esm/worker.js -index cca2a6116bab5349cfa35bc1dba0b6e696af91ee..84c1e701e92810b722a186a941db0a56a615407f 100644 ---- a/dist/esm/worker.js -+++ b/dist/esm/worker.js -@@ -6,21 +6,29 @@ import { ERROR_UNKNOWN_MESSAGE_TYPE, ERROR_NOT_LOADED, ERROR_IMPORT_FAILURE, } f - let ffmpeg; - const load = async ({ coreURL: _coreURL, wasmURL: _wasmURL, workerURL: _workerURL, }) => { - const first = !ffmpeg; -+ if (!_coreURL) -+ _coreURL = CORE_URL; -+ // Parcel emits the core as a self-registering module without native ESM -+ // exports. Expose the registered entry explicitly before importing it in -+ // this module worker; this also avoids forbidden importScripts() calls. -+ const response = await fetch(_coreURL); -+ if (!response.ok) { -+ throw ERROR_IMPORT_FAILURE; -+ } -+ const bundledCore = await response.text(); -+ const exposedCore = bundledCore.replace(/([$_A-Za-z][\w$]*)\("([$_A-Za-z0-9]+)"\);(\s*(?:\}\)\(\);?)?\s*)$/, 'self.createFFmpegCore=$1("$2").default;$3'); -+ if (exposedCore === bundledCore) { -+ throw ERROR_IMPORT_FAILURE; -+ } -+ const coreBlobURL = URL.createObjectURL(new Blob([exposedCore], { type: 'text/javascript' })); - try { -- if (!_coreURL) -- _coreURL = CORE_URL; -- // when web worker type is `classic`. -- importScripts(_coreURL); -+ await import(/* @vite-ignore */ coreBlobURL); - } -- catch { -- if (!_coreURL || _coreURL === CORE_URL) -- _coreURL = CORE_URL.replace('/umd/', '/esm/'); -- // when web worker type is `module`. -- self.createFFmpegCore = (await import( -- /* @vite-ignore */ _coreURL)).default; -- if (!self.createFFmpegCore) { -- throw ERROR_IMPORT_FAILURE; -- } -+ finally { -+ URL.revokeObjectURL(coreBlobURL); -+ } -+ if (!self.createFFmpegCore) { -+ throw ERROR_IMPORT_FAILURE; - } - const coreURL = _coreURL; - const wasmURL = _wasmURL ? _wasmURL : _coreURL.replace(/.js$/g, ".wasm"); diff --git a/SecondDimensionWatcherReDive.Client/mock-server.mjs b/SecondDimensionWatcherReDive.Client/mock-server.mjs index f5ee4d4..a43bb6b 100644 --- a/SecondDimensionWatcherReDive.Client/mock-server.mjs +++ b/SecondDimensionWatcherReDive.Client/mock-server.mjs @@ -1369,7 +1369,14 @@ async function route(method, pathname, searchParams, req, res) { } // --- All remaining endpoints require auth --- - if (!hasAuth(req) && !pathname.startsWith("/api/auth/")) { + const publicTranscodingSession = + (method === "GET" || method === "DELETE") && + pathname.startsWith("/api/transcoding/sessions/"); + if ( + !hasAuth(req) && + !pathname.startsWith("/api/auth/") && + !publicTranscodingSession + ) { return empty(res, 401); } @@ -2498,6 +2505,95 @@ async function route(method, pathname, searchParams, req, res) { // --- Files --- + if (method === "POST" && pathname === "/api/transcoding/prepare") { + const sessionId = randomUUID(); + const token = randomBytes(32).toString("hex"); + const base = `/api/transcoding/sessions/${sessionId}`; + return json(res, { + sessionId, + state: "ready", + strategy: "remux", + isPlayable: true, + cacheHit: false, + progress: 1, + speed: 8.5, + queuePosition: null, + error: null, + videoCodec: "h264", + audioCodec: "aac", + statusUrl: `${base}?token=${token}`, + cancelUrl: `${base}?token=${token}`, + playbackUrl: `${base}/media.m3u8?token=${token}`, + subtitles: [], + unsupportedSubtitleCount: 0, + }); + } + + const transcodeSessionMatch = pathname.match( + /^\/api\/transcoding\/sessions\/([^/]+)\/([^/]+)(?:\/([^/]+))?$/, + ); + if (method === "GET" && transcodeSessionMatch?.[2] === "media.m3u8") { + const sessionId = transcodeSessionMatch[1]; + const token = searchParams.get("token") ?? ""; + res.writeHead(200, { + "Content-Type": "application/vnd.apple.mpegurl", + "Cache-Control": "no-cache, no-store", + }); + return res.end( + `#EXTM3U\n#EXT-X-VERSION:3\n#EXTINF:6,\n/api/transcoding/sessions/${sessionId}/segments/segment-000000.ts?token=${token}\n#EXT-X-ENDLIST\n`, + ); + } + if ( + method === "GET" && + transcodeSessionMatch?.[2] === "segments" && + transcodeSessionMatch?.[3] + ) { + res.writeHead(200, { "Content-Type": "video/mp2t" }); + return res.end("Mock HLS segment"); + } + const transcodeStatusMatch = pathname.match( + /^\/api\/transcoding\/sessions\/([^/]+)$/, + ); + if (method === "GET" && transcodeStatusMatch) { + const sessionId = transcodeStatusMatch[1]; + const token = searchParams.get("token") ?? ""; + const base = `/api/transcoding/sessions/${sessionId}`; + return json(res, { + sessionId, + state: "ready", + strategy: "remux", + isPlayable: true, + cacheHit: true, + progress: 1, + speed: 8.5, + queuePosition: null, + error: null, + videoCodec: "h264", + audioCodec: "aac", + statusUrl: `${base}?token=${token}`, + cancelUrl: `${base}?token=${token}`, + playbackUrl: `${base}/media.m3u8?token=${token}`, + subtitles: [], + unsupportedSubtitleCount: 0, + }); + } + if (method === "DELETE" && transcodeStatusMatch) return empty(res, 204); + + if (method === "GET" && pathname === "/api/transcoding/metrics") { + return json(res, { + queuedJobs: 0, + activeJobs: 0, + completedJobs: 1, + failedJobs: 0, + canceledJobs: 0, + cacheHits: 1, + cacheBytes: 1048576, + averageFirstSegmentSeconds: 0.8, + averageTranscodeSpeed: 8.5, + failureRate: 0, + }); + } + if (method === "GET" && pathname === "/api/file/list") { const id = searchParams.get("id"); const relativeDir = searchParams.get("relativeDir") ?? ""; diff --git a/SecondDimensionWatcherReDive.Client/package.json b/SecondDimensionWatcherReDive.Client/package.json index 5f71a25..e7147e1 100644 --- a/SecondDimensionWatcherReDive.Client/package.json +++ b/SecondDimensionWatcherReDive.Client/package.json @@ -1,7 +1,5 @@ { "dependencies": { - "@ffmpeg/core": "^0.12.10", - "@ffmpeg/ffmpeg": "patch:@ffmpeg/ffmpeg@npm%3A0.12.15#~/.yarn/patches/@ffmpeg-ffmpeg-npm-0.12.15-ef17f09f1b.patch", "@radix-ui/react-dialog": "^1.1.23", "@radix-ui/react-dropdown-menu": "^2.1.24", "@radix-ui/react-progress": "^1.1.16", @@ -12,6 +10,7 @@ "artplayer-proxy-mediabunny": "^1.2.0", "clsx": "^2.1.1", "dayjs": "^1.11.23", + "hls.js": "^1.7.1", "i18next": "^26.4.0", "i18next-browser-languagedetector": "^8.2.1", "lucide-react": "^1.34.0", diff --git a/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/player.json b/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/player.json index 2c3c475..0225168 100644 --- a/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/player.json +++ b/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/player.json @@ -56,25 +56,27 @@ "mkv": { "mode": { "demuxed": "Browser MKV demux", - "transcoded": "Browser transcoded" + "serverRemuxed": "Server HLS remux", + "serverTranscoded": "Server HLS transcode" }, "stages": { "probing": "Inspecting MKV audio and video tracks", "extractingSubtitles": "Extracting and converting embedded subtitles", - "loadingTranscoder": "Loading the browser transcoder", - "downloading": "Downloading the MKV for browser conversion", - "readingTracks": "Reading MKV track information", - "convertingSubtitles": "Converting embedded subtitles to WebVTT", - "transcodingVideo": "Converting unsupported audio or video codecs", - "finalizing": "Preparing the playable video" + "serverQueued": "Waiting in the server transcoding queue (position {{position}})", + "serverProbing": "The server is inspecting media tracks", + "serverRemuxing": "Remuxing compatible tracks into HLS", + "serverTranscoding": "Transcoding unsupported tracks into HLS", + "serverFinalizing": "Streaming generated HLS segments while the server finishes" }, - "playbackPreparationFailed": "This MKV could not be demuxed or converted in the browser", + "playbackPreparationFailed": "The server could not prepare this MKV for browser playback", "subtitleExtractionFailed": "The MKV can play, but its embedded subtitles could not be extracted", - "transcodeNotice": "Browser software conversion downloads the complete file and can take a while. Keep this page open.", + "serverNotice": "Playback starts as soon as the first server segment is ready; the source file is not downloaded to this device.", + "subtitleNotice": "The video is ready while compatible embedded subtitles are prepared in the background.", + "cacheHit": "Cached segments", "codecSummary": "MKV demuxed automatically · video {{video}} · audio {{audio}}", "noAudio": "none", - "bitmapSubtitlesSkipped_one": "Skipped {{count}} subtitle track that the browser could not convert", - "bitmapSubtitlesSkipped_other": "Skipped {{count}} subtitle tracks that the browser could not convert" + "bitmapSubtitlesSkipped_one": "{{count}} bitmap subtitle track is unavailable unless server burn-in is enabled", + "bitmapSubtitlesSkipped_other": "{{count}} bitmap subtitle tracks are unavailable unless server burn-in is enabled" }, "next": { "play": "Next episode", diff --git a/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/player.json b/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/player.json index 4e78732..34d486f 100644 --- a/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/player.json +++ b/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/player.json @@ -53,24 +53,26 @@ "mkv": { "mode": { "demuxed": "ブラウザーで MKV 分離", - "transcoded": "ブラウザーで変換済み" + "serverRemuxed": "サーバー HLS リマックス", + "serverTranscoded": "サーバー HLS 変換" }, "stages": { "probing": "MKV の映像・音声トラックを確認しています", "extractingSubtitles": "内蔵字幕を抽出して変換しています", - "loadingTranscoder": "ブラウザー変換エンジンを読み込んでいます", - "downloading": "ブラウザー変換用に MKV をダウンロードしています", - "readingTracks": "MKV のトラック情報を読み取っています", - "convertingSubtitles": "内蔵字幕を WebVTT に変換しています", - "transcodingVideo": "未対応の映像・音声コーデックを変換しています", - "finalizing": "再生可能な動画を準備しています" + "serverQueued": "サーバー変換待ちです（キュー {{position}} 番）", + "serverProbing": "サーバーがメディアトラックを確認しています", + "serverRemuxing": "互換トラックを HLS にリマックスしています", + "serverTranscoding": "未対応トラックを HLS に変換しています", + "serverFinalizing": "生成済み HLS を再生しながら残りを処理しています" }, - "playbackPreparationFailed": "この MKV をブラウザーで分離または変換できませんでした", + "playbackPreparationFailed": "サーバーでこの MKV をブラウザー再生用に準備できませんでした", "subtitleExtractionFailed": "MKV は再生できますが、内蔵字幕を抽出できませんでした", - "transcodeNotice": "ブラウザーでのソフトウェア変換はファイル全体をダウンロードするため、時間がかかる場合があります。このページを開いたままにしてください。", + "serverNotice": "最初のサーバー分割ができ次第再生し、元ファイル全体は端末へダウンロードしません。", + "subtitleNotice": "動画は再生可能です。対応する内蔵字幕をバックグラウンドで準備しています。", + "cacheHit": "キャッシュ済み分割を再利用", "codecSummary": "MKV を自動分離しました · 映像 {{video}} · 音声 {{audio}}", "noAudio": "なし", - "bitmapSubtitlesSkipped": "ブラウザーで変換できない字幕トラック {{count}} 件をスキップしました" + "bitmapSubtitlesSkipped": "画像字幕 {{count}} 件は利用できません。サーバー側の焼き込み設定で有効化できます" }, "next": { "play": "次のエピソード", diff --git a/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/player.json b/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/player.json index 63a4d8b..417d87f 100644 --- a/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/player.json +++ b/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/player.json @@ -53,24 +53,26 @@ "mkv": { "mode": { "demuxed": "MKV 前端拆包", - "transcoded": "浏览器已转码" + "serverRemuxed": "服务端 HLS 无损封装", + "serverTranscoded": "服务端 HLS 转码" }, "stages": { "probing": "正在检查 MKV 音视频轨道", "extractingSubtitles": "正在拆出并转换内封字幕", - "loadingTranscoder": "正在加载浏览器转码器", - "downloading": "正在下载 MKV 以供前端转换", - "readingTracks": "正在读取 MKV 轨道信息", - "convertingSubtitles": "正在把内封字幕转换为 WebVTT", - "transcodingVideo": "正在转换浏览器不支持的音视频编码", - "finalizing": "正在生成可播放的视频" + "serverQueued": "正在等待服务端转码（队列第 {{position}} 位）", + "serverProbing": "服务端正在探测媒体轨道", + "serverRemuxing": "正在将兼容轨道无损封装为 HLS", + "serverTranscoding": "正在将不兼容轨道转为 HLS", + "serverFinalizing": "已开始播放生成的分片，服务端继续处理剩余内容" }, - "playbackPreparationFailed": "无法在浏览器中拆包或转换此 MKV", + "playbackPreparationFailed": "服务端无法为浏览器准备此 MKV", "subtitleExtractionFailed": "MKV 可以播放，但内封字幕拆出失败", - "transcodeNotice": "前端软件转码需要完整下载文件，首次处理可能较慢，请保持此页面打开。", + "serverNotice": "首个服务端分片就绪后即可播放，源文件不会完整下载到本设备。", + "subtitleNotice": "视频已可播放，兼容的内封字幕仍在后台准备。", + "cacheHit": "已复用缓存分片", "codecSummary": "MKV 已自动拆包 · 视频 {{video}} · 音频 {{audio}}", "noAudio": "无", - "bitmapSubtitlesSkipped": "已跳过 {{count}} 条浏览器无法转换的字幕轨道" + "bitmapSubtitlesSkipped": "{{count}} 条位图字幕不可用；管理员可配置服务端烧录" }, "next": { "play": "下一集", diff --git a/SecondDimensionWatcherReDive.Client/src/pages/PlayerPage.tsx b/SecondDimensionWatcherReDive.Client/src/pages/PlayerPage.tsx index a6d86cc..46edbe4 100644 --- a/SecondDimensionWatcherReDive.Client/src/pages/PlayerPage.tsx +++ b/SecondDimensionWatcherReDive.Client/src/pages/PlayerPage.tsx @@ -1,5 +1,6 @@ import Artplayer from "artplayer"; import artplayerProxyMediabunny from "artplayer-proxy-mediabunny"; +import Hls from "hls.js"; import { CaptionsFileFormat, CaptionsRenderer, @@ -38,15 +39,17 @@ import { } from "../playback/mkv/subtitles"; import { MkvPlaybackProbe, - canCopyVideoCodecToMp4, isAbortError, isMkvPath, probeMkvPlayback, } from "../playback/mkv/support"; import { - MkvTranscodeStage, - transcodeMkvForBrowser, -} from "../playback/mkv/transcoder"; + ServerTranscodingStrategy, + prepareServerTranscoding, + releaseServerTranscoding, + touchServerTranscoding, + watchServerTranscoding, +} from "../playback/serverTranscoding"; import { ExternalSubtitle, PlaybackPreferences, @@ -66,13 +69,21 @@ interface ResolvedSubtitle extends ExternalSubtitle { source: "external" | "embedded"; } -type PlaybackMode = "native" | "mkvProxy" | "transcoded"; +type PlaybackMode = "native" | "mkvProxy" | "hls"; type MkvPreparationStage = - "probing" | "extractingSubtitles" | MkvTranscodeStage; + | "probing" + | "extractingSubtitles" + | "serverQueued" + | "serverProbing" + | "serverRemuxing" + | "serverTranscoding" + | "serverFinalizing"; interface MkvPreparationStatus { stage: MkvPreparationStage; progress?: number; + queuePosition?: number; + speed?: number; } interface BrowserAudioTrack { @@ -248,6 +259,9 @@ export const PlayerPage: React.FC = () => { const [mkvStatus, setMkvStatus] = React.useState( null, ); + const [serverStrategy, setServerStrategy] = + React.useState(null); + const [serverCacheHit, setServerCacheHit] = React.useState(false); const [skippedSubtitleCount, setSkippedSubtitleCount] = React.useState(0); const [subtitleDiscoveryComplete, setSubtitleDiscoveryComplete] = React.useState(false); @@ -301,6 +315,8 @@ export const PlayerPage: React.FC = () => { setPlaybackMode("native"); setMkvProbe(null); setMkvStatus(null); + setServerStrategy(null); + setServerCacheHit(false); setSkippedSubtitleCount(0); setSubtitleDiscoveryComplete(false); setSubtitles([]); @@ -318,6 +334,9 @@ export const PlayerPage: React.FC = () => { if (!animationId || !playbackContext) return; let cancelled = false; let releasePreparedMedia: (() => void) | null = null; + let cancelServerUrl: string | null = null; + let serverKeepAliveTimer: ReturnType | null = + null; const controller = new AbortController(); setLinkLoading(true); setLinkError(null); @@ -357,7 +376,7 @@ export const PlayerPage: React.FC = () => { } catch (error) { if (isAbortError(error)) throw error; // A server/probe incompatibility should still get a chance to use - // the full-file software fallback. + // the server-side probe and streaming fallback. } if (cancelled) return; setMkvProbe(probe); @@ -414,35 +433,78 @@ export const PlayerPage: React.FC = () => { return; } - const transcoded = await transcodeMkvForBrowser( - videoLink.url, - controller.signal, - (update) => { - if (!cancelled) setMkvStatus(update); - }, + const initialSession = await prepareServerTranscoding( { - copyVideo: - probe?.videoDecodable === true && - canCopyVideoCodecToMp4(probe.videoCodec), + id: animationId, + path: playbackContext.media.path, + quality: "auto", + audioLanguage: playbackContext.preferences.audioLanguage, + audioTrackLabel: playbackContext.preferences.audioTrackLabel, + subtitleLanguage: playbackContext.preferences.subtitleLanguage, + subtitleTrackLabel: playbackContext.preferences.subtitleTrackLabel, }, + controller.signal, ); - if (cancelled) { - transcoded.release(); - return; + cancelServerUrl = initialSession.cancelUrl; + const readySession = await watchServerTranscoding( + initialSession, + controller.signal, + (session) => { + if (cancelled) return; + cancelServerUrl = session.cancelUrl; + setServerStrategy(session.strategy); + setServerCacheHit(session.cacheHit); + setSkippedSubtitleCount(session.unsupportedSubtitleCount); + if (session.subtitles.length > 0) { + setSubtitles([ + ...subtitleLinks, + ...session.subtitles.map((subtitle) => ({ + ...subtitle, + source: "embedded" as const, + })), + ]); + } + + if (session.isPlayable && session.playbackUrl) { + setPlaybackMode(session.strategy === "direct" ? "native" : "hls"); + setPlaybackUrl(session.playbackUrl); + setLinkLoading(false); + } + + if (session.state === "ready") { + setMkvStatus(null); + setSubtitleDiscoveryComplete(true); + return; + } + const stage: MkvPreparationStage = + session.state === "queued" + ? "serverQueued" + : session.state === "probing" + ? "serverProbing" + : session.strategy === "remux" + ? "serverRemuxing" + : session.isPlayable + ? "serverFinalizing" + : "serverTranscoding"; + setMkvStatus({ + stage, + progress: session.progress ?? undefined, + queuePosition: session.queuePosition ?? undefined, + speed: session.speed ?? undefined, + }); + }, + ); + if (!cancelled) { + serverKeepAliveTimer = globalThis.setInterval( + () => { + void touchServerTranscoding( + readySession.statusUrl, + controller.signal, + ).catch(() => undefined); + }, + 5 * 60 * 1000, + ); } - releasePreparedMedia = transcoded.release; - setSkippedSubtitleCount(transcoded.skippedSubtitleCount); - setPlaybackMode("transcoded"); - setPlaybackUrl(transcoded.url); - setSubtitles([ - ...subtitleLinks, - ...transcoded.subtitles.map((subtitle) => ({ - ...subtitle, - source: "embedded" as const, - })), - ]); - setSubtitleDiscoveryComplete(true); - setMkvStatus(null); } catch (error) { if (cancelled || isAbortError(error)) return; const message = i18n.t( @@ -463,6 +525,10 @@ export const PlayerPage: React.FC = () => { cancelled = true; controller.abort(); releasePreparedMedia?.(); + if (serverKeepAliveTimer !== null) { + globalThis.clearInterval(serverKeepAliveTimer); + } + if (cancelServerUrl) releaseServerTranscoding(cancelServerUrl); }; }, [ animationId, @@ -613,9 +679,39 @@ export const PlayerPage: React.FC = () => { ? "ja" : "en"; + let hls: Hls | null = null; const art = new Artplayer({ container: playerContainerRef.current, url: playbackUrl, + type: playbackMode === "hls" ? "m3u8" : undefined, + customType: + playbackMode === "hls" + ? { + m3u8: (video: HTMLVideoElement, url: string) => { + if (!Hls.isSupported()) { + video.src = url; + return; + } + hls = new Hls({ + backBufferLength: 90, + maxBufferLength: 30, + manifestLoadingMaxRetry: 6, + levelLoadingMaxRetry: 6, + fragLoadingMaxRetry: 6, + }); + hls.on(Hls.Events.ERROR, (_event, data) => { + if (!data.fatal || !hls) return; + if (data.type === Hls.ErrorTypes.NETWORK_ERROR) { + hls.startLoad(); + } else if (data.type === Hls.ErrorTypes.MEDIA_ERROR) { + hls.recoverMediaError(); + } + }); + hls.loadSource(url); + hls.attachMedia(video); + }, + } + : undefined, proxy: playbackMode === "mkvProxy" ? artplayerProxyMediabunny({ @@ -661,27 +757,40 @@ export const PlayerPage: React.FC = () => { captionsRendererRef.current = captionsRenderer; } + const applyInitialSeek = () => { + const context = contextRef.current; + if (!context || initialSeekAppliedRef.current) return; + const resumeAt = context.state?.positionSeconds ?? 0; + const duration = art.duration; + if ( + playbackMode === "hls" && + !context.state?.isWatched && + resumeAt >= 5 && + (!Number.isFinite(duration) || resumeAt >= duration - 10) + ) { + // The event playlist grows while FFmpeg works. Wait for the requested + // timestamp to appear instead of discarding the cross-device resume. + return; + } + if ( + !context.state?.isWatched && + resumeAt >= 5 && + Number.isFinite(duration) && + resumeAt < duration - 10 + ) { + art.currentTime = resumeAt; + art.notice.show = i18n.t("player:progress.resumed", { + time: new Date(resumeAt * 1000).toISOString().slice(11, 19), + }); + lastSyncedTimeRef.current = resumeAt; + } + initialSeekAppliedRef.current = true; + }; + const onLoadedMetadata = () => { const context = contextRef.current; if (!context) return; - - if (!initialSeekAppliedRef.current) { - const resumeAt = context.state?.positionSeconds ?? 0; - const duration = art.duration; - if ( - !context.state?.isWatched && - resumeAt >= 5 && - Number.isFinite(duration) && - resumeAt < duration - 10 - ) { - art.currentTime = resumeAt; - art.notice.show = i18n.t("player:progress.resumed", { - time: new Date(resumeAt * 1000).toISOString().slice(11, 19), - }); - lastSyncedTimeRef.current = resumeAt; - } - initialSeekAppliedRef.current = true; - } + applyInitialSeek(); const discoveredTracks = readAudioTracks( art.video as VideoWithAudioTracks, @@ -741,6 +850,7 @@ export const PlayerPage: React.FC = () => { const onBeforeUnload = () => persistCurrentProgressRef.current(true, true); art.on("video:loadedmetadata", onLoadedMetadata); + art.on("video:durationchange", applyInitialSeek); art.on("video:timeupdate", onTimeUpdate); art.on("video:pause", onPause); art.on("video:seeked", onSeeked); @@ -754,6 +864,7 @@ export const PlayerPage: React.FC = () => { window.removeEventListener("beforeunload", onBeforeUnload); captionsRenderer?.destroy(); captionsOverlay?.remove(); + hls?.destroy(); if (captionsRendererRef.current === captionsRenderer) { captionsRendererRef.current = null; } @@ -968,7 +1079,13 @@ export const PlayerPage: React.FC = () => { mkvStatus?.progress == null ? null : Math.round(Math.min(1, Math.max(0, mkvStatus.progress)) * 100); - const mkvStatusLabel = mkvStatus ? t(`mkv.stages.${mkvStatus.stage}`) : null; + const mkvStatusLabel = mkvStatus + ? t(`mkv.stages.${mkvStatus.stage}`, { + position: mkvStatus.queuePosition ?? 1, + }) + : null; + const mkvSpeedLabel = + mkvStatus?.speed == null ? null : `${mkvStatus.speed.toFixed(2)}×`; return ( @@ -985,6 +1102,7 @@ export const PlayerPage: React.FC = () => {

{mkvStatusLabel} {mkvProgressPercent == null ? "" : ` · ${mkvProgressPercent}%`} + {mkvSpeedLabel ? ` · ${mkvSpeedLabel}` : ""}

{mkvProgressPercent == null ? null : (
@@ -996,7 +1114,11 @@ export const PlayerPage: React.FC = () => { )} {mkvStatus?.stage === "probing" ? null : (

- {t("mkv.transcodeNotice")} + {t( + mkvStatus?.stage === "extractingSubtitles" + ? "mkv.subtitleNotice" + : "mkv.serverNotice", + )}

)}
@@ -1033,14 +1155,21 @@ export const PlayerPage: React.FC = () => { {t( playbackMode === "mkvProxy" ? "mkv.mode.demuxed" - : "mkv.mode.transcoded", + : serverStrategy === "remux" + ? "mkv.mode.serverRemuxed" + : "mkv.mode.serverTranscoded", )} ) : null} + {serverCacheHit ? ( + + {t("mkv.cacheHit")} + + ) : null}

{mkvStatusLabel - ? `${mkvStatusLabel}${mkvProgressPercent == null ? "" : ` · ${mkvProgressPercent}%`}` + ? `${mkvStatusLabel}${mkvProgressPercent == null ? "" : ` · ${mkvProgressPercent}%`}${mkvSpeedLabel ? ` · ${mkvSpeedLabel}` : ""}` : playbackMode === "mkvProxy" && mkvProbe ? t("mkv.codecSummary", { video: mkvProbe.videoCodec, diff --git a/SecondDimensionWatcherReDive.Client/src/playback/mkv/serverTranscoding.test.ts b/SecondDimensionWatcherReDive.Client/src/playback/mkv/serverTranscoding.test.ts new file mode 100644 index 0000000..c64390b --- /dev/null +++ b/SecondDimensionWatcherReDive.Client/src/playback/mkv/serverTranscoding.test.ts @@ -0,0 +1,84 @@ +import { + ServerTranscodingSession, + watchServerTranscoding, +} from "../serverTranscoding"; + +type TestCallback = () => void | Promise; +type TestFunction = (name: string, callback: TestCallback) => void; + +declare const require: (specifier: string) => unknown; + +const { rejects, strictEqual } = require("node:assert") as { + rejects: ( + promise: Promise, + check: (error: unknown) => boolean, + ) => Promise; + strictEqual: (actual: unknown, expected: unknown) => void; +}; +const { describe, it } = require("node:test") as { + describe: TestFunction; + it: TestFunction; +}; + +const createSession = ( + state: ServerTranscodingSession["state"], +): ServerTranscodingSession => ({ + sessionId: "session", + state, + strategy: state === "queued" ? null : "remux", + isPlayable: state === "ready", + cacheHit: false, + progress: null, + speed: null, + queuePosition: state === "queued" ? 1 : null, + error: state === "failed" ? "fixture failure" : null, + videoCodec: null, + audioCodec: null, + statusUrl: "/status", + cancelUrl: "/cancel", + playbackUrl: state === "ready" ? "/media.m3u8" : null, + subtitles: [], + unsupportedSubtitleCount: 0, +}); + +describe("watchServerTranscoding", () => { + it("returns an already-ready cache entry without polling", async () => { + const controller = new AbortController(); + let updates = 0; + const result = await watchServerTranscoding( + createSession("ready"), + controller.signal, + () => { + updates += 1; + }, + ); + + strictEqual(result.state, "ready"); + strictEqual(updates, 1); + }); + + it("surfaces terminal server failures", async () => { + await rejects( + watchServerTranscoding( + createSession("failed"), + new AbortController().signal, + () => undefined, + ), + (error) => error instanceof Error && error.message === "fixture failure", + ); + }); + + it("aborts queue polling when the player is closed", async () => { + const controller = new AbortController(); + const pending = watchServerTranscoding( + createSession("queued"), + controller.signal, + () => controller.abort(), + ); + + await rejects( + pending, + (error) => error instanceof DOMException && error.name === "AbortError", + ); + }); +}); diff --git a/SecondDimensionWatcherReDive.Client/src/playback/mkv/transcoder.ts b/SecondDimensionWatcherReDive.Client/src/playback/mkv/transcoder.ts deleted file mode 100644 index d1f6742..0000000 --- a/SecondDimensionWatcherReDive.Client/src/playback/mkv/transcoder.ts +++ /dev/null @@ -1,397 +0,0 @@ -import { FFFSType, FFmpeg } from "@ffmpeg/ffmpeg"; -import ffmpegCoreUrl from "url:@ffmpeg/core"; -import ffmpegWasmUrl from "url:@ffmpeg/core/wasm"; - -export type MkvTranscodeStage = - | "loadingTranscoder" - | "downloading" - | "readingTracks" - | "convertingSubtitles" - | "transcodingVideo" - | "finalizing"; - -export interface MkvTranscodeUpdate { - stage: MkvTranscodeStage; - progress?: number; -} - -export interface TranscodedMkvSubtitle { - path: string; - virtualPath: string; - language: string | null; - label: string; - format: "vtt"; - url: string; -} - -export interface TranscodedMkvResult { - url: string; - videoCodec: string; - audioCodec: string | null; - subtitles: TranscodedMkvSubtitle[]; - skippedSubtitleCount: number; - release: () => void; -} - -export interface MkvTranscodeOptions { - /** Preserve a browser-decodable video stream when only audio needs conversion. */ - copyVideo?: boolean; -} - -interface ProbeStream { - index: number; - codec_name?: string; - codec_type?: "video" | "audio" | "subtitle" | string; - disposition?: { - attached_pic?: number; - default?: number; - forced?: number; - }; - tags?: { - language?: string; - title?: string; - }; -} - -interface ProbeResult { - streams?: ProbeStream[]; -} - -interface ExtractTextSubtitlesResult { - subtitles: TranscodedMkvSubtitle[]; - skippedCount: number; -} - -const TEXT_SUBTITLE_CODECS = new Set([ - "ass", - "jacosub", - "microdvd", - "mov_text", - "mpl2", - "realtext", - "sami", - "ssa", - "subrip", - "subviewer", - "subviewer1", - "text", - "vplayer", - "webvtt", -]); - -const abortError = (): DOMException => - new DOMException("The operation was aborted", "AbortError"); - -const clampProgress = (value: number): number => - Math.min(1, Math.max(0, Number.isFinite(value) ? value : 0)); - -const uint8ArrayBuffer = (value: Uint8Array): ArrayBuffer => - value.buffer.slice( - value.byteOffset, - value.byteOffset + value.byteLength, - ) as ArrayBuffer; - -const fetchBlobWithProgress = async ( - url: string, - signal: AbortSignal, - onProgress: (progress?: number) => void, -): Promise => { - const response = await fetch(url, { signal }); - if (!response.ok) { - throw new Error(`Unable to download MKV (${response.status})`); - } - - const total = Number(response.headers.get("content-length")); - if (!response.body) { - const blob = await response.blob(); - onProgress(1); - return blob; - } - - const reader = response.body.getReader(); - let loaded = 0; - const stream = new ReadableStream({ - async pull(controller) { - if (signal.aborted) { - await reader.cancel(); - controller.error(abortError()); - return; - } - const { done, value } = await reader.read(); - if (done) { - controller.close(); - onProgress(1); - return; - } - loaded += value.byteLength; - onProgress( - Number.isFinite(total) && total > 0 ? loaded / total : undefined, - ); - controller.enqueue(value); - }, - cancel(reason) { - return reader.cancel(reason); - }, - }); - - return await new Response(stream, { - headers: { - "Content-Type": - response.headers.get("content-type") ?? "video/x-matroska", - }, - }).blob(); -}; - -const subtitleLabel = (stream: ProbeStream, ordinal: number): string => { - const title = stream.tags?.title?.trim(); - const language = stream.tags?.language?.trim(); - if (title) return title; - if (language) return `${language.toUpperCase()} · Embedded`; - return `Embedded subtitle ${ordinal}`; -}; - -const extractTextSubtitles = async ( - ffmpeg: FFmpeg, - inputPath: string, - streams: ProbeStream[], - signal: AbortSignal, -): Promise => { - const subtitles: TranscodedMkvSubtitle[] = []; - let skippedCount = 0; - - // Convert tracks independently. A malformed or nominally text subtitle must - // not prevent the audio/video fallback from producing playable media. - for (const [ordinal, stream] of streams.entries()) { - const outputPath = `/subtitle-${stream.index}.vtt`; - try { - const exitCode = await ffmpeg.exec( - [ - "-i", - inputPath, - "-map", - `0:${stream.index}`, - "-c:s", - "webvtt", - outputPath, - ], - -1, - { signal }, - ); - if (exitCode !== 0) { - skippedCount += 1; - continue; - } - - const data = await ffmpeg.readFile(outputPath, undefined, { signal }); - if (typeof data === "string") { - skippedCount += 1; - continue; - } - - const url = URL.createObjectURL( - new Blob([uint8ArrayBuffer(data)], { type: "text/vtt;charset=utf-8" }), - ); - subtitles.push({ - path: `__mkv_subtitle_${stream.index}`, - virtualPath: `mkv://subtitle/${stream.index}`, - language: stream.tags?.language ?? null, - label: subtitleLabel(stream, ordinal + 1), - format: "vtt", - url, - }); - } catch { - if (signal.aborted) throw abortError(); - skippedCount += 1; - } - } - - return { subtitles, skippedCount }; -}; - -/** - * Last-resort software conversion for codecs that WebCodecs cannot decode. - * WORKERFS avoids copying the input into the WebAssembly heap; the output is - * still materialized as a Blob because native playback needs a seekable file. - */ -export const transcodeMkvForBrowser = async ( - sourceUrl: string, - signal: AbortSignal, - onUpdate: (update: MkvTranscodeUpdate) => void, - options: MkvTranscodeOptions = {}, -): Promise => { - if (signal.aborted) throw abortError(); - - const ffmpeg = new FFmpeg(); - const createdUrls: string[] = []; - const recentLogs: string[] = []; - const logListener = ({ message }: { message: string }) => { - recentLogs.push(message); - if (recentLogs.length > 8) recentLogs.shift(); - }; - const conversionError = (message: string): Error => - new Error( - recentLogs.length > 0 ? `${message}: ${recentLogs.join(" | ")}` : message, - ); - ffmpeg.on("log", logListener); - let mounted = false; - const onAbort = () => ffmpeg.terminate(); - signal.addEventListener("abort", onAbort, { once: true }); - - try { - onUpdate({ stage: "loadingTranscoder" }); - await ffmpeg.load( - { coreURL: ffmpegCoreUrl, wasmURL: ffmpegWasmUrl }, - { signal }, - ); - - onUpdate({ stage: "downloading", progress: 0 }); - const sourceBlob = await fetchBlobWithProgress( - sourceUrl, - signal, - (progress) => onUpdate({ stage: "downloading", progress }), - ); - - await ffmpeg.createDir("/source", { signal }); - mounted = await ffmpeg.mount( - FFFSType.WORKERFS, - { blobs: [{ name: "episode.mkv", data: sourceBlob }] }, - "/source", - ); - if (!mounted) { - throw conversionError("Unable to mount the MKV in FFmpeg"); - } - const inputPath = "/source/episode.mkv"; - - onUpdate({ stage: "readingTracks" }); - const probePath = "/probe.json"; - const probeExitCode = await ffmpeg.ffprobe( - [ - "-v", - "error", - "-show_streams", - "-of", - "json", - inputPath, - "-o", - probePath, - ], - -1, - { signal }, - ); - let probeData: string | Uint8Array; - try { - probeData = await ffmpeg.readFile(probePath, "utf8", { signal }); - } catch { - throw conversionError( - `Unable to inspect MKV tracks (exit ${probeExitCode})`, - ); - } - const probe = JSON.parse(String(probeData)) as ProbeResult; - const streams = probe.streams ?? []; - const videoStream = streams.find( - (stream) => - stream.codec_type === "video" && stream.disposition?.attached_pic !== 1, - ); - if (!videoStream) throw new Error("The MKV file has no video track"); - const audioStream = streams.find((stream) => stream.codec_type === "audio"); - const subtitleStreams = streams.filter( - (stream) => stream.codec_type === "subtitle", - ); - const textSubtitleStreams = subtitleStreams.filter((stream) => - TEXT_SUBTITLE_CODECS.has(stream.codec_name?.toLowerCase() ?? ""), - ); - - onUpdate({ stage: "convertingSubtitles" }); - const subtitleExtraction = await extractTextSubtitles( - ffmpeg, - inputPath, - textSubtitleStreams, - signal, - ); - const subtitles = subtitleExtraction.subtitles; - createdUrls.push(...subtitles.map((subtitle) => subtitle.url)); - - onUpdate({ stage: "transcodingVideo", progress: 0 }); - recentLogs.length = 0; - const progressListener = ({ progress }: { progress: number }) => { - onUpdate({ - stage: "transcodingVideo", - progress: clampProgress(progress), - }); - }; - ffmpeg.on("progress", progressListener); - const outputPath = "/episode-browser.mp4"; - const transcodeExitCode = await ffmpeg.exec( - [ - "-i", - inputPath, - "-map", - `0:${videoStream.index}`, - ...(audioStream ? ["-map", `0:${audioStream.index}`] : []), - "-sn", - ...(options.copyVideo - ? ["-c:v", "copy"] - : [ - "-c:v", - "libx264", - "-preset", - "ultrafast", - "-crf", - "23", - "-pix_fmt", - "yuv420p", - ]), - ...(audioStream ? ["-c:a", "aac", "-b:a", "192k", "-ac", "2"] : []), - "-movflags", - "+faststart", - "-max_muxing_queue_size", - "1024", - outputPath, - ], - -1, - { signal }, - ); - ffmpeg.off("progress", progressListener); - if (transcodeExitCode !== 0) { - throw conversionError("Unable to convert the MKV video stream"); - } - - onUpdate({ stage: "finalizing" }); - const outputData = await ffmpeg.readFile(outputPath, undefined, { signal }); - if (typeof outputData === "string") { - throw new Error("FFmpeg returned an invalid video payload"); - } - const videoUrl = URL.createObjectURL( - new Blob([uint8ArrayBuffer(outputData)], { type: "video/mp4" }), - ); - createdUrls.push(videoUrl); - - let released = false; - return { - url: videoUrl, - videoCodec: videoStream.codec_name ?? "unknown", - audioCodec: audioStream?.codec_name ?? null, - subtitles, - skippedSubtitleCount: - subtitleStreams.length - - textSubtitleStreams.length + - subtitleExtraction.skippedCount, - release: () => { - if (released) return; - released = true; - createdUrls.forEach((url) => URL.revokeObjectURL(url)); - }, - }; - } catch (error) { - createdUrls.forEach((url) => URL.revokeObjectURL(url)); - if (signal.aborted) throw abortError(); - throw error; - } finally { - signal.removeEventListener("abort", onAbort); - ffmpeg.off("log", logListener); - if (mounted && !signal.aborted && ffmpeg.loaded) { - await ffmpeg.unmount("/source").catch(() => undefined); - } - ffmpeg.terminate(); - } -}; diff --git a/SecondDimensionWatcherReDive.Client/src/playback/serverTranscoding.ts b/SecondDimensionWatcherReDive.Client/src/playback/serverTranscoding.ts new file mode 100644 index 0000000..075c8cb --- /dev/null +++ b/SecondDimensionWatcherReDive.Client/src/playback/serverTranscoding.ts @@ -0,0 +1,121 @@ +import fetcher from "../auth/httpClient"; + +export type ServerTranscodingState = + "queued" | "probing" | "transcoding" | "ready" | "failed" | "canceled"; + +export type ServerTranscodingStrategy = "direct" | "remux" | "transcode"; + +export interface ServerTranscodingSubtitle { + path: string; + virtualPath: string; + language: string | null; + label: string; + format: "vtt"; + url: string; +} + +export interface ServerTranscodingSession { + sessionId: string; + state: ServerTranscodingState; + strategy: ServerTranscodingStrategy | null; + isPlayable: boolean; + cacheHit: boolean; + progress: number | null; + speed: number | null; + queuePosition: number | null; + error: string | null; + videoCodec: string | null; + audioCodec: string | null; + statusUrl: string; + cancelUrl: string; + playbackUrl: string | null; + subtitles: ServerTranscodingSubtitle[]; + unsupportedSubtitleCount: number; +} + +export interface PrepareServerTranscodingRequest { + id: string; + path: string; + quality?: "auto" | "720p" | "1080p"; + audioLanguage?: string | null; + audioTrackLabel?: string | null; + subtitleLanguage?: string | null; + subtitleTrackLabel?: string | null; +} + +const abortError = (): DOMException => + new DOMException("The operation was aborted", "AbortError"); + +const pollDelay = async ( + milliseconds: number, + signal: AbortSignal, +): Promise => { + if (signal.aborted) throw abortError(); + await new Promise((resolve, reject) => { + const onAbort = () => { + globalThis.clearTimeout(timeout); + reject(abortError()); + }; + const timeout = globalThis.setTimeout(() => { + signal.removeEventListener("abort", onAbort); + resolve(); + }, milliseconds); + signal.addEventListener("abort", onAbort, { once: true }); + }); +}; + +export const prepareServerTranscoding = async ( + request: PrepareServerTranscodingRequest, + signal: AbortSignal, +): Promise => + await fetcher("/api/transcoding/prepare", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(request), + signal, + }); + +export const watchServerTranscoding = async ( + initial: ServerTranscodingSession, + signal: AbortSignal, + onUpdate: (session: ServerTranscodingSession) => void, +): Promise => { + let current = initial; + let transientFailures = 0; + while (true) { + if (signal.aborted) throw abortError(); + onUpdate(current); + if (current.state === "ready") return current; + if (current.state === "failed" || current.state === "canceled") { + throw new Error(current.error || `Server transcoding ${current.state}`); + } + + await pollDelay(current.state === "queued" ? 1000 : 750, signal); + try { + const response = await fetch(current.statusUrl, { signal }); + if (!response.ok) + throw new Error(`Transcoding status ${response.status}`); + current = (await response.json()) as ServerTranscodingSession; + transientFailures = 0; + } catch (error) { + if (signal.aborted) throw abortError(); + transientFailures += 1; + if (transientFailures >= 3) throw error; + } + } +}; + +export const touchServerTranscoding = async ( + statusUrl: string, + signal: AbortSignal, +): Promise => { + const response = await fetch(statusUrl, { signal }); + if (!response.ok) throw new Error(`Transcoding status ${response.status}`); + await response.json(); +}; + +export const releaseServerTranscoding = (cancelUrl: string): void => { + void fetch(cancelUrl, { method: "DELETE", keepalive: true }).catch( + () => undefined, + ); +}; diff --git a/SecondDimensionWatcherReDive.Client/src/types/parcel-assets.d.ts b/SecondDimensionWatcherReDive.Client/src/types/parcel-assets.d.ts index 7f2ed7d..d80e98f 100644 --- a/SecondDimensionWatcherReDive.Client/src/types/parcel-assets.d.ts +++ b/SecondDimensionWatcherReDive.Client/src/types/parcel-assets.d.ts @@ -1,15 +1,5 @@ declare module "*.css"; -declare module "url:@ffmpeg/core" { - const url: string; - export default url; -} - -declare module "url:@ffmpeg/core/wasm" { - const url: string; - export default url; -} - declare module "bundle-text:*" { const source: string; export default source; diff --git a/SecondDimensionWatcherReDive.Client/yarn.lock b/SecondDimensionWatcherReDive.Client/yarn.lock index dbd57a3..c4f6f4c 100644 --- a/SecondDimensionWatcherReDive.Client/yarn.lock +++ b/SecondDimensionWatcherReDive.Client/yarn.lock @@ -330,38 +330,6 @@ __metadata: languageName: node linkType: hard -"@ffmpeg/core@npm:^0.12.10": - version: 0.12.10 - resolution: "@ffmpeg/core@npm:0.12.10" - checksum: 10/1385a695b5c3f2ceac75f59a4413c7e529e70b5eef9813d92fb929571e14124c26b8f760bfa89aefc442dc0bbb154c35bfde5a3b31b90a063a6507f7640ebef6 - languageName: node - linkType: hard - -"@ffmpeg/ffmpeg@npm:0.12.15": - version: 0.12.15 - resolution: "@ffmpeg/ffmpeg@npm:0.12.15" - dependencies: - "@ffmpeg/types": "npm:^0.12.4" - checksum: 10/8969f3e99be5ba318c6b2aa635703687d8c534d59a295042af28a395c5aa29d6338627200d4d81c339056c93b7e7782246450dfdebf9392081c3753a3d41028f - languageName: node - linkType: hard - -"@ffmpeg/ffmpeg@patch:@ffmpeg/ffmpeg@npm%3A0.12.15#~/.yarn/patches/@ffmpeg-ffmpeg-npm-0.12.15-ef17f09f1b.patch": - version: 0.12.15 - resolution: "@ffmpeg/ffmpeg@patch:@ffmpeg/ffmpeg@npm%3A0.12.15#~/.yarn/patches/@ffmpeg-ffmpeg-npm-0.12.15-ef17f09f1b.patch::version=0.12.15&hash=cbebad" - dependencies: - "@ffmpeg/types": "npm:^0.12.4" - checksum: 10/124f2a16e18f6dc7e9654f03a7ff05ec4a3f4459ca5e5507a7e5aa6801cef70076eadda43465e5e18443784fc35b4a77466ca8bad15a38e0ff9bbd237642d719 - languageName: node - linkType: hard - -"@ffmpeg/types@npm:^0.12.4": - version: 0.12.4 - resolution: "@ffmpeg/types@npm:0.12.4" - checksum: 10/8b898163e79945d2eba26f2a46f35a22d00296cb53424f16118060f28c36f694d74d48c49f6570f2974320dc089b7aa9f64c3a0fde45109ba125aa9e3c0926db - languageName: node - linkType: hard - "@floating-ui/core@npm:^1.8.0": version: 1.8.0 resolution: "@floating-ui/core@npm:1.8.0" @@ -404,8 +372,6 @@ __metadata: version: 0.0.0-use.local resolution: "@hcgstudio/sdwr-client@workspace:." dependencies: - "@ffmpeg/core": "npm:^0.12.10" - "@ffmpeg/ffmpeg": "patch:@ffmpeg/ffmpeg@npm%3A0.12.15#~/.yarn/patches/@ffmpeg-ffmpeg-npm-0.12.15-ef17f09f1b.patch" "@parcel/core": "npm:^2.16.4" "@parcel/transformer-inline-string": "npm:2.16.4" "@radix-ui/react-dialog": "npm:^1.1.23" @@ -423,6 +389,7 @@ __metadata: clsx: "npm:^2.1.1" color-convert: "npm:^3.1.3" dayjs: "npm:^1.11.23" + hls.js: "npm:^1.7.1" http-proxy-middleware: "npm:^4.2.0" i18next: "npm:^26.4.0" i18next-browser-languagedetector: "npm:^8.2.1" @@ -3692,6 +3659,13 @@ __metadata: languageName: node linkType: hard +"hls.js@npm:^1.7.1": + version: 1.7.1 + resolution: "hls.js@npm:1.7.1" + checksum: 10/5bf1c5ba1cbb82a4274f55afe62c361137bad7a60e6c7a3cfd881ca47a6883357b660c080982a7393c840bcd710c6ca97ad12094a2db72d0c0dbe1cc9e48edb1 + languageName: node + linkType: hard + "hpagent@npm:^1.2.0": version: 1.2.0 resolution: "hpagent@npm:1.2.0" diff --git a/SecondDimensionWatcherReDive.IntegrationTest/Helpers/FakeTranscodingService.cs b/SecondDimensionWatcherReDive.IntegrationTest/Helpers/FakeTranscodingService.cs new file mode 100644 index 0000000..17cb5d0 --- /dev/null +++ b/SecondDimensionWatcherReDive.IntegrationTest/Helpers/FakeTranscodingService.cs @@ -0,0 +1,103 @@ +using SecondDimensionWatcherReDive.Services.Transcoding; + +namespace SecondDimensionWatcherReDive.IntegrationTest.Helpers; + +internal sealed class FakeTranscodingService : IHlsTranscodingService +{ + public Guid SessionId { get; private set; } = Guid.NewGuid(); + public string Token { get; private set; } = "integration-transcoding-token"; + + public void Reset() + { + SessionId = Guid.NewGuid(); + Token = "integration-transcoding-token"; + } + + public Task PrepareAsync( + Guid animationInfoId, + string? relativePath, + TranscodingSelection selection, + CancellationToken cancellationToken) + => Task.FromResult(CreateStatus()); + + public Task GetStatusAsync( + Guid sessionId, + string accessToken, + CancellationToken cancellationToken) + => Task.FromResult(IsValid(sessionId, accessToken) ? CreateStatus() : null); + + public Task GetPlaylistAsync( + Guid sessionId, + string accessToken, + CancellationToken cancellationToken) + => Task.FromResult(IsValid(sessionId, accessToken) + ? "#EXTM3U\n#EXTINF:6,\nsegment-000000.ts\n#EXT-X-ENDLIST\n" + : null); + + public Task OpenSegmentAsync( + Guid sessionId, + string accessToken, + string fileName, + CancellationToken cancellationToken) + => Task.FromResult( + IsValid(sessionId, accessToken) && fileName == "segment-000000.ts" + ? new TranscodingContent( + new MemoryStream([1, 2, 3], writable: false), + "video/mp2t", + fileName, + 3, + DateTimeOffset.UnixEpoch) + : null); + + public Task OpenSubtitleAsync( + Guid sessionId, + string accessToken, + string fileName, + CancellationToken cancellationToken) + => Task.FromResult(null); + + public Task OpenDirectAsync( + Guid sessionId, + string accessToken, + CancellationToken cancellationToken) + => Task.FromResult(null); + + public Task CancelAsync( + Guid sessionId, + string accessToken, + CancellationToken cancellationToken) + => Task.FromResult(IsValid(sessionId, accessToken)); + + public Task GetMetricsAsync(CancellationToken cancellationToken) + => Task.FromResult(new TranscodingMetricsSnapshot( + 0, + 0, + 4, + 1, + 1, + 2, + 4096, + 0.75, + 3.2, + 0.2)); + + private bool IsValid(Guid sessionId, string accessToken) + => sessionId == SessionId && accessToken == Token; + + private TranscodingSessionStatus CreateStatus() + => new( + SessionId, + Token, + TranscodingJobState.Ready, + TranscodingStrategy.Remux, + true, + true, + 1, + 3.2, + null, + null, + "h264", + "aac", + [], + 0); +} diff --git a/SecondDimensionWatcherReDive.IntegrationTest/Transcoding/TranscodingApiTests.cs b/SecondDimensionWatcherReDive.IntegrationTest/Transcoding/TranscodingApiTests.cs new file mode 100644 index 0000000..419ca35 --- /dev/null +++ b/SecondDimensionWatcherReDive.IntegrationTest/Transcoding/TranscodingApiTests.cs @@ -0,0 +1,72 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; + +namespace SecondDimensionWatcherReDive.IntegrationTest.Transcoding; + +[TestClass] +public sealed class TranscodingApiTests +{ + private WebDavWebApplicationFactory _factory = null!; + + [TestInitialize] + public void Setup() + { + _factory = new WebDavWebApplicationFactory(); + _factory.ResetState(); + } + + [TestCleanup] + public void Cleanup() => _factory.Dispose(); + + [TestMethod] + public async Task PrepareRequiresJwtButTokenizedHlsResourcesAreAnonymous() + { + using var anonymous = _factory.CreateUnauthenticatedClient(); + using var unauthorized = await anonymous.PostAsJsonAsync( + "/api/transcoding/prepare", + new { id = Guid.NewGuid(), path = "episode.mkv", quality = "auto" }); + Assert.AreEqual(HttpStatusCode.Unauthorized, unauthorized.StatusCode); + + using var jwt = _factory.CreateJwtClient(); + using var prepared = await jwt.PostAsJsonAsync( + "/api/transcoding/prepare", + new { id = Guid.NewGuid(), path = "episode.mkv", quality = "auto" }); + Assert.AreEqual(HttpStatusCode.OK, prepared.StatusCode); + using var payload = JsonDocument.Parse(await prepared.Content.ReadAsStringAsync()); + var playbackUrl = payload.RootElement.GetProperty("playbackUrl").GetString(); + var statusUrl = payload.RootElement.GetProperty("statusUrl").GetString(); + Assert.IsNotNull(playbackUrl); + Assert.IsNotNull(statusUrl); + + using var status = await anonymous.GetAsync(statusUrl); + Assert.AreEqual(HttpStatusCode.OK, status.StatusCode); + using var playlist = await anonymous.GetAsync(playbackUrl); + Assert.AreEqual(HttpStatusCode.OK, playlist.StatusCode); + Assert.AreEqual("application/vnd.apple.mpegurl", playlist.Content.Headers.ContentType?.MediaType); + var segmentUrl = (await playlist.Content.ReadAsStringAsync()) + .Split('\n', StringSplitOptions.RemoveEmptyEntries) + .Single(line => !line.StartsWith('#')); + StringAssert.Contains(segmentUrl, _factory.TranscodingService.Token); + using var segment = await anonymous.GetAsync(segmentUrl); + Assert.AreEqual(HttpStatusCode.OK, segment.StatusCode); + CollectionAssert.AreEqual(new byte[] { 1, 2, 3 }, await segment.Content.ReadAsByteArrayAsync()); + } + + [TestMethod] + public async Task InvalidSessionTokenIsRejectedAndMetricsRequireJwt() + { + using var anonymous = _factory.CreateUnauthenticatedClient(); + using var invalid = await anonymous.GetAsync( + $"/api/transcoding/sessions/{_factory.TranscodingService.SessionId}?token=wrong"); + Assert.AreEqual(HttpStatusCode.NotFound, invalid.StatusCode); + using var unauthorizedMetrics = await anonymous.GetAsync("/api/transcoding/metrics"); + Assert.AreEqual(HttpStatusCode.Unauthorized, unauthorizedMetrics.StatusCode); + + using var jwt = _factory.CreateJwtClient(); + using var metrics = await jwt.GetAsync("/api/transcoding/metrics"); + Assert.AreEqual(HttpStatusCode.OK, metrics.StatusCode); + using var payload = JsonDocument.Parse(await metrics.Content.ReadAsStringAsync()); + Assert.AreEqual(0.2, payload.RootElement.GetProperty("failureRate").GetDouble()); + } +} diff --git a/SecondDimensionWatcherReDive.IntegrationTest/WebDavWebApplicationFactory.cs b/SecondDimensionWatcherReDive.IntegrationTest/WebDavWebApplicationFactory.cs index ae5cf79..379a089 100644 --- a/SecondDimensionWatcherReDive.IntegrationTest/WebDavWebApplicationFactory.cs +++ b/SecondDimensionWatcherReDive.IntegrationTest/WebDavWebApplicationFactory.cs @@ -18,6 +18,7 @@ using SecondDimensionWatcherReDive.Framework.Tasks; using SecondDimensionWatcherReDive.IntegrationTest.Helpers; using SecondDimensionWatcherReDive.MigrationTasks; +using SecondDimensionWatcherReDive.Services.Transcoding; using FileMapping = SecondDimensionWatcherReDive.Framework.DataRepository.FileMapping; using ApplicationContext = SecondDimensionWatcherReDive.Models.ApplicationContext; @@ -52,6 +53,7 @@ static WebDavWebApplicationFactory() public Mock FileStoreMock { get; } = new(); public Mock FileStoreProviderMock { get; } = new(); public Helpers.FakeFileMappingRepository MappingRepository { get; } + public FakeTranscodingService TranscodingService { get; } = new(); private readonly object _mappingsLock = new(); @@ -122,6 +124,7 @@ protected override void ConfigureWebHost(IWebHostBuilder builder) services.RemoveAll(); services.RemoveAll(); services.RemoveAll(); + services.RemoveAll(); services.AddSingleton(FileStoreMock.Object); services.AddSingleton(FileStoreProviderMock.Object); @@ -130,6 +133,7 @@ protected override void ConfigureWebHost(IWebHostBuilder builder) services.AddSingleton(_ => new FakeWebDavTokenRepository(TestUserName, BCrypt.Net.BCrypt.HashPassword(TestPassword))); services.AddSingleton(); + services.AddSingleton(TranscodingService); }); } @@ -146,6 +150,7 @@ public void ResetState() FileStoreProviderMock .Setup(p => p.GetClient(It.IsAny())) .Returns(FileStoreMock.Object); + TranscodingService.Reset(); } public HttpClient CreateBasicAuthClient(string user = TestUserName, string pass = TestPassword) diff --git a/SecondDimensionWatcherReDive.Test/FfmpegProcessRunnerTests.cs b/SecondDimensionWatcherReDive.Test/FfmpegProcessRunnerTests.cs new file mode 100644 index 0000000..c1a5114 --- /dev/null +++ b/SecondDimensionWatcherReDive.Test/FfmpegProcessRunnerTests.cs @@ -0,0 +1,126 @@ +using System.Diagnostics; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using SecondDimensionWatcherReDive.Services.Transcoding; + +namespace SecondDimensionWatcherReDive.Test; + +[TestClass] +public class FfmpegProcessRunnerTests +{ + [TestMethod] + public async Task ProbeAndGenerateHlsAsync_ProducesProgressivePlaylistFromPipeInput() + { + var root = Path.Combine(Path.GetTempPath(), $"sdw-ffmpeg-test-{Guid.NewGuid():N}"); + Directory.CreateDirectory(root); + try + { + var sourcePath = Path.Combine(root, "sample.mkv"); + await CreateSampleAsync(sourcePath, CancellationToken.None); + var options = Options.Create(new TranscodingOptions + { + FfmpegPath = "ffmpeg", + FfprobePath = "ffprobe", + MaxThreadsPerJob = 1, + SegmentDurationSeconds = 2, + MaxMemoryBytesPerJob = 1024L * 1024 * 1024, + MaxDiskBytesPerJob = 64L * 1024 * 1024 + }); + var runner = new FfmpegProcessRunner( + options, + NullLogger.Instance); + MediaProbe probe; + await using (var source = File.OpenRead(sourcePath)) + probe = await runner.ProbeAsync(source, CancellationToken.None); + var sourceInfo = new FileInfo(sourcePath); + var sourceModel = new TranscodingSource( + Guid.NewGuid(), + Guid.NewGuid(), + "/Anime/Group/sample.mkv", + sourcePath, + "test", + sourceInfo.Name, + sourceInfo.Length, + sourceInfo.LastWriteTimeUtc); + var selection = TranscodingSelection.Create("auto", null, null, null, null); + var plan = TranscodingPlanner.CreatePlan(sourceModel, probe, selection, false); + Assert.AreEqual(TranscodingStrategy.Remux, plan.Strategy); + + var output = Path.Combine(root, "hls"); + Directory.CreateDirectory(output); + var updates = new List(); + FfmpegRunResult result; + await using (var source = File.OpenRead(sourcePath)) + result = await runner.GenerateHlsAsync( + source, + plan, + selection, + output, + useHardwareEncoder: false, + update => updates.Add(update), + CancellationToken.None); + + Assert.AreEqual(0, result.ExitCode, result.ErrorOutput); + Assert.IsTrue(File.Exists(Path.Combine(output, "media.m3u8"))); + Assert.IsTrue(Directory.EnumerateFiles(output, "segment-*.ts").Any()); + Assert.IsTrue(updates.Any(update => update.FirstSegmentReady)); + StringAssert.Contains( + await File.ReadAllTextAsync(Path.Combine(output, "media.m3u8")), + "#EXT-X-ENDLIST"); + IReadOnlyList subtitles; + await using (var source = File.OpenRead(sourcePath)) + subtitles = await runner.ExtractTextSubtitlesAsync( + source, + plan, + output, + CancellationToken.None); + Assert.AreEqual(1, subtitles.Count); + StringAssert.StartsWith( + await File.ReadAllTextAsync(Path.Combine(output, subtitles[0].FileName)), + "WEBVTT"); + } + finally + { + if (Directory.Exists(root)) Directory.Delete(root, recursive: true); + } + } + + private static async Task CreateSampleAsync(string path, CancellationToken cancellationToken) + { + var subtitlePath = Path.ChangeExtension(path, ".srt"); + await File.WriteAllTextAsync( + subtitlePath, + "1\n00:00:00,000 --> 00:00:01,000\nHello from SDW\n", + cancellationToken); + var startInfo = new ProcessStartInfo + { + FileName = "ffmpeg", + UseShellExecute = false, + RedirectStandardError = true, + RedirectStandardOutput = true, + CreateNoWindow = true + }; + foreach (var argument in new[] + { + "-hide_banner", "-loglevel", "error", "-y", + "-f", "lavfi", "-i", "testsrc=size=160x90:rate=10", + "-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000", + "-f", "srt", "-i", subtitlePath, + "-t", "2", + "-map", "0:v:0", "-map", "1:a:0", "-map", "2:s:0", + "-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p", + "-c:a", "aac", "-b:a", "64k", + "-c:s", "srt", + "-f", "matroska", path + }) + startInfo.ArgumentList.Add(argument); + using var process = Process.Start(startInfo) + ?? throw new InvalidOperationException("Unable to start FFmpeg test fixture generation."); + var errorTask = process.StandardError.ReadToEndAsync(cancellationToken); + var outputTask = process.StandardOutput.ReadToEndAsync(cancellationToken); + await process.WaitForExitAsync(cancellationToken); + var error = await errorTask; + await outputTask; + Assert.AreEqual(0, process.ExitCode, error); + } +} diff --git a/SecondDimensionWatcherReDive.Test/HlsTranscodingServiceTests.cs b/SecondDimensionWatcherReDive.Test/HlsTranscodingServiceTests.cs new file mode 100644 index 0000000..e77c747 --- /dev/null +++ b/SecondDimensionWatcherReDive.Test/HlsTranscodingServiceTests.cs @@ -0,0 +1,480 @@ +using Microsoft.AspNetCore.StaticFiles; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Moq; +using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Framework.FileStore; +using SecondDimensionWatcherReDive.Services.Transcoding; + +namespace SecondDimensionWatcherReDive.Test; + +[TestClass] +public class HlsTranscodingServiceTests +{ + [TestMethod] + public async Task PrepareAsync_GeneratesPlayableHlsAndReusesCompletedCache() + { + var runner = new CompletingRunner(); + await using var fixture = await TranscodingFixture.CreateAsync(runner); + + var initial = await fixture.Service.PrepareAsync( + fixture.AnimationInfoId, + "episode.mkv", + TranscodingSelection.Create("auto", "ja", null, "en", null), + CancellationToken.None); + var ready = await WaitForStateAsync( + fixture.Service, + initial, + TranscodingJobState.Ready); + + Assert.IsTrue(ready.IsPlayable); + Assert.AreEqual(TranscodingStrategy.Remux, ready.Strategy); + Assert.AreEqual(1, ready.Subtitles.Count); + Assert.AreEqual(1, runner.GenerateCalls); + StringAssert.Contains( + await fixture.Service.GetPlaylistAsync( + ready.SessionId, + ready.AccessToken, + CancellationToken.None), + "segment-000000.ts"); + var segment = await fixture.Service.OpenSegmentAsync( + ready.SessionId, + ready.AccessToken, + "segment-000000.ts", + CancellationToken.None); + Assert.IsNotNull(segment); + Assert.AreEqual("video/mp2t", segment.ContentType); + await segment.Stream.DisposeAsync(); + + var repeated = await fixture.Service.PrepareAsync( + fixture.AnimationInfoId, + "episode.mkv", + TranscodingSelection.Create("auto", "ja", null, "en", null), + CancellationToken.None); + + Assert.AreEqual(TranscodingJobState.Ready, repeated.State); + Assert.IsTrue(repeated.CacheHit); + Assert.AreEqual(1, runner.GenerateCalls); + await fixture.RestartServiceAsync(); + var afterRestart = await fixture.Service.PrepareAsync( + fixture.AnimationInfoId, + "episode.mkv", + TranscodingSelection.Create("auto", "ja", null, "en", null), + CancellationToken.None); + Assert.AreEqual(TranscodingJobState.Ready, afterRestart.State); + Assert.IsTrue(afterRestart.CacheHit); + Assert.AreEqual(1, runner.GenerateCalls); + var metrics = await fixture.Service.GetMetricsAsync(CancellationToken.None); + Assert.AreEqual(1, metrics.CompletedJobs); + Assert.AreEqual(2, metrics.CacheHits); + Assert.IsTrue(metrics.CacheBytes > 0); + } + + [TestMethod] + public async Task PrepareAsync_SourceVersionChangeDoesNotReuseOldSegments() + { + var runner = new CompletingRunner(); + await using var fixture = await TranscodingFixture.CreateAsync(runner); + + var first = await fixture.Service.PrepareAsync( + fixture.AnimationInfoId, + "episode.mkv", + TranscodingSelection.Create("auto", null, null, null, null), + CancellationToken.None); + await WaitForStateAsync(fixture.Service, first, TranscodingJobState.Ready); + fixture.LastModifiedUtc = fixture.LastModifiedUtc.AddSeconds(1); + + var changed = await fixture.Service.PrepareAsync( + fixture.AnimationInfoId, + "episode.mkv", + TranscodingSelection.Create("auto", null, null, null, null), + CancellationToken.None); + await WaitForStateAsync(fixture.Service, changed, TranscodingJobState.Ready); + + Assert.IsFalse(changed.CacheHit); + Assert.AreEqual(2, runner.GenerateCalls); + } + + [TestMethod] + public async Task PrepareAsync_ConcurrentLimitQueuesAndRejectsOnlyWhenBoundedQueueIsFull() + { + var runner = new BlockingRunner(); + await using var fixture = await TranscodingFixture.CreateAsync( + runner, + queueCapacity: 1, + relativePaths: ["one.mkv", "two.mkv", "three.mkv"]); + + var first = await fixture.Service.PrepareAsync( + fixture.AnimationInfoId, + "one.mkv", + TranscodingSelection.Create("auto", null, null, null, null), + CancellationToken.None); + await runner.Started.Task.WaitAsync(TimeSpan.FromSeconds(3)); + var second = await fixture.Service.PrepareAsync( + fixture.AnimationInfoId, + "two.mkv", + TranscodingSelection.Create("auto", null, null, null, null), + CancellationToken.None); + + Assert.AreEqual(TranscodingJobState.Queued, second.State); + Assert.AreEqual(1, second.QueuePosition); + await Assert.ThrowsExactlyAsync(() => + fixture.Service.PrepareAsync( + fixture.AnimationInfoId, + "three.mkv", + TranscodingSelection.Create("auto", null, null, null, null), + CancellationToken.None)); + var metrics = await fixture.Service.GetMetricsAsync(CancellationToken.None); + Assert.AreEqual(1, metrics.ActiveJobs); + Assert.AreEqual(1, metrics.QueuedJobs); + + Assert.IsTrue(await fixture.Service.CancelAsync( + first.SessionId, + first.AccessToken, + CancellationToken.None)); + Assert.IsTrue(await fixture.Service.CancelAsync( + second.SessionId, + second.AccessToken, + CancellationToken.None)); + using var cleanupTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(3)); + TranscodingMetricsSnapshot afterCancellation; + do + { + afterCancellation = await fixture.Service.GetMetricsAsync(cleanupTimeout.Token); + if (afterCancellation.CanceledJobs == 2 + && !Directory.EnumerateDirectories(fixture.CachePath).Any()) break; + await Task.Delay(10, cleanupTimeout.Token); + } while (true); + Assert.AreEqual(2, afterCancellation.CanceledJobs); + } + + [TestMethod] + public async Task FailedJobDeletesPartialOutputAndReportsFailureRate() + { + await using var fixture = await TranscodingFixture.CreateAsync(new FailingRunner()); + var initial = await fixture.Service.PrepareAsync( + fixture.AnimationInfoId, + "episode.mkv", + TranscodingSelection.Create("auto", null, null, null, null), + CancellationToken.None); + + var failed = await WaitForStateAsync( + fixture.Service, + initial, + TranscodingJobState.Failed); + + StringAssert.Contains(failed.Error, "fixture FFmpeg failure"); + Assert.IsFalse(Directory.EnumerateDirectories(fixture.CachePath).Any()); + var metrics = await fixture.Service.GetMetricsAsync(CancellationToken.None); + Assert.AreEqual(1, metrics.FailedJobs); + Assert.AreEqual(1, metrics.FailureRate); + } + + private static async Task WaitForStateAsync( + IHlsTranscodingService service, + TranscodingSessionStatus session, + TranscodingJobState expected) + { + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (true) + { + var current = await service.GetStatusAsync( + session.SessionId, + session.AccessToken, + timeout.Token); + Assert.IsNotNull(current); + if (current.State == expected) return current; + if (current.State is TranscodingJobState.Failed or TranscodingJobState.Canceled) + Assert.Fail($"Transcoding ended in {current.State}: {current.Error}"); + await Task.Delay(10, timeout.Token); + } + } + + private sealed class CompletingRunner : IFfmpegProcessRunner + { + private int _generateCalls; + public int GenerateCalls => Volatile.Read(ref _generateCalls); + + public Task ProbeAsync(Stream source, CancellationToken cancellationToken) + => Task.FromResult(new MediaProbe( + "matroska", + TimeSpan.FromSeconds(30), + [ + new MediaStreamProbe(0, "video", "h264", null, null, true, false, false), + new MediaStreamProbe(1, "audio", "aac", "jpn", "Japanese", true, false, false), + new MediaStreamProbe(2, "subtitle", "ass", "eng", "English", true, false, false) + ])); + + public async Task GenerateHlsAsync( + Stream source, + TranscodingPlan plan, + TranscodingSelection selection, + string outputDirectory, + bool useHardwareEncoder, + Action onProgress, + CancellationToken cancellationToken) + { + Interlocked.Increment(ref _generateCalls); + await File.WriteAllBytesAsync( + Path.Combine(outputDirectory, "segment-000000.ts"), + [1, 2, 3], + cancellationToken); + await File.WriteAllTextAsync( + Path.Combine(outputDirectory, "media.m3u8"), + "#EXTM3U\n#EXTINF:6,\nsegment-000000.ts\n#EXT-X-ENDLIST\n", + cancellationToken); + onProgress(new FfmpegProgress(30, 2, true)); + return new FfmpegRunResult(0, string.Empty); + } + + public async Task> ExtractTextSubtitlesAsync( + Stream source, + TranscodingPlan plan, + string outputDirectory, + CancellationToken cancellationToken) + { + const string name = "subtitle-2.vtt"; + await File.WriteAllTextAsync( + Path.Combine(outputDirectory, name), + "WEBVTT\n", + cancellationToken); + return [new TranscodingSubtitle(name, "English", "eng", "vtt")]; + } + } + + private sealed class BlockingRunner : IFfmpegProcessRunner + { + public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task ProbeAsync(Stream source, CancellationToken cancellationToken) + => Task.FromResult(new MediaProbe( + "matroska", + TimeSpan.FromSeconds(30), + [ + new MediaStreamProbe(0, "video", "h264", null, null, true, false, false), + new MediaStreamProbe(1, "audio", "aac", null, null, true, false, false) + ])); + + public async Task GenerateHlsAsync( + Stream source, + TranscodingPlan plan, + TranscodingSelection selection, + string outputDirectory, + bool useHardwareEncoder, + Action onProgress, + CancellationToken cancellationToken) + { + await File.WriteAllTextAsync( + Path.Combine(outputDirectory, "partial.tmp"), + "partial", + cancellationToken); + Started.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return new FfmpegRunResult(0, string.Empty); + } + + public Task> ExtractTextSubtitlesAsync( + Stream source, + TranscodingPlan plan, + string outputDirectory, + CancellationToken cancellationToken) + => Task.FromResult>([]); + } + + private sealed class FailingRunner : IFfmpegProcessRunner + { + public Task ProbeAsync(Stream source, CancellationToken cancellationToken) + => Task.FromResult(new MediaProbe( + "matroska", + TimeSpan.FromSeconds(30), + [ + new MediaStreamProbe(0, "video", "h264", null, null, true, false, false), + new MediaStreamProbe(1, "audio", "aac", null, null, true, false, false) + ])); + + public async Task GenerateHlsAsync( + Stream source, + TranscodingPlan plan, + TranscodingSelection selection, + string outputDirectory, + bool useHardwareEncoder, + Action onProgress, + CancellationToken cancellationToken) + { + await File.WriteAllTextAsync( + Path.Combine(outputDirectory, "partial.tmp"), + "partial", + cancellationToken); + return new FfmpegRunResult(1, "fixture FFmpeg failure"); + } + + public Task> ExtractTextSubtitlesAsync( + Stream source, + TranscodingPlan plan, + string outputDirectory, + CancellationToken cancellationToken) + => Task.FromResult>([]); + } + + private sealed class TranscodingFixture : IAsyncDisposable + { + private readonly ServiceProvider _provider; + private readonly TranscodingMetrics _metrics; + private readonly string _cachePath; + private readonly IFfmpegProcessRunner _runner; + private readonly IOptions _options; + + private TranscodingFixture( + ServiceProvider provider, + TranscodingMetrics metrics, + IFfmpegProcessRunner runner, + IOptions options, + HlsTranscodingService service, + string cachePath, + Guid animationInfoId) + { + _provider = provider; + _metrics = metrics; + _runner = runner; + _options = options; + Service = service; + _cachePath = cachePath; + AnimationInfoId = animationInfoId; + } + + public HlsTranscodingService Service { get; private set; } + public Guid AnimationInfoId { get; } + public string CachePath => _cachePath; + public DateTimeOffset LastModifiedUtc { get; set; } = + new(2026, 1, 1, 0, 0, 0, TimeSpan.Zero); + + public static async Task CreateAsync( + IFfmpegProcessRunner runner, + int queueCapacity = 8, + IReadOnlyList? relativePaths = null) + { + var cachePath = Path.Combine(Path.GetTempPath(), $"sdw-transcoding-test-{Guid.NewGuid():N}"); + var animationInfoId = Guid.NewGuid(); + var animation = new Animation(Guid.NewGuid(), "42", "Anime", "Anime", null); + var group = new AnimationGroup(Guid.NewGuid(), "Group"); + var info = new AnimationInfo( + animationInfoId, + "Episode", + string.Empty, + DateTimeOffset.UtcNow, + string.Empty, + string.Empty, + [], + string.Empty, + false, + DateTimeOffset.UnixEpoch, + DateTimeOffset.UnixEpoch, + true, + "test", + "/physical", + 1, + 1, + group, + animation, + true, + 0); + relativePaths ??= ["episode.mkv"]; + var mappings = relativePaths.ToDictionary( + relative => $"/Anime/Group/{relative}", + relative => new FileMapping( + Guid.NewGuid(), + animationInfoId, + $"/Anime/Group/{relative}", + $"/physical/{relative}", + "test")); + + var animationRepository = new Mock(); + animationRepository.Setup(repository => repository.FindByIdWithAnimationAsync( + animationInfoId, + It.IsAny())) + .ReturnsAsync(info); + var mappingRepository = new Mock(); + mappingRepository.Setup(repository => repository.FindByVirtualPathAsync( + It.IsAny(), + It.IsAny())) + .ReturnsAsync((string path, CancellationToken _) => + mappings.GetValueOrDefault(path)); + var store = new Mock(); + store.SetupGet(item => item.Name).Returns("test"); + var storeProvider = new Mock(); + storeProvider.Setup(provider => provider.GetRequiredClient("test")).Returns(store.Object); + storeProvider.Setup(provider => provider.GetClient("test")).Returns(store.Object); + + var services = new ServiceCollection(); + services.AddSingleton(animationRepository.Object); + services.AddSingleton(mappingRepository.Object); + services.AddSingleton(store.Object); + services.AddSingleton(storeProvider.Object); + var provider = services.BuildServiceProvider(); + var options = Options.Create(new TranscodingOptions + { + CachePath = cachePath, + MaxConcurrentJobs = 1, + QueueCapacity = queueCapacity, + CleanupInterval = TimeSpan.FromHours(1), + CacheTtl = TimeSpan.FromDays(1), + SessionTtl = TimeSpan.FromHours(1) + }); + var metrics = new TranscodingMetrics(); + var service = new HlsTranscodingService( + provider.GetRequiredService(), + runner, + metrics, + new FileExtensionContentTypeProvider(), + options, + NullLogger.Instance); + var fixture = new TranscodingFixture( + provider, + metrics, + runner, + options, + service, + cachePath, + animationInfoId); + store.Setup(item => item.FileInfoAsync( + It.IsAny(), + It.IsAny())) + .ReturnsAsync((string path, CancellationToken _) => new FileStoreInfo( + false, + path, + Path.GetFileName(path), + 1024, + fixture.LastModifiedUtc)); + store.Setup(item => item.OpenReadStreamAsync( + It.IsAny(), + It.IsAny())) + .ReturnsAsync(() => new MemoryStream([1, 2, 3])); + await service.StartAsync(CancellationToken.None); + return fixture; + } + + public async Task RestartServiceAsync() + { + await Service.StopAsync(CancellationToken.None); + Service.Dispose(); + Service = new HlsTranscodingService( + _provider.GetRequiredService(), + _runner, + _metrics, + new FileExtensionContentTypeProvider(), + _options, + NullLogger.Instance); + await Service.StartAsync(CancellationToken.None); + } + + public async ValueTask DisposeAsync() + { + await Service.StopAsync(CancellationToken.None); + Service.Dispose(); + _metrics.Dispose(); + await _provider.DisposeAsync(); + if (Directory.Exists(_cachePath)) Directory.Delete(_cachePath, recursive: true); + } + } +} diff --git a/SecondDimensionWatcherReDive.Test/TranscodingControllerTests.cs b/SecondDimensionWatcherReDive.Test/TranscodingControllerTests.cs new file mode 100644 index 0000000..be2779c --- /dev/null +++ b/SecondDimensionWatcherReDive.Test/TranscodingControllerTests.cs @@ -0,0 +1,178 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.Routing; +using Microsoft.AspNetCore.Routing; +using Moq; +using SecondDimensionWatcherReDive.Controllers; +using SecondDimensionWatcherReDive.Controllers.External; +using SecondDimensionWatcherReDive.Services.Transcoding; + +namespace SecondDimensionWatcherReDive.Test; + +[TestClass] +public class TranscodingControllerTests +{ + private StubTranscodingService _service = null!; + private TranscodingController _controller = null!; + + [TestInitialize] + public void Setup() + { + _service = new StubTranscodingService(); + _controller = new TranscodingController(_service) + { + ControllerContext = new ControllerContext { HttpContext = new DefaultHttpContext() }, + Url = CreateUrlHelper() + }; + } + + [TestMethod] + public async Task Prepare_QueuedJobReturnsAcceptedWithStatusAndCancelUrls() + { + var session = CreateStatus(TranscodingJobState.Queued, isPlayable: false); + _service.PrepareResult = session; + + var result = await _controller.Prepare( + new PrepareTranscodingRequest( + Guid.NewGuid(), + "episode.mkv", + "720p", + "ja", + null, + "en", + null), + CancellationToken.None); + + var accepted = Assert.IsInstanceOfType(result); + var response = Assert.IsInstanceOfType(accepted.Value); + Assert.AreEqual("queued", response.State); + Assert.IsNull(response.PlaybackUrl); + StringAssert.Contains(response.StatusUrl, session.SessionId.ToString()); + StringAssert.Contains(response.CancelUrl, session.AccessToken); + } + + [TestMethod] + public async Task Prepare_QueueFullReturns429AndRetryAfter() + { + _service.PrepareException = new TranscodingQueueFullException(); + + var result = await _controller.Prepare( + new PrepareTranscodingRequest(Guid.NewGuid(), "episode.mkv", null, null, null, null, null), + CancellationToken.None); + + var response = Assert.IsInstanceOfType(result); + Assert.AreEqual(StatusCodes.Status429TooManyRequests, response.StatusCode); + Assert.AreEqual("5", _controller.Response.Headers.RetryAfter.ToString()); + } + + [TestMethod] + public async Task GetPlaylist_RewritesEverySegmentWithSessionToken() + { + var sessionId = Guid.NewGuid(); + const string token = "secret-token"; + _service.Playlist = "#EXTM3U\n#EXTINF:6,\nsegment-000000.ts\n#EXT-X-ENDLIST\n"; + + var result = await _controller.GetPlaylist(sessionId, token, CancellationToken.None); + + var content = Assert.IsInstanceOfType(result); + StringAssert.Contains(content.Content, "#EXTM3U"); + StringAssert.Contains(content.Content, "GetSegment"); + StringAssert.Contains(content.Content, "segment-000000.ts"); + StringAssert.Contains(content.Content, token); + Assert.AreEqual("no-cache, no-store", _controller.Response.Headers.CacheControl.ToString()); + } + + private static TranscodingSessionStatus CreateStatus( + TranscodingJobState state, + bool isPlayable) + => new( + Guid.NewGuid(), + "access-token", + state, + state == TranscodingJobState.Queued ? null : TranscodingStrategy.Transcode, + isPlayable, + false, + null, + null, + state == TranscodingJobState.Queued ? 1 : null, + null, + null, + null, + [], + 0); + + private static IUrlHelper CreateUrlHelper() + { + var helper = new Mock(); + var httpContext = new DefaultHttpContext(); + httpContext.Request.Scheme = "https"; + httpContext.Request.Host = new HostString("example.test"); + helper.SetupGet(url => url.ActionContext) + .Returns(new ActionContext(httpContext, new RouteData(), new Microsoft.AspNetCore.Mvc.Abstractions.ActionDescriptor())); + helper.Setup(url => url.Action(It.IsAny())) + .Returns((UrlActionContext context) => + { + var values = new RouteValueDictionary(context.Values); + var suffix = string.Join("&", values.Select(pair => $"{pair.Key}={pair.Value}")); + return $"https://example.test/{context.Action}?{suffix}"; + }); + return helper.Object; + } + + private sealed class StubTranscodingService : IHlsTranscodingService + { + public TranscodingSessionStatus? PrepareResult { get; set; } + public Exception? PrepareException { get; set; } + public string? Playlist { get; set; } + + public Task PrepareAsync( + Guid animationInfoId, + string? relativePath, + TranscodingSelection selection, + CancellationToken cancellationToken) + => PrepareException is null + ? Task.FromResult(PrepareResult!) + : Task.FromException(PrepareException); + + public Task GetStatusAsync( + Guid sessionId, + string accessToken, + CancellationToken cancellationToken) + => Task.FromResult(PrepareResult); + + public Task GetPlaylistAsync( + Guid sessionId, + string accessToken, + CancellationToken cancellationToken) + => Task.FromResult(Playlist); + + public Task OpenSegmentAsync( + Guid sessionId, + string accessToken, + string fileName, + CancellationToken cancellationToken) + => Task.FromResult(null); + + public Task OpenSubtitleAsync( + Guid sessionId, + string accessToken, + string fileName, + CancellationToken cancellationToken) + => Task.FromResult(null); + + public Task OpenDirectAsync( + Guid sessionId, + string accessToken, + CancellationToken cancellationToken) + => Task.FromResult(null); + + public Task CancelAsync( + Guid sessionId, + string accessToken, + CancellationToken cancellationToken) + => Task.FromResult(false); + + public Task GetMetricsAsync(CancellationToken cancellationToken) + => Task.FromResult(new TranscodingMetricsSnapshot(0, 0, 0, 0, 0, 0, 0, null, null, 0)); + } +} diff --git a/SecondDimensionWatcherReDive.Test/TranscodingPlannerTests.cs b/SecondDimensionWatcherReDive.Test/TranscodingPlannerTests.cs new file mode 100644 index 0000000..a696c89 --- /dev/null +++ b/SecondDimensionWatcherReDive.Test/TranscodingPlannerTests.cs @@ -0,0 +1,145 @@ +using SecondDimensionWatcherReDive.Services.Transcoding; + +namespace SecondDimensionWatcherReDive.Test; + +[TestClass] +public class TranscodingPlannerTests +{ + [TestMethod] + public void CreatePlan_BrowserCompatibleMp4_UsesDirectPlay() + { + var plan = TranscodingPlanner.CreatePlan( + CreateSource("episode.mp4"), + CreateProbe(Video("h264"), Audio("aac")), + TranscodingSelection.Create("auto", null, null, null, null), + burnBitmapSubtitles: false); + + Assert.AreEqual(TranscodingStrategy.Direct, plan.Strategy); + Assert.IsTrue(plan.CopyVideo); + Assert.IsTrue(plan.CopyAudio); + } + + [TestMethod] + public void CreatePlan_CompatibleTracksInMkv_UsesLosslessRemux() + { + var plan = TranscodingPlanner.CreatePlan( + CreateSource("episode.mkv"), + CreateProbe(Video("h264"), Audio("aac")), + TranscodingSelection.Create("auto", null, null, null, null), + burnBitmapSubtitles: false); + + Assert.AreEqual(TranscodingStrategy.Remux, plan.Strategy); + Assert.IsTrue(plan.CopyVideo); + Assert.IsTrue(plan.CopyAudio); + } + + [TestMethod] + public void CreatePlan_UnsupportedCodecs_TranscodesAndSelectsPreferredAudio() + { + var japanese = Audio("flac", 1, "jpn", "Japanese"); + var english = Audio("aac", 2, "eng", "English", isDefault: true); + var plan = TranscodingPlanner.CreatePlan( + CreateSource("episode.mkv"), + CreateProbe(Video("hevc"), japanese, english), + TranscodingSelection.Create("720p", "ja", null, null, null), + burnBitmapSubtitles: false); + + Assert.AreEqual(TranscodingStrategy.Transcode, plan.Strategy); + Assert.AreEqual(japanese.Index, plan.Audio?.Index); + Assert.IsFalse(plan.CopyVideo); + Assert.IsFalse(plan.CopyAudio); + } + + [TestMethod] + public void CreatePlan_TextSubtitlesBecomeWebVttAndBitmapTrackCanBeBurned() + { + var ass = Subtitle("ass", 2, "eng", "English signs"); + var pgs = Subtitle("hdmv_pgs_subtitle", 3, "zho", "Chinese PGS"); + var plan = TranscodingPlanner.CreatePlan( + CreateSource("episode.mkv"), + CreateProbe(Video("h264"), Audio("aac"), ass, pgs), + TranscodingSelection.Create("auto", null, null, "zh", null), + burnBitmapSubtitles: true); + + Assert.AreEqual(TranscodingStrategy.Transcode, plan.Strategy); + CollectionAssert.AreEqual(new[] { ass.Index }, plan.TextSubtitles.Select(item => item.Index).ToArray()); + Assert.AreEqual(pgs.Index, plan.BitmapSubtitleToBurn?.Index); + Assert.AreEqual(0, plan.UnsupportedSubtitleCount); + } + + [TestMethod] + public void CreatePlan_SubtitlesOffNeverBurnsDefaultBitmapTrack() + { + var plan = TranscodingPlanner.CreatePlan( + CreateSource("episode.mkv"), + CreateProbe( + Video("h264"), + Audio("aac"), + Subtitle("hdmv_pgs_subtitle", 3, "zho", "Chinese PGS")), + TranscodingSelection.Create("auto", null, null, "off", null), + burnBitmapSubtitles: true); + + Assert.IsNull(plan.BitmapSubtitleToBurn); + Assert.AreEqual(TranscodingStrategy.Remux, plan.Strategy); + Assert.AreEqual(1, plan.UnsupportedSubtitleCount); + } + + [TestMethod] + public void BuildCacheKey_ChangesForSourceVersionTrackAndQuality() + { + var source = CreateSource("episode.mkv"); + var baseline = source.BuildCacheKey( + TranscodingSelection.Create("auto", "ja", null, "zh", null)); + + var changedSource = source with { LastModifiedUtc = source.LastModifiedUtc.AddSeconds(1) }; + var changedTrack = source.BuildCacheKey( + TranscodingSelection.Create("auto", "en", null, "zh", null)); + var changedQuality = source.BuildCacheKey( + TranscodingSelection.Create("720p", "ja", null, "zh", null)); + + Assert.AreNotEqual(baseline, changedSource.BuildCacheKey( + TranscodingSelection.Create("auto", "ja", null, "zh", null))); + Assert.AreNotEqual(baseline, changedTrack); + Assert.AreNotEqual(baseline, changedQuality); + } + + [TestMethod] + public void ToProgressFraction_UsesMediaDurationAndClamps() + { + Assert.AreEqual(0.5, FfmpegProcessRunner.ToProgressFraction(50, TimeSpan.FromSeconds(100))); + Assert.AreEqual(1, FfmpegProcessRunner.ToProgressFraction(110, TimeSpan.FromSeconds(100))); + Assert.IsNull(FfmpegProcessRunner.ToProgressFraction(1, null)); + } + + private static TranscodingSource CreateSource(string fileName) + => new( + Guid.NewGuid(), + Guid.NewGuid(), + $"/Anime/Group/{fileName}", + $"/media/{fileName}", + "test", + fileName, + 1024, + new DateTimeOffset(2026, 1, 2, 3, 4, 5, TimeSpan.Zero)); + + private static MediaProbe CreateProbe(params MediaStreamProbe[] streams) + => new("matroska", TimeSpan.FromMinutes(24), streams); + + private static MediaStreamProbe Video(string codec) + => new(0, "video", codec, null, null, true, false, false); + + private static MediaStreamProbe Audio( + string codec, + int index = 1, + string? language = null, + string? title = null, + bool isDefault = false) + => new(index, "audio", codec, language, title, isDefault, false, false); + + private static MediaStreamProbe Subtitle( + string codec, + int index, + string? language, + string? title) + => new(index, "subtitle", codec, language, title, false, false, false); +} diff --git a/SecondDimensionWatcherReDive/Controllers/External/AppJsonSerializerContext.cs b/SecondDimensionWatcherReDive/Controllers/External/AppJsonSerializerContext.cs index 0f20309..058878f 100644 --- a/SecondDimensionWatcherReDive/Controllers/External/AppJsonSerializerContext.cs +++ b/SecondDimensionWatcherReDive/Controllers/External/AppJsonSerializerContext.cs @@ -65,4 +65,9 @@ namespace SecondDimensionWatcherReDive.Controllers.External; [JsonSerializable(typeof(QueueMediaLibraryScanResponse))] [JsonSerializable(typeof(ApplicationSettingsResponse))] [JsonSerializable(typeof(PatchApplicationSettingsRequest))] +[JsonSerializable(typeof(PrepareTranscodingRequest))] +[JsonSerializable(typeof(TranscodingSessionResponse))] +[JsonSerializable(typeof(TranscodingSubtitleResponse))] +[JsonSerializable(typeof(List))] +[JsonSerializable(typeof(TranscodingMetricsResponse))] internal partial class AppJsonSerializerContext : JsonSerializerContext; diff --git a/SecondDimensionWatcherReDive/Controllers/External/Transcoding.cs b/SecondDimensionWatcherReDive/Controllers/External/Transcoding.cs new file mode 100644 index 0000000..f35dc34 --- /dev/null +++ b/SecondDimensionWatcherReDive/Controllers/External/Transcoding.cs @@ -0,0 +1,50 @@ +using System.ComponentModel.DataAnnotations; + +namespace SecondDimensionWatcherReDive.Controllers.External; + +internal sealed record PrepareTranscodingRequest( + [Required] Guid Id, + [Required] string Path, + string? Quality, + string? AudioLanguage, + string? AudioTrackLabel, + string? SubtitleLanguage, + string? SubtitleTrackLabel); + +internal sealed record TranscodingSubtitleResponse( + string Path, + string VirtualPath, + string? Language, + string Label, + string Format, + string Url); + +internal sealed record TranscodingSessionResponse( + Guid SessionId, + string State, + string? Strategy, + bool IsPlayable, + bool CacheHit, + double? Progress, + double? Speed, + int? QueuePosition, + string? Error, + string? VideoCodec, + string? AudioCodec, + string StatusUrl, + string CancelUrl, + string? PlaybackUrl, + IReadOnlyList Subtitles, + int UnsupportedSubtitleCount); + +internal sealed record TranscodingMetricsResponse( + int QueuedJobs, + int ActiveJobs, + long CompletedJobs, + long FailedJobs, + long CanceledJobs, + long CacheHits, + long CacheBytes, + double? AverageFirstSegmentSeconds, + double? AverageTranscodeSpeed, + double FailureRate); diff --git a/SecondDimensionWatcherReDive/Controllers/FileController.cs b/SecondDimensionWatcherReDive/Controllers/FileController.cs index 0d37050..8416c53 100644 --- a/SecondDimensionWatcherReDive/Controllers/FileController.cs +++ b/SecondDimensionWatcherReDive/Controllers/FileController.cs @@ -8,6 +8,7 @@ using Microsoft.Extensions.Caching.Distributed; using SecondDimensionWatcherReDive.Framework.DataRepository; using SecondDimensionWatcherReDive.Framework.FileStore; +using SecondDimensionWatcherReDive.Utils.FileStore; namespace SecondDimensionWatcherReDive.Controllers; @@ -41,7 +42,7 @@ public async Task GetFileLink([FromBody] External.FileLinkResultR return NotFound(); } - var virtualPath = ResolveVirtualPath(info, payload.Path); + var virtualPath = PlaybackPathResolver.ResolveVirtualPath(info, payload.Path); LogResolvedTargetPath(logger, virtualPath, "virtual path"); var token = GenerateToken(64); @@ -93,7 +94,7 @@ public async Task GetSubDir([FromQuery] [Required] Guid id, return NotFound(); } - var virtualPath = ResolveVirtualPath(info, relativeDir); + var virtualPath = PlaybackPathResolver.ResolveVirtualPath(info, relativeDir); LogListPathInfo(logger, virtualPath, true); var tokens = await fileExplorer.EnumerateDirectoryAsync( @@ -109,29 +110,6 @@ public async Task GetSubDir([FromQuery] [Required] Guid id, return Ok(results); } - private static string ResolveVirtualPath(AnimationInfo info, string? relative) - { - var root = GetAnimationVirtualRoot(info); - if (string.IsNullOrWhiteSpace(relative)) return root; - var trimmed = relative.Trim('/'); - return string.IsNullOrEmpty(trimmed) ? root : $"{root}/{trimmed}"; - } - - private static string GetAnimationVirtualRoot(AnimationInfo info) - { - if (info.Animation is null || info.Season is null) return "/unknown"; - var animationName = SanitizePathSegment(info.Animation.Name); - var subGroup = SanitizePathSegment(info.Group?.Name ?? "Unknown"); - return $"/{animationName}/{subGroup}"; - } - - private static string SanitizePathSegment(string name) - { - var invalid = Path.GetInvalidFileNameChars(); - var sanitized = string.Concat(name.Select(c => invalid.Contains(c) || c == '/' ? '_' : c)).Trim(); - return string.IsNullOrEmpty(sanitized) ? "Unknown" : sanitized; - } - [LoggerMessage(Level = LogLevel.Debug, Message = "GenerateLink request for animation {Id}, relative path: {Path}")] private static partial void LogGenerateLinkRequest(ILogger logger, Guid id, string? path); diff --git a/SecondDimensionWatcherReDive/Controllers/TranscodingController.cs b/SecondDimensionWatcherReDive/Controllers/TranscodingController.cs new file mode 100644 index 0000000..fa1c592 --- /dev/null +++ b/SecondDimensionWatcherReDive/Controllers/TranscodingController.cs @@ -0,0 +1,237 @@ +using System.ComponentModel.DataAnnotations; +using Microsoft.AspNetCore.Authentication.JwtBearer; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Net.Http.Headers; +using SecondDimensionWatcherReDive.Services.Transcoding; + +namespace SecondDimensionWatcherReDive.Controllers; + +[ApiController] +[Route("api/transcoding")] +[Authorize(AuthenticationSchemes = JwtBearerDefaults.AuthenticationScheme)] +internal sealed class TranscodingController(IHlsTranscodingService transcodingService) : ControllerBase +{ + [HttpPost("prepare")] + public async Task Prepare( + [FromBody] External.PrepareTranscodingRequest request, + CancellationToken cancellationToken) + { + try + { + var selection = TranscodingSelection.Create( + request.Quality, + request.AudioLanguage, + request.AudioTrackLabel, + request.SubtitleLanguage, + request.SubtitleTrackLabel); + var status = await transcodingService.PrepareAsync( + request.Id, + request.Path, + selection, + cancellationToken); + var response = ToResponse(status); + return status.State == TranscodingJobState.Ready ? Ok(response) : Accepted(response); + } + catch (ArgumentException exception) + { + return BadRequest(new ProblemDetails { Title = "Invalid transcoding request", Detail = exception.Message }); + } + catch (KeyNotFoundException) + { + return NotFound(); + } + catch (TranscodingQueueFullException exception) + { + Response.Headers.RetryAfter = "5"; + return StatusCode(StatusCodes.Status429TooManyRequests, + new ProblemDetails { Title = "Transcoding queue full", Detail = exception.Message }); + } + catch (TranscodingDisabledException exception) + { + return StatusCode(StatusCodes.Status503ServiceUnavailable, + new ProblemDetails { Title = "Transcoding unavailable", Detail = exception.Message }); + } + } + + [AllowAnonymous] + [HttpGet("sessions/{sessionId:guid}")] + public async Task GetStatus( + Guid sessionId, + [FromQuery][Required] string token, + CancellationToken cancellationToken) + { + var status = await transcodingService.GetStatusAsync(sessionId, token, cancellationToken); + return status is null ? NotFound() : Ok(ToResponse(status)); + } + + [AllowAnonymous] + [HttpDelete("sessions/{sessionId:guid}")] + public async Task Cancel( + Guid sessionId, + [FromQuery][Required] string token, + CancellationToken cancellationToken) + { + return await transcodingService.CancelAsync(sessionId, token, cancellationToken) + ? NoContent() + : NotFound(); + } + + [AllowAnonymous] + [HttpGet("sessions/{sessionId:guid}/source")] + public async Task GetSource( + Guid sessionId, + [FromQuery][Required] string token, + CancellationToken cancellationToken) + { + var content = await transcodingService.OpenDirectAsync(sessionId, token, cancellationToken); + if (content is null) return NotFound(); + SetContentHeaders(content, immutable: false); + return File(content.Stream, content.ContentType, content.FileName, enableRangeProcessing: true); + } + + [AllowAnonymous] + [HttpGet("sessions/{sessionId:guid}/media.m3u8")] + public async Task GetPlaylist( + Guid sessionId, + [FromQuery][Required] string token, + CancellationToken cancellationToken) + { + var playlist = await transcodingService.GetPlaylistAsync(sessionId, token, cancellationToken); + if (playlist is null) return NotFound(); + + var rewritten = new List(); + using var reader = new StringReader(playlist); + while (reader.ReadLine() is { } line) + { + if (line.Length > 0 && line[0] != '#') + { + var segmentUrl = Url.ActionLink( + nameof(GetSegment), + values: new { sessionId, fileName = line, token }); + rewritten.Add(segmentUrl ?? line); + } + else + { + rewritten.Add(line); + } + } + Response.Headers.CacheControl = "no-cache, no-store"; + return Content(string.Join('\n', rewritten) + "\n", "application/vnd.apple.mpegurl"); + } + + [AllowAnonymous] + [HttpGet("sessions/{sessionId:guid}/segments/{fileName}")] + public async Task GetSegment( + Guid sessionId, + string fileName, + [FromQuery][Required] string token, + CancellationToken cancellationToken) + { + var content = await transcodingService.OpenSegmentAsync( + sessionId, + token, + fileName, + cancellationToken); + if (content is null) return NotFound(); + SetContentHeaders(content, immutable: true); + return File(content.Stream, content.ContentType, enableRangeProcessing: true); + } + + [AllowAnonymous] + [HttpGet("sessions/{sessionId:guid}/subtitles/{fileName}")] + public async Task GetSubtitle( + Guid sessionId, + string fileName, + [FromQuery][Required] string token, + CancellationToken cancellationToken) + { + var content = await transcodingService.OpenSubtitleAsync( + sessionId, + token, + fileName, + cancellationToken); + if (content is null) return NotFound(); + SetContentHeaders(content, immutable: true); + return File(content.Stream, content.ContentType, enableRangeProcessing: true); + } + + [HttpGet("metrics")] + public async Task GetMetrics(CancellationToken cancellationToken) + { + var snapshot = await transcodingService.GetMetricsAsync(cancellationToken); + return Ok(new External.TranscodingMetricsResponse( + snapshot.QueuedJobs, + snapshot.ActiveJobs, + snapshot.CompletedJobs, + snapshot.FailedJobs, + snapshot.CanceledJobs, + snapshot.CacheHits, + snapshot.CacheBytes, + snapshot.AverageFirstSegmentSeconds, + snapshot.AverageTranscodeSpeed, + snapshot.FailureRate)); + } + + private External.TranscodingSessionResponse ToResponse(TranscodingSessionStatus status) + { + var statusUrl = Url.ActionLink( + nameof(GetStatus), + values: new { sessionId = status.SessionId, token = status.AccessToken })!; + var cancelUrl = Url.ActionLink( + nameof(Cancel), + values: new { sessionId = status.SessionId, token = status.AccessToken })!; + var playbackUrl = status.IsPlayable + ? status.Strategy == TranscodingStrategy.Direct + ? Url.ActionLink( + nameof(GetSource), + values: new { sessionId = status.SessionId, token = status.AccessToken }) + : Url.ActionLink( + nameof(GetPlaylist), + values: new { sessionId = status.SessionId, token = status.AccessToken }) + : null; + var subtitles = status.Subtitles.Select(subtitle => + new External.TranscodingSubtitleResponse( + $"__server_subtitle_{subtitle.FileName}", + $"transcoding://subtitle/{subtitle.FileName}", + subtitle.Language, + subtitle.Label, + subtitle.Format, + Url.ActionLink( + nameof(GetSubtitle), + values: new + { + sessionId = status.SessionId, + fileName = subtitle.FileName, + token = status.AccessToken + })!)).ToArray(); + return new External.TranscodingSessionResponse( + status.SessionId, + status.State.ToString().ToLowerInvariant(), + status.Strategy?.ToString().ToLowerInvariant(), + status.IsPlayable, + status.CacheHit, + status.Progress, + status.Speed, + status.QueuePosition, + status.Error, + status.VideoCodec, + status.AudioCodec, + statusUrl, + cancelUrl, + playbackUrl, + subtitles, + status.UnsupportedSubtitleCount); + } + + private void SetContentHeaders(TranscodingContent content, bool immutable) + { + Response.Headers.CacheControl = immutable + ? "private, max-age=1209600, immutable" + : "private, no-cache"; + if (content.LastModifiedUtc is { } lastModified) + Response.Headers.LastModified = lastModified.ToUniversalTime().ToString("R"); + if (content.Length is { } length) Response.ContentLength = length; + Response.Headers[HeaderNames.AcceptRanges] = "bytes"; + } +} diff --git a/SecondDimensionWatcherReDive/Program.cs b/SecondDimensionWatcherReDive/Program.cs index 80f5f19..26f593a 100644 --- a/SecondDimensionWatcherReDive/Program.cs +++ b/SecondDimensionWatcherReDive/Program.cs @@ -27,6 +27,7 @@ using SecondDimensionWatcherReDive.Chat; using SecondDimensionWatcherReDive.Plugin; using SecondDimensionWatcherReDive.Services; +using SecondDimensionWatcherReDive.Services.Transcoding; using SecondDimensionWatcherReDive.MigrationTasks; using SecondDimensionWatcherReDive.Utils.Feed; using SecondDimensionWatcherReDive.Utils.FileDownload; @@ -92,6 +93,33 @@ var localStore = builder.Configuration["FileStore:Local"] ?? "./download"; options.DownloadRoot = Path.GetFullPath(localStore); }); +builder.Services.AddOptions() + .Bind(builder.Configuration.GetSection(TranscodingOptions.SectionName)) + .PostConfigure(options => + { + if (string.IsNullOrWhiteSpace(options.CachePath)) + options.CachePath = Path.Combine( + Path.GetDirectoryName(Path.GetFullPath(passwordFile))!, + "transcode-cache"); + else + options.CachePath = Path.GetFullPath(options.CachePath); + }) + .Validate(options => options.MaxConcurrentJobs > 0, "MaxConcurrentJobs must be positive.") + .Validate(options => options.QueueCapacity > 0, "QueueCapacity must be positive.") + .Validate(options => options.MaxThreadsPerJob > 0, "MaxThreadsPerJob must be positive.") + .Validate(options => options.MaxMemoryBytesPerJob > 0, "MaxMemoryBytesPerJob must be positive.") + .Validate(options => options.MaxDiskBytesPerJob > 0, "MaxDiskBytesPerJob must be positive.") + .Validate(options => options.MaxCacheBytes > 0, "MaxCacheBytes must be positive.") + .Validate(options => options.SegmentDurationSeconds is >= 2 and <= 30, + "SegmentDurationSeconds must be between 2 and 30.") + .Validate(options => options.VideoCrf is >= 0 and <= 51, "VideoCrf must be between 0 and 51.") + .Validate(options => !string.IsNullOrWhiteSpace(options.FfmpegPath), "FfmpegPath is required.") + .Validate(options => !string.IsNullOrWhiteSpace(options.FfprobePath), "FfprobePath is required.") + .Validate(options => options.JobTimeout > TimeSpan.Zero, "JobTimeout must be positive.") + .Validate(options => options.CacheTtl > TimeSpan.Zero, "CacheTtl must be positive.") + .Validate(options => options.CleanupInterval > TimeSpan.Zero, "CleanupInterval must be positive.") + .Validate(options => options.SessionTtl > TimeSpan.Zero, "SessionTtl must be positive.") + .ValidateOnStart(); builder.Services.AddDbContext(options => { @@ -216,12 +244,20 @@ // Persistent incident inbox and health probes. builder.Services.AddSingleton(); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(sp => + sp.GetRequiredService()); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(sp => + sp.GetRequiredService()); //Add hosting services builder.Services.AddHostedService(); builder.Services.AddHostedService(); builder.Services.AddHostedService(); builder.Services.AddHostedService(); +builder.Services.AddHostedService(sp => sp.GetRequiredService()); builder.Services.AddSingleton(); builder.Services.AddSingleton(sp => sp.GetRequiredService()); diff --git a/SecondDimensionWatcherReDive/Services/Transcoding/FfmpegProcessRunner.cs b/SecondDimensionWatcherReDive/Services/Transcoding/FfmpegProcessRunner.cs new file mode 100644 index 0000000..c5189a2 --- /dev/null +++ b/SecondDimensionWatcherReDive/Services/Transcoding/FfmpegProcessRunner.cs @@ -0,0 +1,545 @@ +using System.Diagnostics; +using System.Globalization; +using System.Runtime.ExceptionServices; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Extensions.Options; + +namespace SecondDimensionWatcherReDive.Services.Transcoding; + +internal sealed record FfmpegProgress(double? ProcessedSeconds, double? Speed, bool FirstSegmentReady); + +internal sealed record FfmpegRunResult(int ExitCode, string ErrorOutput); + +internal interface IFfmpegProcessRunner +{ + Task ProbeAsync(Stream source, CancellationToken cancellationToken); + + Task GenerateHlsAsync( + Stream source, + TranscodingPlan plan, + TranscodingSelection selection, + string outputDirectory, + bool useHardwareEncoder, + Action onProgress, + CancellationToken cancellationToken); + + Task> ExtractTextSubtitlesAsync( + Stream source, + TranscodingPlan plan, + string outputDirectory, + CancellationToken cancellationToken); +} + +internal sealed partial class FfmpegProcessRunner( + IOptions options, + ILogger logger) : IFfmpegProcessRunner +{ + private readonly TranscodingOptions _options = options.Value; + private static readonly JsonSerializerOptions ProbeJsonOptions = new(JsonSerializerDefaults.Web); + + public async Task ProbeAsync(Stream source, CancellationToken cancellationToken) + { + var startInfo = CreateStartInfo(_options.FfprobePath, redirectOutput: true); + AddArguments(startInfo, + "-v", "error", + "-analyzeduration", "10000000", + "-probesize", "10000000", + "-read_intervals", "%+#32", + "-show_format", + "-show_streams", + "-of", "json", + "-i", "pipe:0"); + + using var process = Start(startInfo); + var pumpTask = PumpInputAsync(process, source, cancellationToken); + var outputTask = process.StandardOutput.ReadToEndAsync(); + var errorTask = process.StandardError.ReadToEndAsync(); + try + { + await process.WaitForExitAsync(cancellationToken); + } + catch + { + Kill(process); + await WaitForExitIgnoringErrorsAsync(process); + await IgnoreBrokenPipeAsync(pumpTask, cancellationToken); + await Task.WhenAll(outputTask, errorTask); + throw; + } + + await IgnoreBrokenPipeAsync(pumpTask, cancellationToken); + var output = await outputTask; + var error = await errorTask; + if (process.ExitCode != 0) + throw new InvalidOperationException( + $"ffprobe exited with code {process.ExitCode}: {TrimError(error)}"); + + var document = JsonSerializer.Deserialize(output, ProbeJsonOptions) + ?? throw new InvalidOperationException("ffprobe returned an empty response."); + var streams = (document.Streams ?? []) + .Where(stream => stream.Index is not null && !string.IsNullOrWhiteSpace(stream.CodecType)) + .Select(stream => new MediaStreamProbe( + stream.Index!.Value, + stream.CodecType!.Trim().ToLowerInvariant(), + string.IsNullOrWhiteSpace(stream.CodecName) + ? "unknown" + : stream.CodecName.Trim().ToLowerInvariant(), + stream.Tags?.Language, + stream.Tags?.Title, + stream.Disposition?.Default == 1, + stream.Disposition?.Forced == 1, + stream.Disposition?.AttachedPic == 1)) + .ToArray(); + var duration = ParseDuration(document.Format?.Duration) + ?? (document.Streams ?? []).Select(stream => ParseDuration(stream.Duration)).FirstOrDefault(value => value is not null); + return new MediaProbe(document.Format?.FormatName ?? "unknown", duration, streams); + } + + public async Task GenerateHlsAsync( + Stream source, + TranscodingPlan plan, + TranscodingSelection selection, + string outputDirectory, + bool useHardwareEncoder, + Action onProgress, + CancellationToken cancellationToken) + { + var startInfo = CreateStartInfo(_options.FfmpegPath, redirectOutput: false); + AddArguments(startInfo, "-hide_banner", "-y"); + if (useHardwareEncoder) + foreach (var argument in _options.HardwareInputArguments) startInfo.ArgumentList.Add(argument); + AddArguments(startInfo, "-i", "pipe:0"); + if (plan.BitmapSubtitleToBurn is null) + AddArguments(startInfo, "-map", $"0:{plan.Video.Index}"); + + if (plan.BitmapSubtitleToBurn is not null) + { + var maximumHeight = selection.Quality switch + { + "720p" => 720, + "1080p" => 1080, + _ => 0 + }; + var filter = $"[0:{plan.Video.Index}][0:{plan.BitmapSubtitleToBurn.Index}]overlay"; + if (maximumHeight > 0) filter += $",scale=-2:min({maximumHeight}\\,ih)"; + filter += "[vout]"; + AddArguments(startInfo, + "-filter_complex", + filter, + "-map", "[vout]"); + } + if (plan.Audio is not null) AddArguments(startInfo, "-map", $"0:{plan.Audio.Index}"); + + if (plan.CopyVideo) + { + AddArguments(startInfo, "-c:v", "copy"); + } + else + { + AddArguments(startInfo, + "-c:v", useHardwareEncoder ? _options.HardwareVideoEncoder! : "libx264"); + if (!useHardwareEncoder) + AddArguments(startInfo, "-preset", _options.VideoPreset, "-crf", _options.VideoCrf.ToString(CultureInfo.InvariantCulture)); + AddArguments(startInfo, "-pix_fmt", "yuv420p"); + var maximumHeight = selection.Quality switch + { + "720p" => 720, + "1080p" => 1080, + _ => 0 + }; + if (maximumHeight > 0 && plan.BitmapSubtitleToBurn is null) + AddArguments(startInfo, "-vf", $"scale=-2:min({maximumHeight}\\,ih)"); + AddArguments(startInfo, + "-force_key_frames", + $"expr:gte(t,n_forced*{_options.SegmentDurationSeconds})"); + } + + if (plan.Audio is not null) + { + if (plan.CopyAudio) AddArguments(startInfo, "-c:a", "copy"); + else AddArguments(startInfo, "-c:a", "aac", "-b:a", "192k", "-ac", "2"); + } + + var playlistPath = Path.Combine(outputDirectory, "media.m3u8"); + var segmentPattern = Path.Combine(outputDirectory, "segment-%06d.ts"); + AddArguments(startInfo, + "-sn", + "-threads", _options.MaxThreadsPerJob.ToString(CultureInfo.InvariantCulture), + "-max_muxing_queue_size", "1024", + "-f", "hls", + "-hls_time", _options.SegmentDurationSeconds.ToString(CultureInfo.InvariantCulture), + "-hls_list_size", "0", + "-hls_playlist_type", "event", + "-hls_flags", "independent_segments+temp_file", + "-hls_segment_filename", segmentPattern, + "-progress", "pipe:2", + "-nostats", + playlistPath); + + return await RunFfmpegAsync( + startInfo, + source, + outputDirectory, + onProgress, + cancellationToken); + } + + public async Task> ExtractTextSubtitlesAsync( + Stream source, + TranscodingPlan plan, + string outputDirectory, + CancellationToken cancellationToken) + { + if (plan.TextSubtitles.Count == 0) return []; + + var startInfo = CreateStartInfo(_options.FfmpegPath, redirectOutput: false); + AddArguments(startInfo, + "-hide_banner", "-y", + "-i", "pipe:0", + "-threads", _options.MaxThreadsPerJob.ToString(CultureInfo.InvariantCulture), + "-nostats"); + var pending = new List<(MediaStreamProbe Stream, string TemporaryPath, string FinalPath)>(); + foreach (var stream in plan.TextSubtitles) + { + var finalPath = Path.Combine(outputDirectory, $"subtitle-{stream.Index}.vtt"); + var temporaryPath = $"{finalPath}.tmp"; + pending.Add((stream, temporaryPath, finalPath)); + AddArguments(startInfo, + "-map", $"0:{stream.Index}", + "-c:s", "webvtt", + "-f", "webvtt", + temporaryPath); + } + var result = await RunFfmpegAsync( + startInfo, + source, + outputDirectory, + _ => { }, + cancellationToken, + detectFirstSegment: false); + if (result.ExitCode != 0) + { + LogSubtitleExtractionFailed(logger, result.ExitCode, result.ErrorOutput); + foreach (var item in pending) TryDelete(item.TemporaryPath); + return []; + } + + var subtitles = new List(); + foreach (var item in pending) + { + if (!File.Exists(item.TemporaryPath)) continue; + File.Move(item.TemporaryPath, item.FinalPath, overwrite: true); + subtitles.Add(new TranscodingSubtitle( + Path.GetFileName(item.FinalPath), + BuildSubtitleLabel(item.Stream, subtitles.Count + 1), + item.Stream.Language, + "vtt")); + } + return subtitles; + } + + private async Task RunFfmpegAsync( + ProcessStartInfo startInfo, + Stream source, + string outputDirectory, + Action onProgress, + CancellationToken cancellationToken, + bool detectFirstSegment = true) + { + using var process = Start(startInfo); + var recentErrors = new Queue(); + var errorGate = new object(); + double? lastSpeed = null; + double? lastProcessedSeconds = null; + var firstSegmentReady = 0; + string? resourceViolation = null; + var pumpTask = PumpInputAsync(process, source, cancellationToken); + var errorTask = Task.Run(async () => + { + while (await process.StandardError.ReadLineAsync(cancellationToken) is { } line) + { + if (TryParseProgress(line, out var processedSeconds, out var speed)) + { + if (processedSeconds is not null) lastProcessedSeconds = processedSeconds; + if (speed is not null) lastSpeed = speed; + onProgress(new FfmpegProgress( + lastProcessedSeconds, + lastSpeed, + Volatile.Read(ref firstSegmentReady) == 1)); + } + lock (errorGate) + { + recentErrors.Enqueue(line); + while (recentErrors.Count > 20) recentErrors.Dequeue(); + } + } + }, CancellationToken.None); + var monitorTask = Task.Run(async () => + { + while (!process.HasExited) + { + await Task.Delay(250, cancellationToken); + if (_options.MaxMemoryBytesPerJob > 0 + && TryGetWorkingSet(process, out var workingSet) + && workingSet > _options.MaxMemoryBytesPerJob) + { + resourceViolation = $"FFmpeg exceeded its {_options.MaxMemoryBytesPerJob} byte memory limit."; + Kill(process); + return; + } + + if (_options.MaxDiskBytesPerJob > 0 + && GetDirectorySize(outputDirectory) > _options.MaxDiskBytesPerJob) + { + resourceViolation = $"FFmpeg exceeded its {_options.MaxDiskBytesPerJob} byte disk limit."; + Kill(process); + return; + } + + if (detectFirstSegment + && Volatile.Read(ref firstSegmentReady) == 0 + && HasPlayableSegment(outputDirectory) + && Interlocked.Exchange(ref firstSegmentReady, 1) == 0) + { + onProgress(new FfmpegProgress(lastProcessedSeconds, lastSpeed, true)); + } + } + }, CancellationToken.None); + + Exception? waitException = null; + try + { + await process.WaitForExitAsync(cancellationToken); + } + catch (Exception exception) + { + Kill(process); + await WaitForExitIgnoringErrorsAsync(process); + waitException = exception; + } + finally + { + await IgnoreBrokenPipeAsync(pumpTask, cancellationToken); + } + + try { await errorTask; } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { } + try { await monitorTask; } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { } + if (waitException is not null) ExceptionDispatchInfo.Capture(waitException).Throw(); + if (resourceViolation is not null) throw new TranscodingResourceLimitException(resourceViolation); + if (detectFirstSegment + && Volatile.Read(ref firstSegmentReady) == 0 + && HasPlayableSegment(outputDirectory) + && Interlocked.Exchange(ref firstSegmentReady, 1) == 0) + onProgress(new FfmpegProgress(lastProcessedSeconds, lastSpeed, true)); + string errors; + lock (errorGate) errors = string.Join(" | ", recentErrors); + return new FfmpegRunResult(process.ExitCode, TrimError(errors)); + } + + private static ProcessStartInfo CreateStartInfo(string path, bool redirectOutput) + => new() + { + FileName = path, + UseShellExecute = false, + CreateNoWindow = true, + RedirectStandardInput = true, + RedirectStandardOutput = redirectOutput, + RedirectStandardError = true + }; + + private static Process Start(ProcessStartInfo startInfo) + { + try + { + return Process.Start(startInfo) + ?? throw new InvalidOperationException($"Unable to start {startInfo.FileName}."); + } + catch (Exception exception) when (exception is System.ComponentModel.Win32Exception or InvalidOperationException) + { + throw new InvalidOperationException( + $"Unable to start {startInfo.FileName}. Install FFmpeg or update Transcoding paths.", + exception); + } + } + + private static void AddArguments(ProcessStartInfo startInfo, params string[] arguments) + { + foreach (var argument in arguments) startInfo.ArgumentList.Add(argument); + } + + private static async Task PumpInputAsync( + Process process, + Stream source, + CancellationToken cancellationToken) + { + try + { + await source.CopyToAsync(process.StandardInput.BaseStream, cancellationToken); + } + finally + { + try { process.StandardInput.Close(); } + catch (IOException) { } + } + } + + private static async Task IgnoreBrokenPipeAsync(Task pumpTask, CancellationToken cancellationToken) + { + try { await pumpTask; } + catch (IOException) { } + catch (ObjectDisposedException) { } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { } + } + + private static bool TryParseProgress( + string line, + out double? processedSeconds, + out double? speed) + { + processedSeconds = null; + speed = null; + var separator = line.IndexOf('='); + if (separator <= 0) return false; + var key = line[..separator]; + var value = line[(separator + 1)..]; + if (key is "out_time_us" or "out_time_ms") + { + // Current FFmpeg reports microseconds for both legacy out_time_ms and out_time_us. + if (long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var microseconds)) + { + processedSeconds = Math.Max(0, microseconds / 1_000_000d); + return true; + } + } + else if (key == "speed") + { + var normalized = value.TrimEnd('x'); + if (double.TryParse(normalized, NumberStyles.Float, CultureInfo.InvariantCulture, out var parsed)) + speed = parsed; + return true; + } + return key == "progress"; + } + + public static double? ToProgressFraction(double? processedSeconds, TimeSpan? duration) + { + if (processedSeconds is null || duration is null || duration.Value.TotalSeconds <= 0) return null; + return Math.Clamp(processedSeconds.Value / duration.Value.TotalSeconds, 0, 1); + } + + private static TimeSpan? ParseDuration(string? value) + => double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var seconds) + && double.IsFinite(seconds) + && seconds > 0 + ? TimeSpan.FromSeconds(seconds) + : null; + + private static bool TryGetWorkingSet(Process process, out long workingSet) + { + try + { + process.Refresh(); + workingSet = process.WorkingSet64; + return true; + } + catch (InvalidOperationException) + { + workingSet = 0; + return false; + } + } + + private static long GetDirectorySize(string path) + { + try + { + return Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories) + .Sum(file => + { + try { return new FileInfo(file).Length; } + catch (IOException) { return 0; } + }); + } + catch (DirectoryNotFoundException) + { + return 0; + } + } + + private static bool HasPlayableSegment(string outputDirectory) + { + var playlist = Path.Combine(outputDirectory, "media.m3u8"); + if (!File.Exists(playlist)) return false; + try + { + return Directory.EnumerateFiles(outputDirectory, "segment-*.ts") + .Any(path => new FileInfo(path).Length > 0); + } + catch (IOException) + { + return false; + } + } + + private static void Kill(Process process) + { + try + { + if (!process.HasExited) process.Kill(entireProcessTree: true); + } + catch (InvalidOperationException) { } + } + + private static async Task WaitForExitIgnoringErrorsAsync(Process process) + { + try { await process.WaitForExitAsync(CancellationToken.None); } + catch (InvalidOperationException) { } + } + + private static string BuildSubtitleLabel(MediaStreamProbe stream, int ordinal) + { + if (!string.IsNullOrWhiteSpace(stream.Title)) return stream.Title; + if (!string.IsNullOrWhiteSpace(stream.Language)) return $"{stream.Language.ToUpperInvariant()} · Embedded"; + return $"Embedded subtitle {ordinal}"; + } + + private static string TrimError(string error) + => error.Length <= 4000 ? error : error[^4000..]; + + private static void TryDelete(string path) + { + try { File.Delete(path); } + catch (IOException) { } + } + + [LoggerMessage(Level = LogLevel.Warning, Message = "FFmpeg subtitle extraction exited with code {ExitCode}: {Error}")] + private static partial void LogSubtitleExtractionFailed(ILogger logger, int exitCode, string error); + + private sealed record FfprobeDocument( + [property: JsonPropertyName("streams")] FfprobeStream[]? Streams, + [property: JsonPropertyName("format")] FfprobeFormat? Format); + + private sealed record FfprobeStream( + [property: JsonPropertyName("index")] int? Index, + [property: JsonPropertyName("codec_name")] string? CodecName, + [property: JsonPropertyName("codec_type")] string? CodecType, + [property: JsonPropertyName("duration")] string? Duration, + [property: JsonPropertyName("disposition")] FfprobeDisposition? Disposition, + [property: JsonPropertyName("tags")] FfprobeTags? Tags); + + private sealed record FfprobeDisposition( + [property: JsonPropertyName("default")] int Default, + [property: JsonPropertyName("forced")] int Forced, + [property: JsonPropertyName("attached_pic")] int AttachedPic); + + private sealed record FfprobeTags( + [property: JsonPropertyName("language")] string? Language, + [property: JsonPropertyName("title")] string? Title); + + private sealed record FfprobeFormat( + [property: JsonPropertyName("format_name")] string? FormatName, + [property: JsonPropertyName("duration")] string? Duration); +} diff --git a/SecondDimensionWatcherReDive/Services/Transcoding/HlsTranscodingService.cs b/SecondDimensionWatcherReDive/Services/Transcoding/HlsTranscodingService.cs new file mode 100644 index 0000000..dcf6537 --- /dev/null +++ b/SecondDimensionWatcherReDive/Services/Transcoding/HlsTranscodingService.cs @@ -0,0 +1,945 @@ +using System.Collections.Concurrent; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Threading.Channels; +using Microsoft.AspNetCore.StaticFiles; +using Microsoft.Extensions.Options; +using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Framework.FileStore; +using SecondDimensionWatcherReDive.Utils.FileStore; + +namespace SecondDimensionWatcherReDive.Services.Transcoding; + +internal sealed partial class HlsTranscodingService : BackgroundService, IHlsTranscodingService +{ + private const int CacheManifestVersion = 1; + private const string CacheOwnershipMarker = ".sdw-transcode-cache"; + private readonly IServiceScopeFactory _scopeFactory; + private readonly IFfmpegProcessRunner _processRunner; + private readonly TranscodingMetrics _metrics; + private readonly IContentTypeProvider _contentTypeProvider; + private readonly ILogger _logger; + private readonly TranscodingOptions _options; + private readonly Channel _queue; + private readonly ConcurrentDictionary _jobs = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _sessions = new(); + private readonly SemaphoreSlim _creationGate = new(1, 1); + private long _queueOrdinal; + + public HlsTranscodingService( + IServiceScopeFactory scopeFactory, + IFfmpegProcessRunner processRunner, + TranscodingMetrics metrics, + IContentTypeProvider contentTypeProvider, + IOptions options, + ILogger logger) + { + _scopeFactory = scopeFactory; + _processRunner = processRunner; + _metrics = metrics; + _contentTypeProvider = contentTypeProvider; + _logger = logger; + _options = options.Value; + _queue = Channel.CreateBounded(new BoundedChannelOptions(_options.QueueCapacity) + { + FullMode = BoundedChannelFullMode.Wait, + SingleReader = _options.MaxConcurrentJobs == 1, + SingleWriter = false, + AllowSynchronousContinuations = false + }); + } + + public async Task PrepareAsync( + Guid animationInfoId, + string? relativePath, + TranscodingSelection selection, + CancellationToken cancellationToken) + { + if (!_options.Enabled) throw new TranscodingDisabledException(); + var source = await ResolveSourceAsync(animationInfoId, relativePath, cancellationToken); + var cacheKey = BuildCacheKey(source, selection); + + await _creationGate.WaitAsync(cancellationToken); + try + { + if (_jobs.TryGetValue(cacheKey, out var terminalJob) + && terminalJob.GetState() is TranscodingJobState.Failed or TranscodingJobState.Canceled) + _jobs.TryRemove(new KeyValuePair(cacheKey, terminalJob)); + + var isNewJob = false; + var cacheHit = false; + if (!_jobs.TryGetValue(cacheKey, out var job)) + { + var cacheDirectory = Path.Combine(_options.CachePath, cacheKey); + var manifest = await TryLoadManifestAsync(cacheDirectory, cancellationToken); + if (manifest is not null) + { + job = TranscodingJob.FromManifest( + cacheKey, + cacheDirectory, + source, + selection, + manifest); + cacheHit = true; + _metrics.RecordCacheHit(); + } + else + { + job = new TranscodingJob( + cacheKey, + cacheDirectory, + source, + selection, + Interlocked.Increment(ref _queueOrdinal)); + isNewJob = true; + } + _jobs[cacheKey] = job; + } + else + { + cacheHit = job.GetState() == TranscodingJobState.Ready; + if (cacheHit) _metrics.RecordCacheHit(); + } + + var session = new TranscodingSession(job, cacheHit, _options.SessionTtl); + _sessions[session.Id] = session; + job.AddSession(session.Id); + + if (isNewJob && !_queue.Writer.TryWrite(job)) + { + _sessions.TryRemove(session.Id, out _); + job.RemoveSession(session.Id); + _jobs.TryRemove(new KeyValuePair(cacheKey, job)); + job.Cancellation.Dispose(); + throw new TranscodingQueueFullException(); + } + + UpdateJobGauges(); + TouchCache(job, session); + return BuildStatus(session); + } + finally + { + _creationGate.Release(); + } + } + + public Task GetStatusAsync( + Guid sessionId, + string accessToken, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var session = FindSession(sessionId, accessToken); + if (session is null) return Task.FromResult(null); + TouchCache(session.Job, session); + return Task.FromResult(BuildStatus(session)); + } + + public async Task GetPlaylistAsync( + Guid sessionId, + string accessToken, + CancellationToken cancellationToken) + { + var session = FindSession(sessionId, accessToken); + if (session is null || !session.Job.GetIsPlayable()) return null; + TouchCache(session.Job, session); + var playlistPath = Path.Combine(session.Job.CacheDirectory, "media.m3u8"); + for (var attempt = 0; attempt < 3; attempt++) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + return await File.ReadAllTextAsync(playlistPath, cancellationToken); + } + catch (IOException) when (attempt < 2) + { + await Task.Delay(25, cancellationToken); + } + catch (FileNotFoundException) + { + return null; + } + } + return null; + } + + public Task OpenSegmentAsync( + Guid sessionId, + string accessToken, + string fileName, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var session = FindSession(sessionId, accessToken); + if (session is null || !session.Job.GetIsPlayable() || !IsSegmentName(fileName)) + return Task.FromResult(null); + + var path = Path.Combine(session.Job.CacheDirectory, fileName); + if (!File.Exists(path)) return Task.FromResult(null); + TouchCache(session.Job, session); + return Task.FromResult(OpenCachedContent(path, "video/mp2t")); + } + + public Task OpenSubtitleAsync( + Guid sessionId, + string accessToken, + string fileName, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var session = FindSession(sessionId, accessToken); + if (session is null || !session.Job.HasSubtitle(fileName)) + return Task.FromResult(null); + + var path = Path.Combine(session.Job.CacheDirectory, fileName); + if (!File.Exists(path)) return Task.FromResult(null); + TouchCache(session.Job, session); + return Task.FromResult(OpenCachedContent(path, "text/vtt; charset=utf-8")); + } + + public async Task OpenDirectAsync( + Guid sessionId, + string accessToken, + CancellationToken cancellationToken) + { + var session = FindSession(sessionId, accessToken); + if (session is null || session.Job.GetStrategy() != TranscodingStrategy.Direct) return null; + session.Touch(_options.SessionTtl); + var stream = await OpenSourceStreamAsync(session.Job.Source, cancellationToken); + var contentType = _contentTypeProvider.TryGetContentType(session.Job.Source.FileName, out var type) + ? type + : "application/octet-stream"; + return new TranscodingContent( + stream, + contentType, + session.Job.Source.FileName, + session.Job.Source.Length, + session.Job.Source.LastModifiedUtc); + } + + public Task CancelAsync( + Guid sessionId, + string accessToken, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var session = FindSession(sessionId, accessToken, touch: false); + if (session is null || !_sessions.TryRemove(sessionId, out _)) return Task.FromResult(false); + ReleaseSession(session); + return Task.FromResult(true); + } + + public Task GetMetricsAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.FromResult(_metrics.Snapshot()); + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + Directory.CreateDirectory(_options.CachePath); + await CleanupCacheAsync(removeIncomplete: true, stoppingToken); + var workers = Enumerable.Range(0, _options.MaxConcurrentJobs) + .Select(_ => RunWorkerAsync(stoppingToken)) + .ToArray(); + var cleanup = RunCleanupLoopAsync(stoppingToken); + await Task.WhenAll(workers.Append(cleanup)); + } + + private async Task RunWorkerAsync(CancellationToken stoppingToken) + { + await foreach (var job in _queue.Reader.ReadAllAsync(stoppingToken)) + { + try + { + await ProcessJobAsync(job, stoppingToken); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + return; + } + } + } + + private async Task ProcessJobAsync(TranscodingJob job, CancellationToken stoppingToken) + { + if (job.Cancellation.IsCancellationRequested) + { + MarkCanceled(job); + return; + } + + using var timeout = new CancellationTokenSource(_options.JobTimeout); + using var linked = CancellationTokenSource.CreateLinkedTokenSource( + stoppingToken, + job.Cancellation.Token, + timeout.Token); + var cancellationToken = linked.Token; + var startedAt = DateTimeOffset.UtcNow; + try + { + job.SetState(TranscodingJobState.Probing); + UpdateJobGauges(); + MediaProbe probe; + await using (var source = await OpenSourceStreamAsync(job.Source, cancellationToken)) + probe = await _processRunner.ProbeAsync(source, cancellationToken); + var plan = TranscodingPlanner.CreatePlan( + job.Source, + probe, + job.Selection, + _options.BurnBitmapSubtitles); + job.SetPlan(plan); + + if (plan.Strategy == TranscodingStrategy.Direct) + { + job.MarkPlayable(); + _metrics.RecordFirstSegment(DateTimeOffset.UtcNow - startedAt); + _metrics.RecordCompleted(); + job.SetReady([]); + UpdateJobGauges(); + return; + } + + RecreateJobDirectory(job.CacheDirectory); + job.SetState(TranscodingJobState.Transcoding); + UpdateJobGauges(); + var firstSegmentRecorded = 0; + void OnProgress(FfmpegProgress update) + { + var fraction = FfmpegProcessRunner.ToProgressFraction(update.ProcessedSeconds, probe.Duration); + job.SetProgress(fraction, update.Speed); + if (update.FirstSegmentReady) + { + job.MarkPlayable(); + if (Interlocked.Exchange(ref firstSegmentRecorded, 1) == 0) + _metrics.RecordFirstSegment(DateTimeOffset.UtcNow - startedAt); + } + } + + var useHardware = !plan.CopyVideo && !string.IsNullOrWhiteSpace(_options.HardwareVideoEncoder); + FfmpegRunResult result; + await using (var source = await OpenSourceStreamAsync(job.Source, cancellationToken)) + result = await _processRunner.GenerateHlsAsync( + source, + plan, + job.Selection, + job.CacheDirectory, + useHardware, + OnProgress, + cancellationToken); + if (result.ExitCode != 0 && useHardware) + { + LogHardwareFallback(_logger, _options.HardwareVideoEncoder!, result.ErrorOutput); + DeleteGeneratedFiles(job.CacheDirectory); + await using var source = await OpenSourceStreamAsync(job.Source, cancellationToken); + result = await _processRunner.GenerateHlsAsync( + source, + plan, + job.Selection, + job.CacheDirectory, + useHardwareEncoder: false, + OnProgress, + cancellationToken); + } + if (result.ExitCode != 0) + throw new InvalidOperationException($"FFmpeg exited with code {result.ExitCode}: {result.ErrorOutput}"); + + IReadOnlyList subtitles; + await using (var source = await OpenSourceStreamAsync(job.Source, cancellationToken)) + subtitles = await _processRunner.ExtractTextSubtitlesAsync( + source, + plan, + job.CacheDirectory, + cancellationToken); + job.SetSubtitles(subtitles); + await WriteManifestAsync(job, cancellationToken); + _metrics.RecordCompleted(); + if (job.GetSpeed() is { } speed) _metrics.RecordSpeed(speed); + UpdateCacheBytes(); + job.SetReady(subtitles); + UpdateJobGauges(); + await CleanupCacheAsync(removeIncomplete: false, cancellationToken); + } + catch (OperationCanceledException) when (job.Cancellation.IsCancellationRequested) + { + MarkCanceled(job); + } + catch (OperationCanceledException) when (timeout.IsCancellationRequested) + { + MarkFailed(job, "The transcoding job exceeded its configured timeout."); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + MarkCanceled(job); + throw; + } + catch (Exception exception) + { + LogJobFailed(_logger, job.Source.VirtualPath, exception); + MarkFailed(job, exception.Message); + } + } + + private async Task RunCleanupLoopAsync(CancellationToken stoppingToken) + { + using var timer = new PeriodicTimer(_options.CleanupInterval); + while (await timer.WaitForNextTickAsync(stoppingToken)) + await CleanupCacheAsync(removeIncomplete: true, stoppingToken); + } + + private async Task ResolveSourceAsync( + Guid animationInfoId, + string? relativePath, + CancellationToken cancellationToken) + { + await using var scope = _scopeFactory.CreateAsyncScope(); + var animationRepository = scope.ServiceProvider.GetRequiredService(); + var info = await animationRepository.FindByIdWithAnimationAsync(animationInfoId, cancellationToken); + if (info is null || !info.IsDownloadFinished) + throw new KeyNotFoundException("The requested animation is not available for playback."); + + var virtualPath = PlaybackPathResolver.ResolveVirtualPath(info, relativePath); + var mapping = await scope.ServiceProvider.GetRequiredService() + .FindByVirtualPathAsync(virtualPath, cancellationToken); + if (mapping is null || mapping.AnimationInfoId != animationInfoId) + throw new KeyNotFoundException("The requested playback file mapping was not found."); + + var store = scope.ServiceProvider.GetRequiredService() + .GetRequiredClient(mapping.FileStore); + var fileInfo = await store.FileInfoAsync(mapping.PhysicalPath, cancellationToken); + if (fileInfo.IsDirectory) + throw new KeyNotFoundException("The requested playback path is a directory."); + return new TranscodingSource( + animationInfoId, + mapping.Id, + mapping.VirtualPath, + mapping.PhysicalPath, + mapping.FileStore, + fileInfo.FileName, + fileInfo.Length ?? 0, + fileInfo.LastModifiedUtc ?? DateTimeOffset.UnixEpoch); + } + + private async Task OpenSourceStreamAsync( + TranscodingSource source, + CancellationToken cancellationToken) + { + var scope = _scopeFactory.CreateAsyncScope(); + try + { + var store = scope.ServiceProvider.GetRequiredService() + .GetRequiredClient(source.FileStore); + var stream = await store.OpenReadStreamAsync(source.PhysicalPath, cancellationToken); + return new ScopeOwnedStream(stream, scope); + } + catch + { + await scope.DisposeAsync(); + throw; + } + } + + private TranscodingSession? FindSession(Guid id, string token, bool touch = true) + { + if (!_sessions.TryGetValue(id, out var session) || !TokensEqual(session.AccessToken, token)) + return null; + if (session.IsExpired) + { + if (_sessions.TryRemove(id, out _)) ReleaseSession(session); + return null; + } + if (touch) session.Touch(_options.SessionTtl); + return session; + } + + private TranscodingSessionStatus BuildStatus(TranscodingSession session) + { + var job = session.Job; + var state = job.GetState(); + int? queuePosition = state == TranscodingJobState.Queued + ? _jobs.Values.Count(candidate => + candidate.GetState() == TranscodingJobState.Queued + && candidate.QueueOrdinal <= job.QueueOrdinal) + : null; + return job.CreateStatus(session.Id, session.AccessToken, session.CacheHit, queuePosition); + } + + private async Task TryLoadManifestAsync( + string directory, + CancellationToken cancellationToken) + { + var path = Path.Combine(directory, "complete.json"); + if (!File.Exists(Path.Combine(directory, CacheOwnershipMarker)) + || !File.Exists(path) + || !File.Exists(Path.Combine(directory, "media.m3u8"))) return null; + try + { + await using var stream = File.OpenRead(path); + var manifest = await JsonSerializer.DeserializeAsync(stream, cancellationToken: cancellationToken); + return manifest?.Version == CacheManifestVersion ? manifest : null; + } + catch (Exception exception) when (exception is IOException or JsonException) + { + LogInvalidCacheManifest(_logger, path, exception); + TryDeleteDirectory(directory); + return null; + } + } + + private async Task WriteManifestAsync(TranscodingJob job, CancellationToken cancellationToken) + { + var manifest = job.CreateManifest(); + var path = Path.Combine(job.CacheDirectory, "complete.json"); + var temporaryPath = $"{path}.tmp"; + await using (var stream = new FileStream( + temporaryPath, + FileMode.Create, + FileAccess.Write, + FileShare.None, + 4096, + FileOptions.Asynchronous | FileOptions.WriteThrough)) + await JsonSerializer.SerializeAsync(stream, manifest, cancellationToken: cancellationToken); + File.Move(temporaryPath, path, overwrite: true); + TouchCache(job, null); + } + + private async Task CleanupCacheAsync(bool removeIncomplete, CancellationToken cancellationToken) + { + CleanupExpiredSessions(); + if (!Directory.Exists(_options.CachePath)) return; + var now = DateTimeOffset.UtcNow; + var candidates = new List(); + foreach (var directory in Directory.EnumerateDirectories(_options.CachePath)) + { + cancellationToken.ThrowIfCancellationRequested(); + var key = Path.GetFileName(directory); + if (!IsCacheKey(key)) continue; + if (!File.Exists(Path.Combine(directory, CacheOwnershipMarker))) continue; + var completePath = Path.Combine(directory, "complete.json"); + if (!File.Exists(completePath)) + { + if (removeIncomplete && !IsActive(key)) TryDeleteDirectory(directory); + continue; + } + var accessPath = Path.Combine(directory, ".access"); + var lastAccess = File.Exists(accessPath) + ? File.GetLastWriteTimeUtc(accessPath) + : File.GetLastWriteTimeUtc(completePath); + candidates.Add(new CacheDirectory(key, directory, lastAccess, GetDirectorySize(directory))); + } + + foreach (var expired in candidates + .Where(candidate => now - candidate.LastAccess > _options.CacheTtl) + .OrderBy(candidate => candidate.LastAccess) + .ToArray()) + { + if (IsInUse(expired.Key)) continue; + RemoveCacheDirectory(expired); + candidates.Remove(expired); + } + + var total = candidates.Sum(candidate => candidate.Size); + foreach (var candidate in candidates.OrderBy(candidate => candidate.LastAccess)) + { + if (total <= _options.MaxCacheBytes) break; + if (IsInUse(candidate.Key)) continue; + RemoveCacheDirectory(candidate); + total -= candidate.Size; + } + _metrics.SetCacheBytes(Math.Max(0, total)); + await Task.CompletedTask; + } + + private void CleanupExpiredSessions() + { + foreach (var pair in _sessions) + if (pair.Value.IsExpired && _sessions.TryRemove(pair.Key, out var session)) ReleaseSession(session); + } + + private void ReleaseSession(TranscodingSession session) + { + var remaining = session.Job.RemoveSession(session.Id); + if (remaining == 0 + && session.Job.GetState() is TranscodingJobState.Queued + or TranscodingJobState.Probing + or TranscodingJobState.Transcoding) + session.Job.Cancellation.Cancel(); + else if (remaining == 0 + && session.Job.GetState() == TranscodingJobState.Ready + && session.Job.GetStrategy() == TranscodingStrategy.Direct + && _jobs.TryRemove(new KeyValuePair( + session.Job.CacheKey, + session.Job))) + session.Job.Cancellation.Dispose(); + } + + private bool IsActive(string key) + => _jobs.TryGetValue(key, out var job) + && job.GetState() is TranscodingJobState.Queued + or TranscodingJobState.Probing + or TranscodingJobState.Transcoding; + + private bool IsInUse(string key) + => _jobs.TryGetValue(key, out var job) && (job.SessionCount > 0 || IsActive(key)); + + private void RemoveCacheDirectory(CacheDirectory candidate) + { + TryDeleteDirectory(candidate.Path); + if (_jobs.TryGetValue(candidate.Key, out var job) && job.GetState() == TranscodingJobState.Ready) + _jobs.TryRemove(new KeyValuePair(candidate.Key, job)); + } + + private void MarkCanceled(TranscodingJob job) + { + job.SetCanceled(); + _jobs.TryRemove(new KeyValuePair(job.CacheKey, job)); + TryDeleteDirectory(job.CacheDirectory); + _metrics.RecordCanceled(); + UpdateCacheBytes(); + UpdateJobGauges(); + } + + private void MarkFailed(TranscodingJob job, string error) + { + job.SetFailed(error); + _jobs.TryRemove(new KeyValuePair(job.CacheKey, job)); + TryDeleteDirectory(job.CacheDirectory); + _metrics.RecordFailed(); + UpdateCacheBytes(); + UpdateJobGauges(); + } + + private void UpdateJobGauges() + { + _metrics.SetQueued(_jobs.Values.Count(job => job.GetState() == TranscodingJobState.Queued)); + _metrics.SetActive(_jobs.Values.Count(job => + job.GetState() is TranscodingJobState.Probing or TranscodingJobState.Transcoding)); + } + + private void UpdateCacheBytes() => _metrics.SetCacheBytes(GetDirectorySize(_options.CachePath)); + + private void TouchCache(TranscodingJob job, TranscodingSession? session) + { + session?.Touch(_options.SessionTtl); + if (!Directory.Exists(job.CacheDirectory)) return; + if (session is not null && !session.ShouldTouchCache) return; + try + { + var marker = Path.Combine(job.CacheDirectory, ".access"); + if (!File.Exists(marker)) File.WriteAllText(marker, string.Empty); + File.SetLastWriteTimeUtc(marker, DateTime.UtcNow); + session?.MarkCacheTouched(); + } + catch (IOException) { } + } + + private string BuildCacheKey(TranscodingSource source, TranscodingSelection selection) + { + var material = string.Join('|', + source.BuildCacheKey(selection), + CacheManifestVersion, + _options.SegmentDurationSeconds, + _options.VideoCrf, + _options.VideoPreset, + _options.HardwareVideoEncoder ?? string.Empty, + string.Join('\u001f', _options.HardwareInputArguments), + _options.BurnBitmapSubtitles); + return Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(material))); + } + + private static TranscodingContent OpenCachedContent(string path, string contentType) + { + var info = new FileInfo(path); + var stream = new FileStream( + path, + FileMode.Open, + FileAccess.Read, + FileShare.ReadWrite | FileShare.Delete, + 64 * 1024, + FileOptions.Asynchronous | FileOptions.SequentialScan); + return new TranscodingContent( + stream, + contentType, + info.Name, + info.Length, + new DateTimeOffset(info.LastWriteTimeUtc, TimeSpan.Zero)); + } + + private static void RecreateJobDirectory(string path) + { + if (Directory.Exists(path) && !File.Exists(Path.Combine(path, CacheOwnershipMarker))) + throw new InvalidOperationException( + $"The transcoding cache path '{path}' is occupied by an unmanaged directory."); + TryDeleteDirectory(path); + Directory.CreateDirectory(path); + File.WriteAllText(Path.Combine(path, CacheOwnershipMarker), string.Empty); + } + + private static void DeleteGeneratedFiles(string directory) + { + foreach (var path in Directory.EnumerateFiles(directory)) + if (Path.GetFileName(path) is not (".access" or CacheOwnershipMarker)) + try { File.Delete(path); } + catch (IOException) { } + } + + private static void TryDeleteDirectory(string path) + { + try + { + if (Directory.Exists(path) + && File.Exists(Path.Combine(path, CacheOwnershipMarker))) + Directory.Delete(path, recursive: true); + } + catch (IOException) { } + catch (UnauthorizedAccessException) { } + } + + private static long GetDirectorySize(string path) + { + if (!Directory.Exists(path)) return 0; + try + { + return Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories) + .Sum(file => + { + try { return new FileInfo(file).Length; } + catch (IOException) { return 0; } + }); + } + catch (IOException) + { + return 0; + } + } + + private static bool TokensEqual(string expected, string actual) + { + if (expected.Length != actual.Length) return false; + return CryptographicOperations.FixedTimeEquals( + Encoding.UTF8.GetBytes(expected), + Encoding.UTF8.GetBytes(actual)); + } + + private static bool IsSegmentName(string name) + => name == Path.GetFileName(name) + && name.StartsWith("segment-", StringComparison.Ordinal) + && name.EndsWith(".ts", StringComparison.Ordinal) + && name[8..^3].All(char.IsAsciiDigit); + + private static bool IsCacheKey(string name) + => name.Length == 64 && name.All(char.IsAsciiHexDigit); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Hardware encoder {Encoder} failed; retrying with the CPU encoder. FFmpeg: {Error}")] + private static partial void LogHardwareFallback(ILogger logger, string encoder, string error); + + [LoggerMessage(Level = LogLevel.Error, Message = "Transcoding failed for {VirtualPath}")] + private static partial void LogJobFailed(ILogger logger, string virtualPath, Exception exception); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Ignoring invalid transcoding cache manifest {Path}")] + private static partial void LogInvalidCacheManifest(ILogger logger, string path, Exception exception); + + private sealed record CacheDirectory(string Key, string Path, DateTimeOffset LastAccess, long Size); + + private sealed record CacheManifest( + int Version, + TranscodingStrategy Strategy, + string VideoCodec, + string? AudioCodec, + TranscodingSubtitle[] Subtitles, + int UnsupportedSubtitleCount); + + private sealed class TranscodingSession + { + private long _expiresAtTicks; + private long _lastCacheTouchTicks; + + public TranscodingSession(TranscodingJob job, bool cacheHit, TimeSpan ttl) + { + Job = job; + CacheHit = cacheHit; + Id = Guid.NewGuid(); + AccessToken = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(32)); + Touch(ttl); + } + + public Guid Id { get; } + public string AccessToken { get; } + public TranscodingJob Job { get; } + public bool CacheHit { get; } + public bool IsExpired => DateTimeOffset.UtcNow.UtcTicks > Interlocked.Read(ref _expiresAtTicks); + public bool ShouldTouchCache + => DateTimeOffset.UtcNow.UtcTicks - Interlocked.Read(ref _lastCacheTouchTicks) > TimeSpan.FromMinutes(1).Ticks; + + public void Touch(TimeSpan ttl) + => Interlocked.Exchange(ref _expiresAtTicks, (DateTimeOffset.UtcNow + ttl).UtcTicks); + + public void MarkCacheTouched() + => Interlocked.Exchange(ref _lastCacheTouchTicks, DateTimeOffset.UtcNow.UtcTicks); + } + + private sealed class TranscodingJob + { + private readonly object _gate = new(); + private readonly HashSet _sessions = []; + private TranscodingJobState _state = TranscodingJobState.Queued; + private TranscodingPlan? _plan; + private bool _isPlayable; + private double? _progress; + private double? _speed; + private string? _error; + private IReadOnlyList _subtitles = []; + + public TranscodingJob( + string cacheKey, + string cacheDirectory, + TranscodingSource source, + TranscodingSelection selection, + long queueOrdinal) + { + CacheKey = cacheKey; + CacheDirectory = cacheDirectory; + Source = source; + Selection = selection; + QueueOrdinal = queueOrdinal; + } + + public string CacheKey { get; } + public string CacheDirectory { get; } + public TranscodingSource Source { get; } + public TranscodingSelection Selection { get; } + public long QueueOrdinal { get; } + public CancellationTokenSource Cancellation { get; } = new(); + public int SessionCount { get { lock (_gate) return _sessions.Count; } } + + public static TranscodingJob FromManifest( + string cacheKey, + string cacheDirectory, + TranscodingSource source, + TranscodingSelection selection, + CacheManifest manifest) + { + var job = new TranscodingJob(cacheKey, cacheDirectory, source, selection, 0) + { + _state = TranscodingJobState.Ready, + _isPlayable = true, + _progress = 1, + _subtitles = manifest.Subtitles + }; + var video = new MediaStreamProbe(0, "video", manifest.VideoCodec, null, null, true, false, false); + var audio = manifest.AudioCodec is null + ? null + : new MediaStreamProbe(1, "audio", manifest.AudioCodec, null, null, true, false, false); + job._plan = new TranscodingPlan( + manifest.Strategy, + video, + audio, + null, + [], + manifest.UnsupportedSubtitleCount, + manifest.Strategy == TranscodingStrategy.Remux, + manifest.Strategy == TranscodingStrategy.Remux); + return job; + } + + public void AddSession(Guid id) { lock (_gate) _sessions.Add(id); } + public int RemoveSession(Guid id) { lock (_gate) { _sessions.Remove(id); return _sessions.Count; } } + public TranscodingJobState GetState() { lock (_gate) return _state; } + public TranscodingStrategy? GetStrategy() { lock (_gate) return _plan?.Strategy; } + public bool GetIsPlayable() { lock (_gate) return _isPlayable; } + public double? GetSpeed() { lock (_gate) return _speed; } + public bool HasSubtitle(string fileName) + { + lock (_gate) return fileName == Path.GetFileName(fileName) && _subtitles.Any(item => item.FileName == fileName); + } + + public void SetState(TranscodingJobState state) { lock (_gate) _state = state; } + public void SetPlan(TranscodingPlan plan) { lock (_gate) _plan = plan; } + public void MarkPlayable() { lock (_gate) _isPlayable = true; } + public void SetProgress(double? progress, double? speed) + { + lock (_gate) + { + _progress = progress; + if (speed is not null) _speed = speed; + } + } + + public void SetReady(IReadOnlyList subtitles) + { + lock (_gate) + { + _subtitles = subtitles; + _progress = 1; + _isPlayable = true; + _state = TranscodingJobState.Ready; + } + } + + public void SetSubtitles(IReadOnlyList subtitles) + { + lock (_gate) _subtitles = subtitles; + } + + public void SetCanceled() + { + lock (_gate) + { + _state = TranscodingJobState.Canceled; + _isPlayable = false; + _error = "The transcoding job was canceled."; + } + } + + public void SetFailed(string error) + { + lock (_gate) + { + _state = TranscodingJobState.Failed; + _isPlayable = false; + _error = error; + } + } + + public TranscodingSessionStatus CreateStatus( + Guid sessionId, + string token, + bool cacheHit, + int? queuePosition) + { + lock (_gate) + return new TranscodingSessionStatus( + sessionId, + token, + _state, + _plan?.Strategy, + _isPlayable, + cacheHit, + _progress, + _speed, + queuePosition, + _error, + _plan?.Video.CodecName, + _plan?.Audio?.CodecName, + _subtitles, + _plan?.UnsupportedSubtitleCount ?? 0); + } + + public CacheManifest CreateManifest() + { + lock (_gate) + { + var plan = _plan ?? throw new InvalidOperationException("A completed job has no transcoding plan."); + return new CacheManifest( + CacheManifestVersion, + plan.Strategy, + plan.Video.CodecName, + plan.Audio?.CodecName, + _subtitles.ToArray(), + plan.UnsupportedSubtitleCount); + } + } + } +} diff --git a/SecondDimensionWatcherReDive/Services/Transcoding/IHlsTranscodingService.cs b/SecondDimensionWatcherReDive/Services/Transcoding/IHlsTranscodingService.cs new file mode 100644 index 0000000..b2fc65a --- /dev/null +++ b/SecondDimensionWatcherReDive/Services/Transcoding/IHlsTranscodingService.cs @@ -0,0 +1,44 @@ +namespace SecondDimensionWatcherReDive.Services.Transcoding; + +internal interface IHlsTranscodingService +{ + Task PrepareAsync( + Guid animationInfoId, + string? relativePath, + TranscodingSelection selection, + CancellationToken cancellationToken); + + Task GetStatusAsync( + Guid sessionId, + string accessToken, + CancellationToken cancellationToken); + + Task GetPlaylistAsync( + Guid sessionId, + string accessToken, + CancellationToken cancellationToken); + + Task OpenSegmentAsync( + Guid sessionId, + string accessToken, + string fileName, + CancellationToken cancellationToken); + + Task OpenSubtitleAsync( + Guid sessionId, + string accessToken, + string fileName, + CancellationToken cancellationToken); + + Task OpenDirectAsync( + Guid sessionId, + string accessToken, + CancellationToken cancellationToken); + + Task CancelAsync( + Guid sessionId, + string accessToken, + CancellationToken cancellationToken); + + Task GetMetricsAsync(CancellationToken cancellationToken); +} diff --git a/SecondDimensionWatcherReDive/Services/Transcoding/ScopeOwnedStream.cs b/SecondDimensionWatcherReDive/Services/Transcoding/ScopeOwnedStream.cs new file mode 100644 index 0000000..362ea00 --- /dev/null +++ b/SecondDimensionWatcherReDive/Services/Transcoding/ScopeOwnedStream.cs @@ -0,0 +1,97 @@ +using Microsoft.Extensions.DependencyInjection; + +namespace SecondDimensionWatcherReDive.Services.Transcoding; + +///

+/// Keeps the scoped file-store implementation alive for as long as its stream. +/// +internal sealed class ScopeOwnedStream(Stream inner, AsyncServiceScope scope) : Stream +{ + private int _disposed; + + public override bool CanRead => inner.CanRead; + public override bool CanSeek => inner.CanSeek; + public override bool CanWrite => inner.CanWrite; + public override long Length => inner.Length; + + public override long Position + { + get => inner.Position; + set => inner.Position = value; + } + + public override void Flush() => inner.Flush(); + + public override Task FlushAsync(CancellationToken cancellationToken) + => inner.FlushAsync(cancellationToken); + + public override int Read(byte[] buffer, int offset, int count) + => inner.Read(buffer, offset, count); + + public override int Read(Span buffer) => inner.Read(buffer); + + public override ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default) + => inner.ReadAsync(buffer, cancellationToken); + + public override Task ReadAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken) + => inner.ReadAsync(buffer, offset, count, cancellationToken); + + public override long Seek(long offset, SeekOrigin origin) => inner.Seek(offset, origin); + + public override void SetLength(long value) => inner.SetLength(value); + + public override void Write(byte[] buffer, int offset, int count) + => inner.Write(buffer, offset, count); + + public override void Write(ReadOnlySpan buffer) => inner.Write(buffer); + + public override ValueTask WriteAsync( + ReadOnlyMemory buffer, + CancellationToken cancellationToken = default) + => inner.WriteAsync(buffer, cancellationToken); + + public override Task WriteAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken) + => inner.WriteAsync(buffer, offset, count, cancellationToken); + + protected override void Dispose(bool disposing) + { + if (disposing && Interlocked.Exchange(ref _disposed, 1) == 0) + { + try + { + inner.Dispose(); + } + finally + { + scope.DisposeAsync().AsTask().GetAwaiter().GetResult(); + } + } + base.Dispose(disposing); + } + + public override async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) + { + try + { + await inner.DisposeAsync(); + } + finally + { + await scope.DisposeAsync(); + } + } + GC.SuppressFinalize(this); + } +} diff --git a/SecondDimensionWatcherReDive/Services/Transcoding/TranscodingMetrics.cs b/SecondDimensionWatcherReDive/Services/Transcoding/TranscodingMetrics.cs new file mode 100644 index 0000000..a7fce59 --- /dev/null +++ b/SecondDimensionWatcherReDive/Services/Transcoding/TranscodingMetrics.cs @@ -0,0 +1,107 @@ +using System.Diagnostics.Metrics; + +namespace SecondDimensionWatcherReDive.Services.Transcoding; + +internal sealed class TranscodingMetrics : IDisposable +{ + private readonly Meter _meter = new("SecondDimensionWatcherReDive.Transcoding", "1.0.0"); + private readonly Counter _completedCounter; + private readonly Counter _failedCounter; + private readonly Counter _canceledCounter; + private readonly Counter _cacheHitCounter; + private readonly Histogram _firstSegmentHistogram; + private readonly Histogram _speedHistogram; + private long _completed; + private long _failed; + private long _canceled; + private long _cacheHits; + private long _cacheBytes; + private long _firstSegmentSamples; + private long _firstSegmentMilliseconds; + private long _speedSamples; + private double _speedTotal; + private readonly object _speedGate = new(); + private int _queued; + private int _active; + + public TranscodingMetrics() + { + _completedCounter = _meter.CreateCounter("sdw.transcoding.jobs.completed"); + _failedCounter = _meter.CreateCounter("sdw.transcoding.jobs.failed"); + _canceledCounter = _meter.CreateCounter("sdw.transcoding.jobs.canceled"); + _cacheHitCounter = _meter.CreateCounter("sdw.transcoding.cache.hits"); + _firstSegmentHistogram = _meter.CreateHistogram("sdw.transcoding.first_segment.seconds", "s"); + _speedHistogram = _meter.CreateHistogram("sdw.transcoding.speed", "x"); + _meter.CreateObservableGauge("sdw.transcoding.jobs.queued", () => Volatile.Read(ref _queued)); + _meter.CreateObservableGauge("sdw.transcoding.jobs.active", () => Volatile.Read(ref _active)); + _meter.CreateObservableGauge("sdw.transcoding.cache.bytes", () => Interlocked.Read(ref _cacheBytes), "By"); + } + + public void SetQueued(int value) => Volatile.Write(ref _queued, value); + public void SetActive(int value) => Volatile.Write(ref _active, value); + public void SetCacheBytes(long value) => Interlocked.Exchange(ref _cacheBytes, value); + + public void RecordCompleted() + { + Interlocked.Increment(ref _completed); + _completedCounter.Add(1); + } + + public void RecordFailed() + { + Interlocked.Increment(ref _failed); + _failedCounter.Add(1); + } + + public void RecordCanceled() + { + Interlocked.Increment(ref _canceled); + _canceledCounter.Add(1); + } + + public void RecordCacheHit() + { + Interlocked.Increment(ref _cacheHits); + _cacheHitCounter.Add(1); + } + + public void RecordFirstSegment(TimeSpan elapsed) + { + Interlocked.Increment(ref _firstSegmentSamples); + Interlocked.Add(ref _firstSegmentMilliseconds, (long)elapsed.TotalMilliseconds); + _firstSegmentHistogram.Record(elapsed.TotalSeconds); + } + + public void RecordSpeed(double speed) + { + if (!double.IsFinite(speed) || speed <= 0) return; + Interlocked.Increment(ref _speedSamples); + lock (_speedGate) _speedTotal += speed; + _speedHistogram.Record(speed); + } + + public TranscodingMetricsSnapshot Snapshot() + { + var firstSamples = Interlocked.Read(ref _firstSegmentSamples); + var speedSamples = Interlocked.Read(ref _speedSamples); + var completed = Interlocked.Read(ref _completed); + var failed = Interlocked.Read(ref _failed); + double speedTotal; + lock (_speedGate) speedTotal = _speedTotal; + return new TranscodingMetricsSnapshot( + Volatile.Read(ref _queued), + Volatile.Read(ref _active), + completed, + failed, + Interlocked.Read(ref _canceled), + Interlocked.Read(ref _cacheHits), + Interlocked.Read(ref _cacheBytes), + firstSamples == 0 + ? null + : Interlocked.Read(ref _firstSegmentMilliseconds) / 1000d / firstSamples, + speedSamples == 0 ? null : speedTotal / speedSamples, + completed + failed == 0 ? 0 : failed / (double)(completed + failed)); + } + + public void Dispose() => _meter.Dispose(); +} diff --git a/SecondDimensionWatcherReDive/Services/Transcoding/TranscodingModels.cs b/SecondDimensionWatcherReDive/Services/Transcoding/TranscodingModels.cs new file mode 100644 index 0000000..860130d --- /dev/null +++ b/SecondDimensionWatcherReDive/Services/Transcoding/TranscodingModels.cs @@ -0,0 +1,161 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text; + +namespace SecondDimensionWatcherReDive.Services.Transcoding; + +internal enum TranscodingJobState +{ + Queued, + Probing, + Transcoding, + Ready, + Failed, + Canceled +} + +internal enum TranscodingStrategy +{ + Direct, + Remux, + Transcode +} + +internal sealed record TranscodingSelection( + string Quality, + string? AudioLanguage, + string? AudioTrackLabel, + string? SubtitleLanguage, + string? SubtitleTrackLabel) +{ + public static TranscodingSelection Create( + string? quality, + string? audioLanguage, + string? audioTrackLabel, + string? subtitleLanguage, + string? subtitleTrackLabel) + { + var normalizedQuality = string.IsNullOrWhiteSpace(quality) + ? "auto" + : quality.Trim().ToLowerInvariant(); + if (normalizedQuality is not ("auto" or "720p" or "1080p")) + throw new ArgumentException("Quality must be auto, 720p, or 1080p.", nameof(quality)); + + return new TranscodingSelection( + normalizedQuality, + Normalize(audioLanguage), + Normalize(audioTrackLabel), + Normalize(subtitleLanguage), + Normalize(subtitleTrackLabel)); + } + + private static string? Normalize(string? value) + => string.IsNullOrWhiteSpace(value) ? null : value.Trim().ToLowerInvariant(); +} + +internal sealed record TranscodingSource( + Guid AnimationInfoId, + Guid MappingId, + string VirtualPath, + string PhysicalPath, + string FileStore, + string FileName, + long Length, + DateTimeOffset LastModifiedUtc) +{ + public string BuildCacheKey(TranscodingSelection selection) + { + var material = string.Join('\n', + MappingId.ToString("N"), + VirtualPath, + PhysicalPath, + FileStore, + Length.ToString(CultureInfo.InvariantCulture), + LastModifiedUtc.UtcTicks.ToString(CultureInfo.InvariantCulture), + selection.Quality, + selection.AudioLanguage ?? string.Empty, + selection.AudioTrackLabel ?? string.Empty, + selection.SubtitleLanguage ?? string.Empty, + selection.SubtitleTrackLabel ?? string.Empty); + return Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(material))); + } +} + +internal sealed record MediaStreamProbe( + int Index, + string CodecType, + string CodecName, + string? Language, + string? Title, + bool IsDefault, + bool IsForced, + bool IsAttachedPicture); + +internal sealed record MediaProbe( + string Container, + TimeSpan? Duration, + IReadOnlyList Streams) +{ + public MediaStreamProbe? Video => Streams.FirstOrDefault(stream => + stream.CodecType == "video" && !stream.IsAttachedPicture); +} + +internal sealed record TranscodingPlan( + TranscodingStrategy Strategy, + MediaStreamProbe Video, + MediaStreamProbe? Audio, + MediaStreamProbe? BitmapSubtitleToBurn, + IReadOnlyList TextSubtitles, + int UnsupportedSubtitleCount, + bool CopyVideo, + bool CopyAudio); + +internal sealed record TranscodingSubtitle( + string FileName, + string Label, + string? Language, + string Format); + +internal sealed record TranscodingSessionStatus( + Guid SessionId, + string AccessToken, + TranscodingJobState State, + TranscodingStrategy? Strategy, + bool IsPlayable, + bool CacheHit, + double? Progress, + double? Speed, + int? QueuePosition, + string? Error, + string? VideoCodec, + string? AudioCodec, + IReadOnlyList Subtitles, + int UnsupportedSubtitleCount); + +internal sealed record TranscodingContent( + Stream Stream, + string ContentType, + string? FileName, + long? Length, + DateTimeOffset? LastModifiedUtc); + +internal sealed record TranscodingMetricsSnapshot( + int QueuedJobs, + int ActiveJobs, + long CompletedJobs, + long FailedJobs, + long CanceledJobs, + long CacheHits, + long CacheBytes, + double? AverageFirstSegmentSeconds, + double? AverageTranscodeSpeed, + double FailureRate); + +internal sealed class TranscodingQueueFullException() + : InvalidOperationException("The transcoding queue is full. Try again later."); + +internal sealed class TranscodingDisabledException() + : InvalidOperationException("Server-side transcoding is disabled."); + +internal sealed class TranscodingResourceLimitException(string message) + : InvalidOperationException(message); diff --git a/SecondDimensionWatcherReDive/Services/Transcoding/TranscodingOptions.cs b/SecondDimensionWatcherReDive/Services/Transcoding/TranscodingOptions.cs new file mode 100644 index 0000000..20e39fd --- /dev/null +++ b/SecondDimensionWatcherReDive/Services/Transcoding/TranscodingOptions.cs @@ -0,0 +1,27 @@ +namespace SecondDimensionWatcherReDive.Services.Transcoding; + +internal sealed class TranscodingOptions +{ + public const string SectionName = "Transcoding"; + + public bool Enabled { get; set; } = true; + public string CachePath { get; set; } = string.Empty; + public string FfmpegPath { get; set; } = "ffmpeg"; + public string FfprobePath { get; set; } = "ffprobe"; + public int MaxConcurrentJobs { get; set; } = 1; + public int QueueCapacity { get; set; } = 8; + public int MaxThreadsPerJob { get; set; } = 2; + public long MaxMemoryBytesPerJob { get; set; } = 2L * 1024 * 1024 * 1024; + public long MaxDiskBytesPerJob { get; set; } = 20L * 1024 * 1024 * 1024; + public long MaxCacheBytes { get; set; } = 100L * 1024 * 1024 * 1024; + public TimeSpan JobTimeout { get; set; } = TimeSpan.FromHours(6); + public TimeSpan CacheTtl { get; set; } = TimeSpan.FromDays(14); + public TimeSpan CleanupInterval { get; set; } = TimeSpan.FromMinutes(5); + public TimeSpan SessionTtl { get; set; } = TimeSpan.FromMinutes(15); + public int SegmentDurationSeconds { get; set; } = 6; + public int VideoCrf { get; set; } = 23; + public string VideoPreset { get; set; } = "veryfast"; + public string? HardwareVideoEncoder { get; set; } + public string[] HardwareInputArguments { get; set; } = []; + public bool BurnBitmapSubtitles { get; set; } +} diff --git a/SecondDimensionWatcherReDive/Services/Transcoding/TranscodingPlanner.cs b/SecondDimensionWatcherReDive/Services/Transcoding/TranscodingPlanner.cs new file mode 100644 index 0000000..ef4a2db --- /dev/null +++ b/SecondDimensionWatcherReDive/Services/Transcoding/TranscodingPlanner.cs @@ -0,0 +1,126 @@ +namespace SecondDimensionWatcherReDive.Services.Transcoding; + +internal static class TranscodingPlanner +{ + private static readonly HashSet TextSubtitleCodecs = new(StringComparer.OrdinalIgnoreCase) + { + "ass", "jacosub", "microdvd", "mov_text", "mpl2", "realtext", "sami", "ssa", + "subrip", "subviewer", "subviewer1", "text", "vplayer", "webvtt" + }; + + private static readonly HashSet HlsAudioCodecs = new(StringComparer.OrdinalIgnoreCase) + { + "aac", "mp3" + }; + + public static TranscodingPlan CreatePlan( + TranscodingSource source, + MediaProbe probe, + TranscodingSelection selection, + bool burnBitmapSubtitles) + { + var video = probe.Video + ?? throw new InvalidOperationException("The selected file has no video track."); + var audio = SelectStream( + probe.Streams.Where(stream => stream.CodecType == "audio"), + selection.AudioLanguage, + selection.AudioTrackLabel); + var subtitles = probe.Streams.Where(stream => stream.CodecType == "subtitle").ToArray(); + var textSubtitles = subtitles.Where(stream => TextSubtitleCodecs.Contains(stream.CodecName)).ToArray(); + var bitmapSubtitles = subtitles.Where(stream => !TextSubtitleCodecs.Contains(stream.CodecName)).ToArray(); + var hasSubtitlePreference = selection.SubtitleLanguage is not null + || selection.SubtitleTrackLabel is not null; + var bitmapToBurn = burnBitmapSubtitles + && selection.SubtitleLanguage != "off" + ? SelectStream( + bitmapSubtitles, + selection.SubtitleLanguage, + selection.SubtitleTrackLabel, + fallbackToDefault: !hasSubtitlePreference) + : null; + + var extension = Path.GetExtension(source.FileName); + var copyVideo = video.CodecName.Equals("h264", StringComparison.OrdinalIgnoreCase) + && selection.Quality == "auto" + && bitmapToBurn is null; + var copyAudio = audio is null || HlsAudioCodecs.Contains(audio.CodecName); + var direct = IsDirectPlayContainer(extension, video.CodecName, audio?.CodecName) + && selection.Quality == "auto" + && bitmapToBurn is null; + var strategy = direct + ? TranscodingStrategy.Direct + : copyVideo && copyAudio + ? TranscodingStrategy.Remux + : TranscodingStrategy.Transcode; + + return new TranscodingPlan( + strategy, + video, + audio, + bitmapToBurn, + textSubtitles, + bitmapSubtitles.Length - (bitmapToBurn is null ? 0 : 1), + copyVideo, + copyAudio); + } + + private static MediaStreamProbe? SelectStream( + IEnumerable streams, + string? preferredLanguage, + string? preferredLabel, + bool fallbackToDefault = true) + { + var candidates = streams.ToArray(); + if (preferredLabel is not null) + { + var labelMatch = candidates.FirstOrDefault(stream => + string.Equals(stream.Title, preferredLabel, StringComparison.OrdinalIgnoreCase)); + if (labelMatch is not null) return labelMatch; + } + + if (preferredLanguage is not null) + { + var languageMatch = candidates.FirstOrDefault(stream => + LanguagesMatch(stream.Language, preferredLanguage)); + if (languageMatch is not null) return languageMatch; + } + + return fallbackToDefault + ? candidates.FirstOrDefault(stream => stream.IsDefault) ?? candidates.FirstOrDefault() + : null; + } + + private static bool LanguagesMatch(string? actual, string preferred) + { + if (actual is null) return false; + var normalizedActual = NormalizeLanguage(actual); + return normalizedActual == NormalizeLanguage(preferred); + } + + private static string NormalizeLanguage(string language) + { + var normalized = language.Trim().ToLowerInvariant().Replace('_', '-'); + if (normalized is "chi" or "zho" || normalized.StartsWith("zh-", StringComparison.Ordinal)) return "zh"; + if (normalized is "jpn" || normalized.StartsWith("ja-", StringComparison.Ordinal)) return "ja"; + if (normalized is "eng" || normalized.StartsWith("en-", StringComparison.Ordinal)) return "en"; + var separator = normalized.IndexOf('-'); + return separator < 0 ? normalized : normalized[..separator]; + } + + private static bool IsDirectPlayContainer(string extension, string videoCodec, string? audioCodec) + { + if (extension.Equals(".mp4", StringComparison.OrdinalIgnoreCase) + || extension.Equals(".m4v", StringComparison.OrdinalIgnoreCase)) + return videoCodec.Equals("h264", StringComparison.OrdinalIgnoreCase) + && (audioCodec is null || audioCodec.Equals("aac", StringComparison.OrdinalIgnoreCase)); + + if (!extension.Equals(".webm", StringComparison.OrdinalIgnoreCase)) return false; + var supportedVideo = videoCodec.Equals("vp8", StringComparison.OrdinalIgnoreCase) + || videoCodec.Equals("vp9", StringComparison.OrdinalIgnoreCase) + || videoCodec.Equals("av1", StringComparison.OrdinalIgnoreCase); + var supportedAudio = audioCodec is null + || audioCodec.Equals("opus", StringComparison.OrdinalIgnoreCase) + || audioCodec.Equals("vorbis", StringComparison.OrdinalIgnoreCase); + return supportedVideo && supportedAudio; + } +} diff --git a/SecondDimensionWatcherReDive/Utils/FileStore/PlaybackPathResolver.cs b/SecondDimensionWatcherReDive/Utils/FileStore/PlaybackPathResolver.cs new file mode 100644 index 0000000..d818129 --- /dev/null +++ b/SecondDimensionWatcherReDive/Utils/FileStore/PlaybackPathResolver.cs @@ -0,0 +1,29 @@ +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Utils.FileStore; + +internal static class PlaybackPathResolver +{ + public static string ResolveVirtualPath(AnimationInfo info, string? relative) + { + var root = GetAnimationVirtualRoot(info); + if (string.IsNullOrWhiteSpace(relative)) return root; + var trimmed = relative.Trim('/'); + return string.IsNullOrEmpty(trimmed) ? root : $"{root}/{trimmed}"; + } + + private static string GetAnimationVirtualRoot(AnimationInfo info) + { + if (info.Animation is null || info.Season is null) return "/unknown"; + var animationName = SanitizePathSegment(info.Animation.Name); + var subGroup = SanitizePathSegment(info.Group?.Name ?? "Unknown"); + return $"/{animationName}/{subGroup}"; + } + + private static string SanitizePathSegment(string name) + { + var invalid = Path.GetInvalidFileNameChars(); + var sanitized = string.Concat(name.Select(c => invalid.Contains(c) || c == '/' ? '_' : c)).Trim(); + return string.IsNullOrEmpty(sanitized) ? "Unknown" : sanitized; + } +} diff --git a/SecondDimensionWatcherReDive/appsettings.example.json b/SecondDimensionWatcherReDive/appsettings.example.json index da9d5ab..67745cc 100644 --- a/SecondDimensionWatcherReDive/appsettings.example.json +++ b/SecondDimensionWatcherReDive/appsettings.example.json @@ -27,6 +27,29 @@ "SettlingPeriod": "00:00:30", "MissingGracePeriod": "1.00:00:00" }, + // Server-side HLS fallback for containers/codecs that browsers cannot play. + // FFmpeg/ffprobe must be installed. HardwareVideoEncoder is optional (for example + // h264_nvenc or h264_vaapi); a failed hardware attempt automatically retries on CPU. + "Transcoding": { + "Enabled": true, + "CachePath": "/var/lib/sdw-redive/transcode-cache", + "FfmpegPath": "ffmpeg", + "FfprobePath": "ffprobe", + "MaxConcurrentJobs": 1, + "QueueCapacity": 8, + "MaxThreadsPerJob": 2, + "MaxMemoryBytesPerJob": 2147483648, + "MaxDiskBytesPerJob": 21474836480, + "MaxCacheBytes": 107374182400, + "JobTimeout": "06:00:00", + "CacheTtl": "14.00:00:00", + "CleanupInterval": "00:05:00", + "SessionTtl": "00:15:00", + "SegmentDurationSeconds": 6, + "HardwareVideoEncoder": null, + "HardwareInputArguments": [], + "BurnBitmapSubtitles": false + }, "MikananiFeeds": [], "TmdbApiKey": "", diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 9bfc688..f90b183 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -3,8 +3,7 @@ This project is licensed under the **Apache License 2.0** (see [`LICENSE`](LICENSE)). It bundles, redistributes, or links against the third-party software listed below. This file exists to satisfy the notice / attribution requirements those -licenses impose on downstream distributors. Components are used as published -except for the `@ffmpeg/ffmpeg` Parcel compatibility patch noted below. +licenses impose on downstream distributors. Components are used as published. If you ship our binaries or container images, you are also distributing some of these components — please carry this notice along. @@ -71,8 +70,7 @@ time. All are MIT-licensed unless noted. | mediabunny | **MPL-2.0** | https://github.com/Vanilagy/mediabunny | | media-captions | MIT | https://github.com/vidstack/media-captions | | matroska-subtitles | MIT | https://github.com/mathiasvr/matroska-subtitles | -| @ffmpeg/ffmpeg | MIT | https://github.com/ffmpegwasm/ffmpeg.wasm | -| @ffmpeg/core | **GPL-2.0-or-later** | https://github.com/ffmpegwasm/ffmpeg.wasm | +| hls.js | Apache-2.0 | https://github.com/video-dev/hls.js | | clsx | MIT | https://github.com/lukeed/clsx | | dayjs | MIT | https://github.com/iamkun/dayjs | | i18next + react-i18next + i18next-browser-languagedetector | MIT | https://github.com/i18next/i18next | @@ -85,20 +83,23 @@ Build-only / development dependencies (Parcel, Prettier, TypeScript, etc.) are listed in `SecondDimensionWatcherReDive.Client/package.json` — they are not embedded in shipping artifacts and are not enumerated here. -### Browser MKV support +### Browser media support `mediabunny` is distributed under MPL-2.0; modifications to MPL-covered files must remain available under that license. This project does not modify its -sources. - -The unsupported-codec fallback ships the `@ffmpeg/core` WebAssembly binary, -which is GPL-2.0-or-later. Distributors enabling or shipping the web client must -comply with the GPL's source and license requirements for that component. Its -corresponding source is the upstream `ffmpeg.wasm` project linked above. - -The MIT-licensed `@ffmpeg/ffmpeg` wrapper is patched locally so its worker uses -the ESM core loader accepted by Parcel. The complete patch is distributed in -`SecondDimensionWatcherReDive.Client/.yarn/patches/`. +sources. `hls.js` provides Media Source Extensions playback for the server-side +HLS fallback on browsers without native HLS support. + +### FFmpeg / ffprobe runtime dependency + +Server-side media probing, remuxing, transcoding, segmentation, and WebVTT +conversion invoke the separately installed FFmpeg command-line tools. Official +container images install their base distribution's FFmpeg package; Linux system +packages list FFmpeg as a runtime dependency. FFmpeg's effective +license depends on the codecs enabled by the distributor (the current container +build is GPL-licensed); downstream redistributors must carry the corresponding +distro package notices and source offer. The application does not statically link +FFmpeg or copy its libraries into the .NET binaries. --- diff --git a/deployments/podman-compose.yml b/deployments/podman-compose.yml index 7a4f463..afe8adf 100644 --- a/deployments/podman-compose.yml +++ b/deployments/podman-compose.yml @@ -57,6 +57,7 @@ services: MediaLibrary__MissingGracePeriod: "1.00:00:00" PasswordFile: "/app/data/password.json" DataProtection__KeyRingPath: "/app/data/data-protection-keys" + Transcoding__CachePath: "/app/data/transcode-cache" Torrent__Remote__Url: "http://qbittorrent:8080" Torrent__Remote__UserName: "" Torrent__Remote__Password: "" diff --git a/docs/container-deployment.md b/docs/container-deployment.md index f14757e..77aa138 100644 --- a/docs/container-deployment.md +++ b/docs/container-deployment.md @@ -19,7 +19,7 @@ - `downloads` — sdw-redive 和 qbittorrent **共享**，用于下载文件的读写 - `pgdata` — PostgreSQL 数据持久化 - `valkeydata` — Valkey 缓存数据持久化 -- `appdata` — 登录密码文件与运行时敏感配置的 Data Protection 密钥环 +- `appdata` — 登录密码、Data Protection 密钥环与可复用 HLS 转码缓存 ## 快速开始 @@ -120,6 +120,11 @@ podman logs qbittorrent 2>&1 | grep "temporary password" | `MediaLibrary__SettlingPeriod` | 新文件写入完成后的稳定等待时间 | `00:00:30` | | `MediaLibrary__MissingGracePeriod` | 条目缺失后保留观看/审核记录的宽限期 | `1.00:00:00` | | `MediaLibrary__AllowedRoots__0`, `__1`, ... | 允许导入的服务端根目录白名单 | `/media` | +| `Transcoding__CachePath` | 服务端 HLS 分片缓存（应挂载持久卷） | `/app/data/transcode-cache` | +| `Transcoding__MaxConcurrentJobs` | 同时运行的 FFmpeg 任务数 | `1` | +| `Transcoding__QueueCapacity` | 等待队列容量；满时返回 429 | `8` | +| `Transcoding__MaxMemoryBytesPerJob` | 单个 FFmpeg 工作集上限 | `2147483648` | +| `Transcoding__MaxCacheBytes` | HLS 缓存总上限 | `107374182400` | | `Torrent__Remote__Url` | qBittorrent API 地址 | `http://qbittorrent:8080` | | `Valkey__ConnectionString` | Valkey 连接字符串 | 空（使用内存缓存） | | `TmdbApiKey` | TMDB API 密钥 | 空（海报功能不可用） | @@ -135,6 +140,11 @@ podman logs qbittorrent 2>&1 | grep "temporary password" | `AI__CodexAppServer__BearerToken` | app-server / 反向代理要求的 Bearer token | 空 | | `AI__CodexAppServer__PermissionProfile` | `:read-only` 或管理员定义的 permission profile id | `:read-only` | +镜像已包含 FFmpeg。浏览器会继续优先直放兼容源；只有不兼容轨道才进入有界服务端队列， +首个 HLS 分片生成后立即开始播放。`appdata` 必须留有足够空间，缓存会按 TTL/LRU 自动清理。 +硬件转码需要额外映射 GPU 设备/驱动并设置 `Transcoding__HardwareVideoEncoder`；硬件失败会 +自动回退 CPU。 + ### 网页运行时设置 首次登录后可在「设置」中修改 AI/TMDB、qBittorrent、媒体库扫描、异常阈值和 NFS。网页值保存在 PostgreSQL，优先于上表的环境变量；敏感值加密后存储且不会通过 API 回显。`appdata` 卷中的 Data Protection 密钥环必须保留，否则重启后的应用无法解密已保存的密钥。 diff --git a/docs/server-deployment.md b/docs/server-deployment.md index 5814636..2ca46ca 100644 --- a/docs/server-deployment.md +++ b/docs/server-deployment.md @@ -15,6 +15,7 @@ - **ASP.NET Core 10 Runtime** — 应用以 framework-dependent 方式打包，需预先安装运行时 - **PostgreSQL** — 数据库 - **qBittorrent** — 开启 Web API +- **FFmpeg / ffprobe** — 浏览器不兼容媒体的服务端 HLS 探测、封装与转码 ### 安装 ASP.NET Core Runtime @@ -31,6 +32,19 @@ sudo dnf install aspnetcore-runtime-10.0 sudo pacman -S aspnet-runtime-10.0 ``` +通过系统包安装时，FFmpeg 会作为依赖一并安装。使用 tar.gz 或手动部署时请另外安装： + +```bash +# Debian / Ubuntu +sudo apt install ffmpeg + +# Fedora / RHEL +sudo dnf install ffmpeg + +# Arch Linux +sudo pacman -S ffmpeg +``` + ## 安装 ### 快速安装（推荐） @@ -67,6 +81,7 @@ sudo pacman -U sdw-redive-*.pkg.tar.zst | `/etc/sdw-redive/appsettings.yml` | 配置文件（YAML 格式，升级时保留用户修改） | | `/var/lib/sdw-redive/downloads/` | 默认下载存储目录 | | `/var/lib/sdw-redive/data-protection-keys/` | 网页保存的敏感配置所用持久加密密钥环 | +| `/var/lib/sdw-redive/transcode-cache/` | 可复用的 HLS 分片、WebVTT 字幕与缓存清单 | | `/usr/lib/systemd/system/sdw-redive.service` | systemd 服务单元 | 安装时自动创建 `sdw-redive` 系统用户和组用于运行服务。 @@ -103,6 +118,19 @@ MediaLibrary: SettlingPeriod: "00:00:30" MissingGracePeriod: "1.00:00:00" +Transcoding: + Enabled: true + CachePath: /var/lib/sdw-redive/transcode-cache + MaxConcurrentJobs: 1 + QueueCapacity: 8 + MaxThreadsPerJob: 2 + MaxMemoryBytesPerJob: 2147483648 # 2 GiB + MaxDiskBytesPerJob: 21474836480 # 20 GiB / job + MaxCacheBytes: 107374182400 # 100 GiB total + CacheTtl: "14.00:00:00" + SessionTtl: "00:15:00" + SegmentDurationSeconds: 6 + # TMDB API 密钥（用于海报和元数据） TmdbApiKey: "YOUR_TMDB_API_KEY" @@ -131,6 +159,31 @@ Inference: # InstanceName: "sdw-redive:" ``` +### 服务端流式播放与转码 + +网页播放器仍优先使用原文件直放；可由浏览器解码的 MKV 使用按需 Range 拆包。不兼容的 +容器或轨道才会提交到服务端：H.264/AAC 等兼容轨道优先无损封装为 HLS，只有不兼容轨道 +才转码。首个分片完成后即可播放、拖动已生成范围并同步观看进度，源文件不会先完整下载到 +浏览器。文本内封字幕会转换为 WebVTT；位图字幕默认明确标记为不可用，可用 +`Transcoding:BurnBitmapSubtitles=true` 按字幕偏好烧录（会强制视频转码）。 + +同一源版本、音轨/字幕偏好与质量会复用缓存。源文件长度或修改时间变化时会生成新缓存键； +后台按 `CacheTtl` 和 LRU 清理，且始终优先保留正在播放或生成的任务。并发数、队列长度、 +FFmpeg 线程、工作集内存、单任务磁盘、总缓存和任务超时均可配置。队列满时 API 返回 429， +不会启动额外 FFmpeg 进程。登录用户可通过 `GET /api/transcoding/metrics` 查看排队/活动任务、 +成功/失败/取消、失败率、缓存命中与占用、平均首分片时间和平均转码速度。 + +可选硬件编码示例（实际编码器及输入参数取决于主机 FFmpeg 构建和设备映射）： + +```yaml +Transcoding: + HardwareVideoEncoder: h264_nvenc + HardwareInputArguments: ["-hwaccel", "cuda"] +``` + +硬件进程失败时会删除不完整输出并自动用 `libx264` 重试。容器部署还需把对应 GPU 设备和 +驱动映射进容器；未配置硬件编码器时始终使用 CPU。 + `DataProtection:KeyRingPath` 是运行时敏感设置的解密根密钥，不是普通缓存。请持久化并备份该目录，权限应仅允许应用服务账号读取。多副本连接同一个 PostgreSQL 数据库时，**所有副本必须挂载同一份共享密钥环**；否则一个副本写入的 API key/密码无法被其他副本解密。所有副本也必须保持应用内置的 Data Protection application name 一致（`SecondDimensionWatcherReDive`）。 > **注意**：配置文件在包升级时不会被覆盖（标记为 conffile / noreplace）。 diff --git a/packaging/appsettings.yml b/packaging/appsettings.yml index 4cf660f..6bccbac 100644 --- a/packaging/appsettings.yml +++ b/packaging/appsettings.yml @@ -29,6 +29,27 @@ MediaLibrary: SettlingPeriod: "00:00:30" MissingGracePeriod: "1.00:00:00" +# 浏览器不兼容媒体的服务端 HLS 后备路径。硬件编码器失败时自动回退 CPU。 +Transcoding: + Enabled: true + CachePath: /var/lib/sdw-redive/transcode-cache + FfmpegPath: ffmpeg + FfprobePath: ffprobe + MaxConcurrentJobs: 1 + QueueCapacity: 8 + MaxThreadsPerJob: 2 + MaxMemoryBytesPerJob: 2147483648 + MaxDiskBytesPerJob: 21474836480 + MaxCacheBytes: 107374182400 + JobTimeout: "06:00:00" + CacheTtl: "14.00:00:00" + CleanupInterval: "00:05:00" + SessionTtl: "00:15:00" + SegmentDurationSeconds: 6 + HardwareVideoEncoder: null + HardwareInputArguments: [] + BurnBitmapSubtitles: false + # 密码文件路径 PasswordFile: /var/lib/sdw-redive/password.json diff --git a/packaging/nfpm.yaml b/packaging/nfpm.yaml index 3dcd289..3a0a580 100644 --- a/packaging/nfpm.yaml +++ b/packaging/nfpm.yaml @@ -11,6 +11,7 @@ license: "Apache-2.0" depends: - aspnetcore-runtime-10.0 + - ffmpeg recommends: - valkey @@ -19,6 +20,7 @@ overrides: archlinux: depends: - aspnet-runtime-10.0 + - ffmpeg recommends: - valkey @@ -77,6 +79,13 @@ contents: owner: sdw-redive group: sdw-redive + - dst: /var/lib/sdw-redive/transcode-cache + type: dir + file_info: + mode: 0750 + owner: sdw-redive + group: sdw-redive + scripts: postinstall: ./packaging/postinstall.sh preremove: ./packaging/preremove.sh diff --git a/packaging/postinstall.sh b/packaging/postinstall.sh index 1530a32..016e6bd 100755 --- a/packaging/postinstall.sh +++ b/packaging/postinstall.sh @@ -44,6 +44,8 @@ chown -R sdw-redive:sdw-redive /var/lib/sdw-redive # private directory also protects keys created by future application runs. install -d -m 0700 -o sdw-redive -g sdw-redive \ /var/lib/sdw-redive/data-protection-keys +install -d -m 0750 -o sdw-redive -g sdw-redive \ + /var/lib/sdw-redive/transcode-cache if [ -f /var/lib/sdw-redive/password.json ]; then chown sdw-redive:sdw-redive /var/lib/sdw-redive/password.json chmod 0600 /var/lib/sdw-redive/password.json From b2c685c5d6d7eb1a7f4df3064cd10055267452cb Mon Sep 17 00:00:00 2001 From: mahoshojoHCG Date: Sat, 29 Aug 2026 23:31:41 +0800 Subject: [PATCH 03/45] feat: add searchable library and safe release upgrades --- .../Tools/TmdbTool.cs | 23 + .../mock-server.mjs | 560 ++++++-- .../src/Main.tsx | 10 + .../src/components/AppHeader.tsx | 39 +- .../components/SubscriptionPolicySheet.tsx | 104 +- .../src/i18n/locales/en/common.json | 2 + .../src/i18n/locales/en/feeds.json | 11 + .../src/i18n/locales/en/library.json | 70 + .../src/i18n/locales/ja/common.json | 2 + .../src/i18n/locales/ja/feeds.json | 11 + .../src/i18n/locales/ja/library.json | 70 + .../src/i18n/locales/zh-CN/common.json | 2 + .../src/i18n/locales/zh-CN/feeds.json | 11 + .../src/i18n/locales/zh-CN/library.json | 70 + .../src/i18n/resources.ts | 6 + .../src/library/api.ts | 36 + .../src/library/types.ts | 68 + .../src/pages/SearchPage.tsx | 542 ++++++++ .../src/subscriptionPolicy/types.ts | 13 +- .../DataRepository/AnimationInfo.cs | 15 +- .../IAnimationInfoRepository.cs | 2 + .../ILibrarySearchRepository.cs | 14 + .../DataRepository/LibrarySearch.cs | 101 ++ .../DataRepository/ReleaseUpgrade.cs | 100 ++ .../SubscriptionAutomationPolicy.cs | 5 +- .../Feed/AnimationAddRequest.cs | 4 +- .../Feed/IReleaseScoringService.cs | 12 + .../Feed/ReleaseIdentity.cs | 32 + .../FileMappingRepositoryPostgreSqlTests.cs | 131 ++ .../ReleaseUpgradeCoordinatorTests.cs | 122 ++ .../SubscriptionAutomationMatcherTests.cs | 24 + .../SyncFeedTests.cs | 33 +- .../Controllers/Converter.cs | 83 +- .../External/AppJsonSerializerContext.cs | 7 + .../Controllers/External/Library.cs | 84 ++ .../External/SubscriptionAutomationPolicy.cs | 10 +- .../Controllers/LibraryController.cs | 153 +++ .../SubscriptionPoliciesController.cs | 19 +- ...ibrarySearchAndReleaseUpgrades.Designer.cs | 1193 +++++++++++++++++ ...1303_AddLibrarySearchAndReleaseUpgrades.cs | 401 ++++++ .../ApplicationContextModelSnapshot.cs | 212 ++- .../Models/AnimationInfo.cs | 25 + .../Models/ApplicationContext.cs | 123 ++ .../Models/ReleaseUpgradeOperation.cs | 35 + .../Models/SubscriptionAutomationPolicy.cs | 6 + SecondDimensionWatcherReDive/Program.cs | 6 + .../Repositories/AnimationInfoRepository.cs | 26 + ...eMappingRepositoryPostgreSqlTestFixture.cs | 280 +++- .../Repositories/LibrarySearchRepository.cs | 322 +++++ .../Repositories/ReleaseUpgradeRepository.cs | 468 +++++++ .../Repositories/RepositoryConverter.cs | 54 +- .../CompleteDownloadBackgroundService.cs | 11 + .../Services/InferAnimationMetadata.cs | 10 + .../Services/MediaLibraryScanner.cs | 7 +- .../ReleaseUpgradeBackgroundService.cs | 55 + .../Services/SyncFeed.cs | 132 +- .../Feed/MikananiSubscriptionFeedReader.cs | 4 +- .../Utils/Feed/ReleaseScoringService.cs | 104 ++ .../IReleaseUpgradeCoordinator.cs | 27 + .../ReleaseUpgradeCoordinator.cs | 262 ++++ 60 files changed, 6176 insertions(+), 188 deletions(-) create mode 100644 SecondDimensionWatcherReDive.Client/src/i18n/locales/en/library.json create mode 100644 SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/library.json create mode 100644 SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/library.json create mode 100644 SecondDimensionWatcherReDive.Client/src/library/api.ts create mode 100644 SecondDimensionWatcherReDive.Client/src/library/types.ts create mode 100644 SecondDimensionWatcherReDive.Client/src/pages/SearchPage.tsx create mode 100644 SecondDimensionWatcherReDive.Framework/DataRepository/ILibrarySearchRepository.cs create mode 100644 SecondDimensionWatcherReDive.Framework/DataRepository/LibrarySearch.cs create mode 100644 SecondDimensionWatcherReDive.Framework/DataRepository/ReleaseUpgrade.cs create mode 100644 SecondDimensionWatcherReDive.Framework/Feed/IReleaseScoringService.cs create mode 100644 SecondDimensionWatcherReDive.Framework/Feed/ReleaseIdentity.cs create mode 100644 SecondDimensionWatcherReDive.Test/ReleaseUpgradeCoordinatorTests.cs create mode 100644 SecondDimensionWatcherReDive/Controllers/External/Library.cs create mode 100644 SecondDimensionWatcherReDive/Controllers/LibraryController.cs create mode 100644 SecondDimensionWatcherReDive/Migrations/20260829151303_AddLibrarySearchAndReleaseUpgrades.Designer.cs create mode 100644 SecondDimensionWatcherReDive/Migrations/20260829151303_AddLibrarySearchAndReleaseUpgrades.cs create mode 100644 SecondDimensionWatcherReDive/Models/ReleaseUpgradeOperation.cs create mode 100644 SecondDimensionWatcherReDive/Repositories/LibrarySearchRepository.cs create mode 100644 SecondDimensionWatcherReDive/Repositories/ReleaseUpgradeRepository.cs create mode 100644 SecondDimensionWatcherReDive/Services/ReleaseUpgradeBackgroundService.cs create mode 100644 SecondDimensionWatcherReDive/Utils/Feed/ReleaseScoringService.cs create mode 100644 SecondDimensionWatcherReDive/Utils/ReleaseUpgrades/IReleaseUpgradeCoordinator.cs create mode 100644 SecondDimensionWatcherReDive/Utils/ReleaseUpgrades/ReleaseUpgradeCoordinator.cs diff --git a/Plugins/SecondDimensionWatcherReDive.Inference.AI/Tools/TmdbTool.cs b/Plugins/SecondDimensionWatcherReDive.Inference.AI/Tools/TmdbTool.cs index 0d30c24..22ef384 100644 --- a/Plugins/SecondDimensionWatcherReDive.Inference.AI/Tools/TmdbTool.cs +++ b/Plugins/SecondDimensionWatcherReDive.Inference.AI/Tools/TmdbTool.cs @@ -178,6 +178,29 @@ public async Task GetSeasonEpisodesAsync(int tmdbId, int seasonNumber, C } } + public async Task GetExpectedEpisodeCountAsync( + int tmdbId, + int seasonNumber, + CancellationToken cancellationToken) + { + var tmdbClient = GetClient(); + if (tmdbClient is null || seasonNumber <= 0) return null; + + try + { + var show = await tmdbClient.GetTvShowAsync(tmdbId, cancellationToken: cancellationToken); + var count = show?.Seasons? + .FirstOrDefault(season => season.SeasonNumber == seasonNumber) + ?.EpisodeCount; + return count is > 0 ? count : null; + } + catch (Exception ex) + { + LogGetSeasonsFailed(_logger, ex, tmdbId); + return null; + } + } + /// /// Fetches localized name, original name, and overview for a TV show from TMDB, /// using the server's current culture as the language. diff --git a/SecondDimensionWatcherReDive.Client/mock-server.mjs b/SecondDimensionWatcherReDive.Client/mock-server.mjs index f5ee4d4..443809f 100644 --- a/SecondDimensionWatcherReDive.Client/mock-server.mjs +++ b/SecondDimensionWatcherReDive.Client/mock-server.mjs @@ -385,6 +385,7 @@ function initAnimations() { } : null, isAiProcessed: !!entry.animeName, + isMediaLibraryImport: i === 2, }); }); } @@ -692,50 +693,153 @@ let feeds = [ // Per-feed subscription automation policies and historical releases. const POLICY_CREATED_AT = new Date(Date.now() - 86400_000).toISOString(); const subscriptionPolicies = new Map([ - [feeds[0].id, { - feedId: feeds[0].id, - subtitleGroups: ["LoliHouse", "喵萌奶茶屋"], - resolutions: ["1080p"], - codecs: ["HEVC"], - languages: ["简中", "繁中"], - minSizeBytes: 300 * 1024 * 1024, - maxSizeBytes: 1600 * 1024 * 1024, - excludedKeywords: ["合集", "NCOP"], - mode: "ManualConfirm", - createdAt: POLICY_CREATED_AT, - updatedAt: new Date(Date.now() - 3600_000 * 8).toISOString(), - }], - [feeds[1].id, { - feedId: feeds[1].id, - subtitleGroups: ["ANi", "SubsPlease"], - resolutions: ["1080p"], - codecs: [], - languages: ["繁中"], - minSizeBytes: null, - maxSizeBytes: 1400 * 1024 * 1024, - excludedKeywords: ["预告"], - mode: "AutoDownload", - createdAt: POLICY_CREATED_AT, - updatedAt: new Date(Date.now() - 3600_000 * 3).toISOString(), - }], + [ + feeds[0].id, + { + feedId: feeds[0].id, + subtitleGroups: ["LoliHouse", "喵萌奶茶屋"], + resolutions: ["1080p"], + codecs: ["HEVC"], + languages: ["简中", "繁中"], + minSizeBytes: 300 * 1024 * 1024, + maxSizeBytes: 1600 * 1024 * 1024, + excludedKeywords: ["合集", "NCOP"], + mode: "ManualConfirm", + enableVersionUpgrade: true, + minimumUpgradeScore: 80, + upgradeRollbackHours: 72, + createdAt: POLICY_CREATED_AT, + updatedAt: new Date(Date.now() - 3600_000 * 8).toISOString(), + }, + ], + [ + feeds[1].id, + { + feedId: feeds[1].id, + subtitleGroups: ["ANi", "SubsPlease"], + resolutions: ["1080p"], + codecs: [], + languages: ["繁中"], + minSizeBytes: null, + maxSizeBytes: 1400 * 1024 * 1024, + excludedKeywords: ["预告"], + mode: "AutoDownload", + enableVersionUpgrade: false, + minimumUpgradeScore: 25, + upgradeRollbackHours: 72, + createdAt: POLICY_CREATED_AT, + updatedAt: new Date(Date.now() - 3600_000 * 3).toISOString(), + }, + ], ]); const RELEASE_HISTORY_BY_FEED = new Map([ - [feeds[0].id, [ - { id: randomUUID(), title: "[LoliHouse] 葬送的芙莉莲 - 28 [WebRip 1080p HEVC-10bit AAC][简繁内封]", publishedAt: new Date(Date.now() - 3600_000 * 5).toISOString(), sizeBytes: 824 * 1024 * 1024, subtitleGroup: "LoliHouse", resolution: "1080p", codec: "HEVC", languages: ["简中", "繁中"] }, - { id: randomUUID(), title: "[ANi] 葬送的芙莉莲 - 28 [1080P][繁日双语]", publishedAt: new Date(Date.now() - 3600_000 * 12).toISOString(), sizeBytes: 516 * 1024 * 1024, subtitleGroup: "ANi", resolution: "1080p", codec: "AVC", languages: ["繁中", "日语"] }, - { id: randomUUID(), title: "[喵萌奶茶屋] 葬送的芙莉莲 01-28 合集 [1080p HEVC][简繁]", publishedAt: new Date(Date.now() - 86400_000).toISOString(), sizeBytes: 18.4 * 1024 * 1024 * 1024, subtitleGroup: "喵萌奶茶屋", resolution: "1080p", codec: "HEVC", languages: ["简中", "繁中"] }, - { id: randomUUID(), title: "[LoliHouse] 葬送的芙莉莲 - 27 [2160p HEVC][简繁]", publishedAt: new Date(Date.now() - 86400_000 * 2).toISOString(), sizeBytes: 2250 * 1024 * 1024, subtitleGroup: "LoliHouse", resolution: "2160p", codec: "HEVC", languages: ["简中", "繁中"] }, - ]], - [feeds[1].id, [ - { id: randomUUID(), title: "[ANi] 迷宫饭 - 24 [1080P][繁日双语]", publishedAt: new Date(Date.now() - 3600_000 * 7).toISOString(), sizeBytes: 612 * 1024 * 1024, subtitleGroup: "ANi", resolution: "1080p", codec: "AVC", languages: ["繁中", "日语"] }, - { id: randomUUID(), title: "[SubsPlease] Dungeon Meshi - 24 (1080p) [English]", publishedAt: new Date(Date.now() - 3600_000 * 18).toISOString(), sizeBytes: 1380 * 1024 * 1024, subtitleGroup: "SubsPlease", resolution: "1080p", codec: "AVC", languages: ["English"] }, - { id: randomUUID(), title: "[ANi] 迷宫饭 完结纪念预告 [1080P][繁中]", publishedAt: new Date(Date.now() - 86400_000 * 2).toISOString(), sizeBytes: 92 * 1024 * 1024, subtitleGroup: "ANi", resolution: "1080p", codec: "AVC", languages: ["繁中"] }, - ]], - [feeds[2].id, [ - { id: randomUUID(), title: "[LoliHouse] 药屋少女的呢喃 - 24 [WebRip 1080p HEVC][简繁]", publishedAt: new Date(Date.now() - 3600_000 * 10).toISOString(), sizeBytes: 745 * 1024 * 1024, subtitleGroup: "LoliHouse", resolution: "1080p", codec: "HEVC", languages: ["简中", "繁中"] }, - { id: randomUUID(), title: "[ANi] 药屋少女的呢喃 - 24 [720P][繁中]", publishedAt: new Date(Date.now() - 86400_000).toISOString(), sizeBytes: 324 * 1024 * 1024, subtitleGroup: "ANi", resolution: "720p", codec: "AVC", languages: ["繁中"] }, - ]], + [ + feeds[0].id, + [ + { + id: randomUUID(), + title: + "[LoliHouse] 葬送的芙莉莲 - 28 [WebRip 1080p HEVC-10bit AAC][简繁内封]", + publishedAt: new Date(Date.now() - 3600_000 * 5).toISOString(), + sizeBytes: 824 * 1024 * 1024, + subtitleGroup: "LoliHouse", + resolution: "1080p", + codec: "HEVC", + languages: ["简中", "繁中"], + }, + { + id: randomUUID(), + title: "[ANi] 葬送的芙莉莲 - 28 [1080P][繁日双语]", + publishedAt: new Date(Date.now() - 3600_000 * 12).toISOString(), + sizeBytes: 516 * 1024 * 1024, + subtitleGroup: "ANi", + resolution: "1080p", + codec: "AVC", + languages: ["繁中", "日语"], + }, + { + id: randomUUID(), + title: "[喵萌奶茶屋] 葬送的芙莉莲 01-28 合集 [1080p HEVC][简繁]", + publishedAt: new Date(Date.now() - 86400_000).toISOString(), + sizeBytes: 18.4 * 1024 * 1024 * 1024, + subtitleGroup: "喵萌奶茶屋", + resolution: "1080p", + codec: "HEVC", + languages: ["简中", "繁中"], + }, + { + id: randomUUID(), + title: "[LoliHouse] 葬送的芙莉莲 - 27 [2160p HEVC][简繁]", + publishedAt: new Date(Date.now() - 86400_000 * 2).toISOString(), + sizeBytes: 2250 * 1024 * 1024, + subtitleGroup: "LoliHouse", + resolution: "2160p", + codec: "HEVC", + languages: ["简中", "繁中"], + }, + ], + ], + [ + feeds[1].id, + [ + { + id: randomUUID(), + title: "[ANi] 迷宫饭 - 24 [1080P][繁日双语]", + publishedAt: new Date(Date.now() - 3600_000 * 7).toISOString(), + sizeBytes: 612 * 1024 * 1024, + subtitleGroup: "ANi", + resolution: "1080p", + codec: "AVC", + languages: ["繁中", "日语"], + }, + { + id: randomUUID(), + title: "[SubsPlease] Dungeon Meshi - 24 (1080p) [English]", + publishedAt: new Date(Date.now() - 3600_000 * 18).toISOString(), + sizeBytes: 1380 * 1024 * 1024, + subtitleGroup: "SubsPlease", + resolution: "1080p", + codec: "AVC", + languages: ["English"], + }, + { + id: randomUUID(), + title: "[ANi] 迷宫饭 完结纪念预告 [1080P][繁中]", + publishedAt: new Date(Date.now() - 86400_000 * 2).toISOString(), + sizeBytes: 92 * 1024 * 1024, + subtitleGroup: "ANi", + resolution: "1080p", + codec: "AVC", + languages: ["繁中"], + }, + ], + ], + [ + feeds[2].id, + [ + { + id: randomUUID(), + title: "[LoliHouse] 药屋少女的呢喃 - 24 [WebRip 1080p HEVC][简繁]", + publishedAt: new Date(Date.now() - 3600_000 * 10).toISOString(), + sizeBytes: 745 * 1024 * 1024, + subtitleGroup: "LoliHouse", + resolution: "1080p", + codec: "HEVC", + languages: ["简中", "繁中"], + }, + { + id: randomUUID(), + title: "[ANi] 药屋少女的呢喃 - 24 [720P][繁中]", + publishedAt: new Date(Date.now() - 86400_000).toISOString(), + sizeBytes: 324 * 1024 * 1024, + subtitleGroup: "ANi", + resolution: "720p", + codec: "AVC", + languages: ["繁中"], + }, + ], + ], ]); function simulatePolicy(feedId, policy) { @@ -755,28 +859,59 @@ function simulatePolicy(feedId, policy) { if (field === "resolution") { normalized = normalized.replace(/\s/g, ""); const aliases = { - "4K": "2160P", UHD: "2160P", "2160": "2160P", - "1440": "1440P", FHD: "1080P", "1080": "1080P", - HD: "720P", "720": "720P", "576": "576P", "480": "480P", + "4K": "2160P", + UHD: "2160P", + 2160: "2160P", + 1440: "1440P", + FHD: "1080P", + 1080: "1080P", + HD: "720P", + 720: "720P", + 576: "576P", + 480: "480P", }; return aliases[normalized] ?? normalized; } if (field === "codec") { normalized = normalized.replace(/[.\-\s]/g, ""); const aliases = { - H265: "HEVC", X265: "HEVC", H264: "AVC", X264: "AVC", + H265: "HEVC", + X265: "HEVC", + H264: "AVC", + X264: "AVC", }; return aliases[normalized] ?? normalized; } if (field === "languages") { normalized = normalized.replace(/[_\-\s]/g, ""); const aliases = { - CHS: "ZHHANS", SC: "ZHHANS", GB: "ZHHANS", ZHCN: "ZHHANS", - "简体": "ZHHANS", "简中": "ZHHANS", "簡中": "ZHHANS", "简体中文": "ZHHANS", - CHT: "ZHHANT", TC: "ZHHANT", BIG5: "ZHHANT", ZHTW: "ZHHANT", ZHHK: "ZHHANT", - "繁体": "ZHHANT", "繁體": "ZHHANT", "繁中": "ZHHANT", "繁體中文": "ZHHANT", - JPN: "JA", JAP: "JA", "日语": "JA", "日語": "JA", "日本語": "JA", JAPANESE: "JA", - ENG: "EN", "英语": "EN", "英語": "EN", ENGLISH: "EN", + CHS: "ZHHANS", + SC: "ZHHANS", + GB: "ZHHANS", + ZHCN: "ZHHANS", + 简体: "ZHHANS", + 简中: "ZHHANS", + 簡中: "ZHHANS", + 简体中文: "ZHHANS", + CHT: "ZHHANT", + TC: "ZHHANT", + BIG5: "ZHHANT", + ZHTW: "ZHHANT", + ZHHK: "ZHHANT", + 繁体: "ZHHANT", + 繁體: "ZHHANT", + 繁中: "ZHHANT", + 繁體中文: "ZHHANT", + JPN: "JA", + JAP: "JA", + 日语: "JA", + 日語: "JA", + 日本語: "JA", + JAPANESE: "JA", + ENG: "EN", + 英语: "EN", + 英語: "EN", + ENGLISH: "EN", }; return aliases[normalized] ?? normalized; } @@ -786,7 +921,13 @@ function simulatePolicy(feedId, policy) { const actual = actualValues.filter(Boolean); const expected = (expectedValues ?? []).filter(Boolean); if (expected.length === 0) { - return { field, passed: true, actual: actual.join(", ") || null, expected: null, message: "anyValueAllowed" }; + return { + field, + passed: true, + actual: actual.join(", ") || null, + expected: null, + message: "anyValueAllowed", + }; } const normalizedExpected = new Set( expected.map((value) => normalizeAllowedValue(field, value)), @@ -794,28 +935,47 @@ function simulatePolicy(feedId, policy) { const passed = actual.some((value) => normalizedExpected.has(normalizeAllowedValue(field, value)), ); - return { field, passed, actual: actual.join(", ") || null, expected: expected.join(", "), message: passed ? "allowedValueMatched" : "allowedValueMissed" }; + return { + field, + passed, + actual: actual.join(", ") || null, + expected: expected.join(", "), + message: passed ? "allowedValueMatched" : "allowedValueMissed", + }; }; const entries = history.map((item) => { const explanations = [ - checkAllowed("subtitleGroup", [item.subtitleGroup], policy.subtitleGroups), + checkAllowed( + "subtitleGroup", + [item.subtitleGroup], + policy.subtitleGroups, + ), checkAllowed("resolution", [item.resolution], policy.resolutions), checkAllowed("codec", [item.codec], policy.codecs), checkAllowed("languages", item.languages, policy.languages), ]; - const min = typeof policy.minSizeBytes === "number" ? policy.minSizeBytes : null; - const max = typeof policy.maxSizeBytes === "number" ? policy.maxSizeBytes : null; - const sizePassed = (min == null || item.sizeBytes >= min) && (max == null || item.sizeBytes <= max); + const min = + typeof policy.minSizeBytes === "number" ? policy.minSizeBytes : null; + const max = + typeof policy.maxSizeBytes === "number" ? policy.maxSizeBytes : null; + const sizePassed = + (min == null || item.sizeBytes >= min) && + (max == null || item.sizeBytes <= max); explanations.push({ field: "size", passed: sizePassed, actual: formatBytes(item.sizeBytes), - expected: min == null && max == null ? null : `${min == null ? "0 B" : formatBytes(min)} – ${max == null ? "∞" : formatBytes(max)}`, + expected: + min == null && max == null + ? null + : `${min == null ? "0 B" : formatBytes(min)} – ${max == null ? "∞" : formatBytes(max)}`, message: sizePassed ? "withinSizeRange" : "outsideSizeRange", }); const excluded = (policy.excludedKeywords ?? []).filter(Boolean); - const found = excluded.find((keyword) => item.title.toLowerCase().includes(keyword.toLowerCase())); + const found = excluded.find((keyword) => + item.title.toLowerCase().includes(keyword.toLowerCase()), + ); explanations.push({ field: "excludedKeywords", passed: !found, @@ -823,10 +983,21 @@ function simulatePolicy(feedId, policy) { expected: excluded.length > 0 ? excluded.join(", ") : null, message: found ? "excludedKeywordFound" : "noExcludedKeyword", }); - return { id: item.id, title: item.title, publishedAt: item.publishedAt, sizeBytes: item.sizeBytes, matched: explanations.every((reason) => reason.passed), explanations }; + return { + id: item.id, + title: item.title, + publishedAt: item.publishedAt, + sizeBytes: item.sizeBytes, + matched: explanations.every((reason) => reason.passed), + explanations, + }; }); - return { total: entries.length, matched: entries.filter((entry) => entry.matched).length, entries }; + return { + total: entries.length, + matched: entries.filter((entry) => entry.matched).length, + entries, + }; } // WebDAV access tokens @@ -1052,7 +1223,9 @@ function playablePaths() { !entry.isDirectory && /\.(mkv|mp4|webm|avi|flv|wmv|mov|m4v|ts|m2ts)$/i.test(entry.fileName) ) { - paths.push(directory ? `${directory}/${entry.fileName}` : entry.fileName); + paths.push( + directory ? `${directory}/${entry.fileName}` : entry.fileName, + ); } } } @@ -1122,7 +1295,9 @@ function associatedSubtitles(animation, videoPath) { entry.fileName.toLowerCase().startsWith(stem.toLowerCase()), ) .map((entry) => { - const path = directory ? `${directory}/${entry.fileName}` : entry.fileName; + const path = directory + ? `${directory}/${entry.fileName}` + : entry.fileName; const language = entry.fileName.includes("zh-Hans") ? "zh-Hans" : entry.fileName.includes(".en.") @@ -1153,7 +1328,8 @@ if (finishedForPlayback[0]) { }); } const previousEpisode = finishedForPlayback.find( - (animation) => animation.animation?.tmdbId === "209867" && animation.episode === 27, + (animation) => + animation.animation?.tmdbId === "209867" && animation.episode === 27, ); if (previousEpisode) { const path = "Season 1/EP01.mp4"; @@ -1172,7 +1348,8 @@ let mockIncidents = [ type: "feedFailure", severity: "error", title: "Mikan RSS returned HTTP 503", - detail: "The feed could not be refreshed during the last three sync attempts.", + detail: + "The feed could not be refreshed during the last three sync attempts.", sourceId: feeds[0]?.id ?? null, detectedAt: new Date(Date.now() - 42 * 60_000).toISOString(), updatedAt: new Date(Date.now() - 12 * 60_000).toISOString(), @@ -1187,7 +1364,8 @@ let mockIncidents = [ type: "downloadStalled", severity: "warning", title: "Download has not progressed for 20 minutes", - detail: "No peers are currently available. Retry will reannounce the torrent.", + detail: + "No peers are currently available. Retry will reannounce the torrent.", sourceId: finishedForPlayback[1]?.id ?? null, detectedAt: new Date(Date.now() - 25 * 60_000).toISOString(), updatedAt: new Date(Date.now() - 5 * 60_000).toISOString(), @@ -1217,7 +1395,8 @@ let mockIncidents = [ type: "fileMappingFailure", severity: "error", title: "Downloaded files could not be mapped", - detail: "The download completed, but no playable video mapping was produced.", + detail: + "The download completed, but no playable video mapping was produced.", sourceId: finishedForPlayback[2]?.id ?? null, detectedAt: new Date(Date.now() - 2 * 3600_000).toISOString(), updatedAt: new Date(Date.now() - 2 * 3600_000).toISOString(), @@ -1555,7 +1734,11 @@ async function route(method, pathname, searchParams, req, res) { if (method === "GET" && pathname === "/api/playback/context") { const animation = animations.get(searchParams.get("animationInfoId")); const path = searchParams.get("path"); - if (!animation || !animation.isDownloadFinished || !playablePaths().includes(path)) { + if ( + !animation || + !animation.isDownloadFinished || + !playablePaths().includes(path) + ) { return empty(res, 404); } return json(res, { @@ -1576,7 +1759,8 @@ async function route(method, pathname, searchParams, req, res) { if (method === "PUT" && pathname === "/api/playback/progress") { const body = await readBody(req); const animation = animations.get(body.animationInfoId); - if (!animation || !playablePaths().includes(body.path)) return empty(res, 404); + if (!animation || !playablePaths().includes(body.path)) + return empty(res, 404); const positionSeconds = Math.max(0, Number(body.positionSeconds) || 0); const durationSeconds = Math.max(0, Number(body.durationSeconds) || 0); const key = playbackKey(animation.id, body.path); @@ -1586,7 +1770,10 @@ async function route(method, pathname, searchParams, req, res) { (durationSeconds > 0 && positionSeconds / durationSeconds >= 0.9); const updatedAt = new Date().toISOString(); const stored = { - positionSeconds: Math.min(positionSeconds, durationSeconds || positionSeconds), + positionSeconds: Math.min( + positionSeconds, + durationSeconds || positionSeconds, + ), durationSeconds, isWatched, updatedAt, @@ -1599,7 +1786,8 @@ async function route(method, pathname, searchParams, req, res) { if (method === "PUT" && pathname === "/api/playback/watched") { const body = await readBody(req); const animation = animations.get(body.animationInfoId); - if (!animation || !playablePaths().includes(body.path)) return empty(res, 404); + if (!animation || !playablePaths().includes(body.path)) + return empty(res, 404); const key = playbackKey(animation.id, body.path); const previous = playbackProgress.get(key) ?? { positionSeconds: 0, @@ -1638,7 +1826,10 @@ async function route(method, pathname, searchParams, req, res) { if (method === "GET" && pathname === "/api/incidents") { const type = searchParams.get("type"); const includeResolved = searchParams.get("includeResolved") === "true"; - const skip = Math.max(0, parseInt(searchParams.get("skip") ?? "0", 10) || 0); + const skip = Math.max( + 0, + parseInt(searchParams.get("skip") ?? "0", 10) || 0, + ); const take = Math.min( 200, Math.max(1, parseInt(searchParams.get("take") ?? "50", 10) || 50), @@ -1680,7 +1871,8 @@ async function route(method, pathname, searchParams, req, res) { incident.lastRetryError = null; incident.canRetry = false; } else { - incident.lastRetryError = "Free space is still below the configured threshold"; + incident.lastRetryError = + "Free space is still below the configured threshold"; } results.push({ incidentId: incident.id, @@ -1701,15 +1893,21 @@ async function route(method, pathname, searchParams, req, res) { if (method === "POST" && match) { const incident = mockIncidents.find((item) => item.id === match[1]); if (!incident) return empty(res, 404); - if (incident.resolvedAt) return json(res, { error: "Already resolved" }, 409); + if (incident.resolvedAt) + return json(res, { error: "Already resolved" }, 409); incident.retryCount += 1; incident.lastRetryAt = new Date().toISOString(); incident.updatedAt = incident.lastRetryAt; if (incident.type === "diskSpaceLow") { - incident.lastRetryError = "Free space is still below the configured threshold"; + incident.lastRetryError = + "Free space is still below the configured threshold"; return json( res, - { incidentId: incident.id, success: false, error: incident.lastRetryError }, + { + incidentId: incident.id, + success: false, + error: incident.lastRetryError, + }, 422, ); } @@ -1990,6 +2188,165 @@ async function route(method, pathname, searchParams, req, res) { // --- Animation Info --- + if (method === "GET" && pathname === "/api/library/search") { + const q = (searchParams.get("q") ?? "").toLocaleLowerCase(); + const season = searchParams.get("season"); + const episode = searchParams.get("episode"); + const source = searchParams.get("source") ?? "Any"; + const downloadState = searchParams.get("downloadState") ?? "Any"; + const resolution = searchParams.get("resolution"); + const codec = searchParams.get("codec"); + const pathQuery = (searchParams.get("path") ?? "").toLocaleLowerCase(); + const take = Math.min( + 100, + Math.max(1, Number(searchParams.get("take") ?? 30)), + ); + let offset = 0; + try { + if (searchParams.get("cursor")) + offset = + Number( + Buffer.from(searchParams.get("cursor"), "base64url").toString( + "utf8", + ), + ) || 0; + } catch {} + + const mapped = [...animations.values()] + .map((item, index) => { + const group = item.group?.name ?? null; + const itemResolution = /2160p/i.test(item.title) ? "2160p" : "1080p"; + const itemCodec = /HEVC/i.test(item.title) ? "HEVC" : "AVC"; + const name = item.animation?.name ?? item.title; + const virtualPaths = item.isDownloadFinished + ? [ + `/${name}/${group ?? "Unknown"}/${name} S${String(item.season ?? 1).padStart(2, "0")}E${String(item.episode ?? 1).padStart(2, "0")}.mkv`, + ] + : []; + return { + animationInfoId: item.id, + title: item.title, + animationName: item.animation?.name ?? null, + animationOriginalName: item.animation?.originalName ?? null, + tmdbId: item.animation?.tmdbId ?? null, + season: item.season, + episode: item.episode, + subtitleGroup: group, + resolution: itemResolution, + codec: itemCodec, + languages: index % 2 ? ["ja"] : ["zh-CN"], + isDownloadTracked: item.isDownloadTracked, + isDownloadFinished: item.isDownloadFinished, + isMediaLibraryImport: item.isMediaLibraryImport, + isWatched: false, + playbackPositionSeconds: null, + virtualPaths, + releaseScore: 260 + (index % 5) * 55, + scoreReasons: [ + `resolution:${itemResolution}:+200`, + `codec:${itemCodec}:+40`, + ], + publishedAt: item.publishTime, + }; + }) + .filter((item) => { + const haystack = [ + item.title, + item.animationName, + item.animationOriginalName, + item.tmdbId, + item.subtitleGroup, + ...item.virtualPaths, + ] + .filter(Boolean) + .join(" ") + .toLocaleLowerCase(); + if (q && !haystack.includes(q)) return false; + if (season && item.season !== Number(season)) return false; + if (episode && item.episode !== Number(episode)) return false; + if (source === "MediaLibraryImport" && !item.isMediaLibraryImport) + return false; + if (source === "Torrent" && item.isMediaLibraryImport) return false; + if (downloadState === "Downloaded" && !item.isDownloadFinished) + return false; + if ( + downloadState === "Downloading" && + (!item.isDownloadTracked || item.isDownloadFinished) + ) + return false; + if (downloadState === "NotDownloaded" && item.isDownloadTracked) + return false; + if ( + resolution && + item.resolution.toLocaleLowerCase() !== resolution.toLocaleLowerCase() + ) + return false; + if ( + codec && + item.codec.toLocaleLowerCase() !== codec.toLocaleLowerCase() + ) + return false; + if ( + pathQuery && + !item.virtualPaths.some((path) => + path.toLocaleLowerCase().includes(pathQuery), + ) + ) + return false; + return true; + }); + const items = mapped.slice(offset, offset + take); + const nextCursor = + offset + take < mapped.length + ? Buffer.from(String(offset + take)).toString("base64url") + : null; + return json(res, { items, nextCursor }); + } + + if (method === "GET" && pathname === "/api/library/integrity") { + const values = [...animations.values()]; + const current = values[0]; + const candidate = values[21] ?? values[1]; + return json(res, [ + { + tmdbId: current.animation?.tmdbId ?? "209867", + animationName: current.animation?.name ?? "葬送的芙莉莲", + season: 1, + expectedEpisodeCount: 28, + missingEpisodes: [25], + duplicateEpisodes: [ + { episode: 28, releaseIds: [current.id, candidate.id] }, + ], + unidentifiedReleaseCount: 1, + upgradeCandidates: [ + { + currentReleaseId: current.id, + candidateReleaseId: candidate.id, + animationName: current.animation?.name ?? "葬送的芙莉莲", + season: 1, + episode: 28, + currentScore: 300, + candidateScore: 480, + scoreReasons: ["resolution:2160p:+400", "codec:AV1:+80"], + automatic: true, + }, + ], + }, + ]); + } + + if (method === "POST" && pathname === "/api/library/upgrades/execute") { + const body = await readBody(req); + return json(res, { + isSuccess: true, + outcome: body.dryRun ? "ready" : "download_queued", + dryRun: !!body.dryRun, + requiresDownload: true, + operation: body.dryRun ? null : { id: randomUUID() }, + validationErrors: [], + }); + } + if (method === "GET" && pathname === "/api/animationinfo") { const skip = parseInt(searchParams.get("skip") ?? "0", 10); const take = parseInt(searchParams.get("take") ?? "10", 10); @@ -2026,8 +2383,7 @@ async function route(method, pathname, searchParams, req, res) { g.episodes.sort( (a, b) => new Date(b.publishTime).getTime() - - new Date(a.publishTime).getTime() || - b.id.localeCompare(a.id), + new Date(a.publishTime).getTime() || b.id.localeCompare(a.id), ); g.episodeCount = g.episodes.length; return g; @@ -2324,11 +2680,15 @@ async function route(method, pathname, searchParams, req, res) { // POST /api/subscription-policies/:feedId/simulate { - const m = pathname.match(/^\/api\/subscription-policies\/([^/]+)\/simulate$/); + const m = pathname.match( + /^\/api\/subscription-policies\/([^/]+)\/simulate$/, + ); if (method === "POST" && m) { const feedId = decodeURIComponent(m[1]); if (!feeds.some((feed) => feed.id === feedId)) return empty(res, 404); - return readBody(req).then((body) => json(res, simulatePolicy(feedId, body))); + return readBody(req).then((body) => + json(res, simulatePolicy(feedId, body)), + ); } } @@ -2349,18 +2709,41 @@ async function route(method, pathname, searchParams, req, res) { const existing = subscriptionPolicies.get(feedId); const policy = { feedId, - subtitleGroups: Array.isArray(body.subtitleGroups) ? body.subtitleGroups : [], - resolutions: Array.isArray(body.resolutions) ? body.resolutions : [], + subtitleGroups: Array.isArray(body.subtitleGroups) + ? body.subtitleGroups + : [], + resolutions: Array.isArray(body.resolutions) + ? body.resolutions + : [], codecs: Array.isArray(body.codecs) ? body.codecs : [], languages: Array.isArray(body.languages) ? body.languages : [], - minSizeBytes: typeof body.minSizeBytes === "number" ? body.minSizeBytes : null, - maxSizeBytes: typeof body.maxSizeBytes === "number" ? body.maxSizeBytes : null, - excludedKeywords: Array.isArray(body.excludedKeywords) ? body.excludedKeywords : [], - mode: ["NotifyOnly", "ManualConfirm", "AutoDownload"].includes(body.mode) ? body.mode : "ManualConfirm", + minSizeBytes: + typeof body.minSizeBytes === "number" ? body.minSizeBytes : null, + maxSizeBytes: + typeof body.maxSizeBytes === "number" ? body.maxSizeBytes : null, + excludedKeywords: Array.isArray(body.excludedKeywords) + ? body.excludedKeywords + : [], + mode: ["NotifyOnly", "ManualConfirm", "AutoDownload"].includes( + body.mode, + ) + ? body.mode + : "ManualConfirm", + enableVersionUpgrade: !!body.enableVersionUpgrade, + minimumUpgradeScore: Number.isInteger(body.minimumUpgradeScore) + ? body.minimumUpgradeScore + : 25, + upgradeRollbackHours: Number.isInteger(body.upgradeRollbackHours) + ? body.upgradeRollbackHours + : 72, createdAt: existing?.createdAt ?? new Date().toISOString(), updatedAt: new Date().toISOString(), }; - if (policy.minSizeBytes != null && policy.maxSizeBytes != null && policy.minSizeBytes > policy.maxSizeBytes) { + if ( + policy.minSizeBytes != null && + policy.maxSizeBytes != null && + policy.minSizeBytes > policy.maxSizeBytes + ) { return json(res, { error: "Invalid size range" }, 400); } subscriptionPolicies.set(feedId, policy); @@ -2903,8 +3286,7 @@ server.listen(PORT, () => { (animation) => animation.isDownloadFinished, ).length; const downloadingCount = [...animations.values()].filter( - (animation) => - animation.isDownloadTracked && !animation.isDownloadFinished, + (animation) => animation.isDownloadTracked && !animation.isDownloadFinished, ).length; console.log( ` ${animations.size} anime entries (${finishedCount} finished, ${downloadingCount} active downloads, rest untracked)`, diff --git a/SecondDimensionWatcherReDive.Client/src/Main.tsx b/SecondDimensionWatcherReDive.Client/src/Main.tsx index 47a5f24..6e0a634 100644 --- a/SecondDimensionWatcherReDive.Client/src/Main.tsx +++ b/SecondDimensionWatcherReDive.Client/src/Main.tsx @@ -15,6 +15,7 @@ import { LoginPage } from "./pages/LoginPage"; import { EpisodeListPage, MainPage } from "./pages/MainPage"; import { MetadataReviewPage } from "./pages/MetadataReviewPage"; import { PlayerPage } from "./pages/PlayerPage"; +import { SearchPage } from "./pages/SearchPage"; import { SettingsPage } from "./pages/SettingsPage"; import { TasksPage } from "./pages/TasksPage"; @@ -73,6 +74,15 @@ const router = createBrowserRouter([ ), errorElement: , }, + { + path: "/search", + element: ( + + + + ), + errorElement: , + }, { path: "/play/:animationId", element: ( diff --git a/SecondDimensionWatcherReDive.Client/src/components/AppHeader.tsx b/SecondDimensionWatcherReDive.Client/src/components/AppHeader.tsx index 0a626d6..33cfcad 100644 --- a/SecondDimensionWatcherReDive.Client/src/components/AppHeader.tsx +++ b/SecondDimensionWatcherReDive.Client/src/components/AppHeader.tsx @@ -15,6 +15,7 @@ import { List, Menu, MessageSquare, + Search, Settings, User, } from "lucide-react"; @@ -44,6 +45,7 @@ interface NavItem { const createNavItems = (incidentCount?: number): NavItem[] => [ { icon: , labelKey: "nav.home", path: "/" }, + { icon: , labelKey: "nav.search", path: "/search" }, { icon: , labelKey: "nav.downloading", @@ -223,6 +225,17 @@ export const AppHeader: React.FC = () => { const { data: incidents } = useIncidents({ take: 1 }); const navigate = useNavigate(); const items = createNavItems(incidents?.openCount); + const location = useLocation(); + const [searchQuery, setSearchQuery] = React.useState( + location.pathname === "/search" + ? (new URLSearchParams(location.search).get("q") ?? "") + : "", + ); + + React.useEffect(() => { + if (location.pathname === "/search") + setSearchQuery(new URLSearchParams(location.search).get("q") ?? ""); + }, [location.pathname, location.search]); return (
@@ -248,7 +261,31 @@ export const AppHeader: React.FC = () => { ))} -
+
+ {status ? ( +
{ + event.preventDefault(); + const q = searchQuery.trim(); + navigate(q ? `/search?q=${encodeURIComponent(q)}` : "/search"); + }} + > + +
+ ) : null} {status ? ( ) : ( diff --git a/SecondDimensionWatcherReDive.Client/src/components/SubscriptionPolicySheet.tsx b/SecondDimensionWatcherReDive.Client/src/components/SubscriptionPolicySheet.tsx index f212a31..c85193d 100644 --- a/SecondDimensionWatcherReDive.Client/src/components/SubscriptionPolicySheet.tsx +++ b/SecondDimensionWatcherReDive.Client/src/components/SubscriptionPolicySheet.tsx @@ -130,6 +130,13 @@ export const SubscriptionPolicySheet: React.FC< draft.minSizeBytes != null && draft.maxSizeBytes != null && draft.minSizeBytes > draft.maxSizeBytes; + const upgradePolicyIsInvalid = + !Number.isInteger(draft.minimumUpgradeScore) || + draft.minimumUpgradeScore < 1 || + draft.minimumUpgradeScore > 1000 || + !Number.isInteger(draft.upgradeRollbackHours) || + draft.upgradeRollbackHours < 1 || + draft.upgradeRollbackHours > 720; const updateDraft = React.useCallback( (update: React.SetStateAction) => { @@ -153,7 +160,7 @@ export const SubscriptionPolicySheet: React.FC< ); const handleSave = React.useCallback(async () => { - if (!feed || saving || sizeRangeIsInvalid) return; + if (!feed || saving || sizeRangeIsInvalid || upgradePolicyIsInvalid) return; setSaving(true); try { const saved = await saveSubscriptionPolicy(feed.id, draft); @@ -172,7 +179,16 @@ export const SubscriptionPolicySheet: React.FC< } finally { setSaving(false); } - }, [feed, saving, sizeRangeIsInvalid, draft, onPolicyChanged, addToast, t]); + }, [ + feed, + saving, + sizeRangeIsInvalid, + upgradePolicyIsInvalid, + draft, + onPolicyChanged, + addToast, + t, + ]); const handleDelete = React.useCallback(async () => { if (!feed || !hasSavedPolicy) return; @@ -396,10 +412,85 @@ export const SubscriptionPolicySheet: React.FC<
+
+ + +
+ + +
+ {upgradePolicyIsInvalid ? ( +

+ + {t("automation.upgrades.rangeError")} +

+ ) : null} +
+
@@ -457,7 +548,12 @@ export const SubscriptionPolicySheet: React.FC< + + +
+
+

+ {t("filters.title", { count: activeFilterCount })} +

+ +
+
+ + + + + + + + + + + +
+
+ +
+
+

+ {t("results.title")} +

+ {data ? ( + + {t("results.pageCount", { count: data.items.length })} + + ) : null} +
+ {isLoading ? ( +
+ +
+ ) : error ? ( + } title={t("results.error")} /> + ) : !data?.items.length ? ( + } title={t("results.empty")} /> + ) : ( +
+ {data.items.map((item) => ( +
+
+
+
+ + {item.season != null && item.episode != null + ? `S${String(item.season).padStart(2, "0")}E${String(item.episode).padStart(2, "0")}` + : t("results.unidentified")} + + + {item.isMediaLibraryImport + ? t("values.MediaLibraryImport") + : t("values.Torrent")} + + + {new Date(item.publishedAt).toLocaleDateString()} + +
+

+ {item.animationName ?? item.title} +

+ {item.animationOriginalName ? ( +

+ {item.animationOriginalName} +

+ ) : null} +
+ {[ + item.subtitleGroup, + item.resolution, + item.codec, + ...item.languages, + ] + .filter(Boolean) + .map((value) => ( + + {value} + + ))} +
+ {item.virtualPaths.map((path) => ( +

+ {path} +

+ ))} +
+
+

+ {item.releaseScore} +

+

{t("results.score")}

+

+ {item.isDownloadFinished + ? t("values.Downloaded") + : item.isDownloadTracked + ? t("values.Downloading") + : t("values.NotDownloaded")} +

+
+
+ {item.scoreReasons.length ? ( +
    + {item.scoreReasons.map((reason) => ( +
  • + {reason}
    • + ))} +
+ ) : null} +
+ ))} +
+ )} + {data?.nextCursor ? ( +
+ +
+ ) : null} +
+ +
+
+
+

+ {t("integrity.title")} +

+

+ {t("integrity.description")} +

+
+ +
+
+ {integrity + ?.filter( + (item) => + item.missingEpisodes.length || + item.duplicateEpisodes.length || + item.unidentifiedReleaseCount || + item.upgradeCandidates.length, + ) + .map((item) => ( +
+
+
+

+ {item.animationName} +

+

+ {t("integrity.season", { + season: item.season, + count: item.expectedEpisodeCount ?? "?", + })} +

+
+ {item.missingEpisodes.length === 0 && + item.duplicateEpisodes.length === 0 && + item.unidentifiedReleaseCount === 0 ? ( + + ) : ( + + )} +
+
+ + entry.episode) + .join(", ") || "—" + } + /> + +
+ {item.upgradeCandidates.map((candidate) => ( +
+
+
+

+ {t("upgrade.episode", { episode: candidate.episode })} +

+

+ {candidate.currentScore} → {candidate.candidateScore}{" "} + (+{candidate.candidateScore - candidate.currentScore}) +

+
+
+ + +
+
+
    + {candidate.scoreReasons.map((reason) => ( +
  • + {reason}
    • + ))} +
+
+ ))} +
+ ))} +
+
+ + ); +}; + +const FilterInput: React.FC<{ + label: string; + name: string; + params: URLSearchParams; + onChange: (name: string, value: string) => void; + type?: string; +}> = ({ label, name, params, onChange, type = "text" }) => ( + +); + +const FilterSelect: React.FC<{ + label: string; + name: string; + params: URLSearchParams; + onChange: (name: string, value: string) => void; + options: string[]; + t: (key: string) => string; +}> = ({ label, name, params, onChange, options, t }) => ( + +); + +const Metric: React.FC<{ label: string; value: string }> = ({ + label, + value, +}) => ( +
+
{label}
+
+ {value} +
+
+); diff --git a/SecondDimensionWatcherReDive.Client/src/subscriptionPolicy/types.ts b/SecondDimensionWatcherReDive.Client/src/subscriptionPolicy/types.ts index 7456f60..2ca20d9 100644 --- a/SecondDimensionWatcherReDive.Client/src/subscriptionPolicy/types.ts +++ b/SecondDimensionWatcherReDive.Client/src/subscriptionPolicy/types.ts @@ -1,7 +1,5 @@ export type SubscriptionPolicyMode = - | "NotifyOnly" - | "ManualConfirm" - | "AutoDownload"; + "NotifyOnly" | "ManualConfirm" | "AutoDownload"; export interface ISubscriptionPolicyDraft { subtitleGroups: string[]; @@ -12,6 +10,9 @@ export interface ISubscriptionPolicyDraft { maxSizeBytes: number | null; excludedKeywords: string[]; mode: SubscriptionPolicyMode; + enableVersionUpgrade: boolean; + minimumUpgradeScore: number; + upgradeRollbackHours: number; } export interface ISubscriptionPolicy extends ISubscriptionPolicyDraft { @@ -51,6 +52,9 @@ export const createEmptySubscriptionPolicy = (): ISubscriptionPolicyDraft => ({ maxSizeBytes: null, excludedKeywords: [], mode: "ManualConfirm", + enableVersionUpgrade: false, + minimumUpgradeScore: 25, + upgradeRollbackHours: 72, }); export const toSubscriptionPolicyDraft = ( @@ -64,4 +68,7 @@ export const toSubscriptionPolicyDraft = ( maxSizeBytes: policy.maxSizeBytes, excludedKeywords: [...policy.excludedKeywords], mode: policy.mode, + enableVersionUpgrade: policy.enableVersionUpgrade ?? false, + minimumUpgradeScore: policy.minimumUpgradeScore ?? 25, + upgradeRollbackHours: policy.upgradeRollbackHours ?? 72, }); diff --git a/SecondDimensionWatcherReDive.Framework/DataRepository/AnimationInfo.cs b/SecondDimensionWatcherReDive.Framework/DataRepository/AnimationInfo.cs index e02fed8..be05aaa 100644 --- a/SecondDimensionWatcherReDive.Framework/DataRepository/AnimationInfo.cs +++ b/SecondDimensionWatcherReDive.Framework/DataRepository/AnimationInfo.cs @@ -34,4 +34,17 @@ public sealed record AnimationInfo( Guid? DownloadAttemptId = null, Guid? DownloadCancellationId = null, Guid? MediaLibrarySourceId = null, - DateTimeOffset? MediaLibraryMissingSince = null); + DateTimeOffset? MediaLibraryMissingSince = null, + string? ReleaseIdentity = null, + string? FeedItemGuid = null, + string? EnclosureId = null, + string? TorrentInfoHash = null, + string? ReleaseSubtitleGroup = null, + string? ReleaseResolution = null, + string? ReleaseCodec = null, + IReadOnlyList? ReleaseLanguages = null, + int ReleaseScore = 0, + string? ReleaseScoreReasonsJson = null, + int? ExpectedEpisodeCount = null, + DateTimeOffset? IngestedAt = null, + bool IsActiveRelease = true); diff --git a/SecondDimensionWatcherReDive.Framework/DataRepository/IAnimationInfoRepository.cs b/SecondDimensionWatcherReDive.Framework/DataRepository/IAnimationInfoRepository.cs index 60e555f..47c85b0 100644 --- a/SecondDimensionWatcherReDive.Framework/DataRepository/IAnimationInfoRepository.cs +++ b/SecondDimensionWatcherReDive.Framework/DataRepository/IAnimationInfoRepository.cs @@ -56,6 +56,8 @@ Task> GetDownloadedWithoutFileMappingsAsync( Task AddAsync(AnimationInfo info, CancellationToken cancellationToken); + Task TryAddReleaseAsync(AnimationInfo info, CancellationToken cancellationToken); + Task UpdateAsync(AnimationInfo info, CancellationToken cancellationToken); Task TryStartDownloadAsync( diff --git a/SecondDimensionWatcherReDive.Framework/DataRepository/ILibrarySearchRepository.cs b/SecondDimensionWatcherReDive.Framework/DataRepository/ILibrarySearchRepository.cs new file mode 100644 index 0000000..626da76 --- /dev/null +++ b/SecondDimensionWatcherReDive.Framework/DataRepository/ILibrarySearchRepository.cs @@ -0,0 +1,14 @@ +namespace SecondDimensionWatcherReDive.Framework.DataRepository; + +public interface ILibrarySearchRepository +{ + Task SearchAsync( + LibrarySearchRequest request, + CancellationToken cancellationToken); + + Task> GetIntegrityAsync( + string? tmdbId, + int? season, + CancellationToken cancellationToken); +} + diff --git a/SecondDimensionWatcherReDive.Framework/DataRepository/LibrarySearch.cs b/SecondDimensionWatcherReDive.Framework/DataRepository/LibrarySearch.cs new file mode 100644 index 0000000..c7bd196 --- /dev/null +++ b/SecondDimensionWatcherReDive.Framework/DataRepository/LibrarySearch.cs @@ -0,0 +1,101 @@ +namespace SecondDimensionWatcherReDive.Framework.DataRepository; + +public enum LibraryDownloadState +{ + Any, + NotDownloaded, + Downloading, + Downloaded +} + +public enum LibraryWatchState +{ + Any, + Unwatched, + InProgress, + Watched +} + +public enum LibrarySourceKind +{ + Any, + Torrent, + MediaLibraryImport +} + +public enum LibrarySearchSort +{ + PublishedDescending, + TitleAscending, + EpisodeAscending, + ScoreDescending +} + +public sealed record LibrarySearchRequest( + string? Query, + int? Season, + int? Episode, + string? SubtitleGroup, + string? Resolution, + string? Codec, + string? Language, + LibraryDownloadState DownloadState, + LibraryWatchState WatchState, + string? VirtualPath, + LibrarySourceKind Source, + LibrarySearchSort Sort, + string? Cursor, + int Take, + Guid UserId); + +public sealed record LibrarySearchItem( + Guid AnimationInfoId, + string Title, + string? AnimationName, + string? AnimationOriginalName, + string? TmdbId, + int? Season, + int? Episode, + string? SubtitleGroup, + string? Resolution, + string? Codec, + IReadOnlyList Languages, + bool IsDownloadTracked, + bool IsDownloadFinished, + bool IsMediaLibraryImport, + bool IsWatched, + double? PlaybackPositionSeconds, + IReadOnlyList VirtualPaths, + int ReleaseScore, + IReadOnlyList ScoreReasons, + DateTimeOffset PublishedAt); + +public sealed record LibrarySearchResult( + IReadOnlyList Items, + string? NextCursor); + +public sealed record EpisodeDuplicate( + int Episode, + IReadOnlyList ReleaseIds); + +public sealed record ReleaseUpgradeCandidate( + Guid CurrentReleaseId, + Guid CandidateReleaseId, + string AnimationName, + int Season, + int Episode, + int CurrentScore, + int CandidateScore, + IReadOnlyList ScoreReasons, + bool Automatic); + +public sealed record LibraryIntegritySummary( + string TmdbId, + string AnimationName, + int Season, + int? ExpectedEpisodeCount, + IReadOnlyList MissingEpisodes, + IReadOnlyList DuplicateEpisodes, + int UnidentifiedReleaseCount, + IReadOnlyList UpgradeCandidates); + diff --git a/SecondDimensionWatcherReDive.Framework/DataRepository/ReleaseUpgrade.cs b/SecondDimensionWatcherReDive.Framework/DataRepository/ReleaseUpgrade.cs new file mode 100644 index 0000000..547cd5d --- /dev/null +++ b/SecondDimensionWatcherReDive.Framework/DataRepository/ReleaseUpgrade.cs @@ -0,0 +1,100 @@ +namespace SecondDimensionWatcherReDive.Framework.DataRepository; + +public enum ReleaseUpgradeStatus +{ + Downloading, + Verifying, + Applied, + Failed, + RolledBack, + Completed +} + +public enum ReleaseUpgradeMappingKind +{ + Previous, + Candidate +} + +public sealed record ReleaseUpgradeOperation( + Guid Id, + Guid CurrentReleaseId, + Guid CandidateReleaseId, + ReleaseUpgradeStatus Status, + int CurrentScore, + int CandidateScore, + DateTimeOffset CreatedAt, + DateTimeOffset? VerifiedAt, + DateTimeOffset? AppliedAt, + DateTimeOffset? RollbackUntil, + DateTimeOffset? CompletedAt, + string? FailureSummary); + +public sealed record ReleaseUpgradeMappingSnapshot( + Guid Id, + Guid OperationId, + ReleaseUpgradeMappingKind Kind, + Guid OriginalMappingId, + Guid AnimationInfoId, + string VirtualPath, + string PhysicalPath, + string FileStore); + +public sealed record ReleaseUpgradeActivation( + ReleaseUpgradeOperation Operation, + IReadOnlyList PreviousMappings, + IReadOnlyList CandidateMappings); + +public sealed record ReleaseUpgradeMutationResult( + bool IsSuccess, + string Outcome, + ReleaseUpgradeOperation? Operation); + +public interface IReleaseUpgradeRepository +{ + Task> GetCandidatesAsync( + bool automaticOnly, + int take, + CancellationToken cancellationToken); + + Task TryBeginAsync( + ReleaseUpgradeCandidate candidate, + DateTimeOffset createdAt, + CancellationToken cancellationToken); + + Task FindActiveByCandidateAsync( + Guid candidateReleaseId, + CancellationToken cancellationToken); + + Task> GetReadyCandidateIdsAsync( + int take, + CancellationToken cancellationToken); + + Task GetActivationAsync( + Guid candidateReleaseId, + CancellationToken cancellationToken); + + Task ActivateAsync( + Guid operationId, + DateTimeOffset verifiedAt, + DateTimeOffset rollbackUntil, + CancellationToken cancellationToken); + + Task MarkFailedAsync( + Guid operationId, + string failureSummary, + CancellationToken cancellationToken); + + Task RollbackAsync( + Guid operationId, + DateTimeOffset rolledBackAt, + CancellationToken cancellationToken); + + Task> GetHistoryAsync( + int take, + CancellationToken cancellationToken); + + Task CompleteExpiredAsync( + DateTimeOffset completedAt, + CancellationToken cancellationToken); +} diff --git a/SecondDimensionWatcherReDive.Framework/DataRepository/SubscriptionAutomationPolicy.cs b/SecondDimensionWatcherReDive.Framework/DataRepository/SubscriptionAutomationPolicy.cs index 11fec01..df78f8c 100644 --- a/SecondDimensionWatcherReDive.Framework/DataRepository/SubscriptionAutomationPolicy.cs +++ b/SecondDimensionWatcherReDive.Framework/DataRepository/SubscriptionAutomationPolicy.cs @@ -15,4 +15,7 @@ public sealed record SubscriptionAutomationPolicy( IReadOnlyList ExcludedKeywords, SubscriptionAutomationMode Mode, DateTimeOffset CreatedAt, - DateTimeOffset UpdatedAt); + DateTimeOffset UpdatedAt, + bool EnableVersionUpgrade = false, + int MinimumUpgradeScore = 25, + int UpgradeRollbackHours = 72); diff --git a/SecondDimensionWatcherReDive.Framework/Feed/AnimationAddRequest.cs b/SecondDimensionWatcherReDive.Framework/Feed/AnimationAddRequest.cs index 294951c..ace7c0f 100644 --- a/SecondDimensionWatcherReDive.Framework/Feed/AnimationAddRequest.cs +++ b/SecondDimensionWatcherReDive.Framework/Feed/AnimationAddRequest.cs @@ -8,4 +8,6 @@ public record AnimationAddRequest( string DownloadType, string AdditionalDownloadInfo, Guid? FeedId = null, - long? ContentLength = null); + long? ContentLength = null, + string? FeedItemGuid = null, + string? EnclosureId = null); diff --git a/SecondDimensionWatcherReDive.Framework/Feed/IReleaseScoringService.cs b/SecondDimensionWatcherReDive.Framework/Feed/IReleaseScoringService.cs new file mode 100644 index 0000000..07780bb --- /dev/null +++ b/SecondDimensionWatcherReDive.Framework/Feed/IReleaseScoringService.cs @@ -0,0 +1,12 @@ +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Framework.Feed; + +public sealed record ReleaseScore(int Value, IReadOnlyList Reasons); + +public interface IReleaseScoringService +{ + ReleaseScore Score( + SubscriptionReleaseMetadata metadata, + SubscriptionAutomationPolicy? policy); +} diff --git a/SecondDimensionWatcherReDive.Framework/Feed/ReleaseIdentity.cs b/SecondDimensionWatcherReDive.Framework/Feed/ReleaseIdentity.cs new file mode 100644 index 0000000..668b622 --- /dev/null +++ b/SecondDimensionWatcherReDive.Framework/Feed/ReleaseIdentity.cs @@ -0,0 +1,32 @@ +using System.Security.Cryptography; +using System.Text; + +namespace SecondDimensionWatcherReDive.Framework.Feed; + +public static class ReleaseIdentity +{ + public static string Create( + Guid? feedId, + string? feedItemGuid, + string? enclosureId, + string? torrentInfoHash, + string downloadUrl) + { + if (!string.IsNullOrWhiteSpace(torrentInfoHash)) + return $"torrent:{torrentInfoHash.Trim().ToLowerInvariant()}"; + + var source = !string.IsNullOrWhiteSpace(feedItemGuid) + ? $"feed:{feedId?.ToString("N") ?? "static"}:{feedItemGuid.Trim()}" + : !string.IsNullOrWhiteSpace(enclosureId) + ? $"enclosure:{feedId?.ToString("N") ?? "static"}:{enclosureId.Trim()}" + : $"url:{downloadUrl.Trim()}"; + return $"external:{Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(source))).ToLowerInvariant()}"; + } + + public static string CreateMediaImport(Guid sourceId, string fileStore, string storePath) + { + var source = $"{sourceId:N}\n{fileStore}\n{storePath}"; + return $"import:{Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(source))).ToLowerInvariant()}"; + } +} + diff --git a/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/FileMappingRepositoryPostgreSqlTests.cs b/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/FileMappingRepositoryPostgreSqlTests.cs index 1a06401..91313ad 100644 --- a/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/FileMappingRepositoryPostgreSqlTests.cs +++ b/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/FileMappingRepositoryPostgreSqlTests.cs @@ -88,6 +88,137 @@ static async Task WriteAsync(FileMappingRepositoryPostgreSqlTestFixture fi CollectionAssert.AreEquivalent(new long[] { 0, 1 }, versions); } + [TestMethod] + public async Task StableReleaseIdentity_ConcurrentRepositoryInserts_PersistExactlyOnce() + { + var identity = "torrent:" + Guid.NewGuid().ToString("N"); + + var results = await Task.WhenAll( + Fixture.TryAddReleaseAsync(identity, CancellationToken.None), + Fixture.TryAddReleaseAsync(identity, CancellationToken.None)); + + Assert.AreEqual(1, results.Count(result => result)); + Assert.AreEqual(1, await Fixture.CountReleaseIdentityAsync(identity, CancellationToken.None)); + } + + [TestMethod] + public async Task Search_ComposesImportPathAndWatchFilters_AndCursorIgnoresConcurrentInsert() + { + var scenario = await Fixture.SeedLibraryScenarioAsync(CancellationToken.None); + var filtered = await Fixture.SearchAsync(new LibrarySearchRequest( + "Attack", 1, 2, null, "2160p", "AV1", "ja", + LibraryDownloadState.Downloaded, + LibraryWatchState.InProgress, + "Imported", + LibrarySourceKind.MediaLibraryImport, + LibrarySearchSort.ScoreDescending, + null, + 20, + scenario.UserId), + CancellationToken.None); + + Assert.HasCount(1, filtered.Items); + Assert.AreEqual(scenario.ImportedReleaseId, filtered.Items[0].AnimationInfoId); + Assert.IsTrue(filtered.Items[0].IsMediaLibraryImport); + + var firstPage = await Fixture.SearchAsync(AnySearch(scenario.UserId, null, 2), CancellationToken.None); + Assert.IsNotNull(firstPage.NextCursor); + var insertedId = await Fixture.InsertConcurrentSearchReleaseAsync(CancellationToken.None); + var secondPage = await Fixture.SearchAsync( + AnySearch(scenario.UserId, firstPage.NextCursor, 2), + CancellationToken.None); + + Assert.IsFalse(secondPage.Items.Any(item => item.AnimationInfoId == insertedId)); + Assert.IsFalse(firstPage.Items.Select(item => item.AnimationInfoId) + .Intersect(secondPage.Items.Select(item => item.AnimationInfoId)).Any()); + } + + [TestMethod] + public async Task Integrity_ReportsMissingDuplicateUnidentifiedAndExplainableUpgrade() + { + await Fixture.SeedLibraryScenarioAsync(CancellationToken.None); + + var summaries = await Fixture.GetIntegrityAsync(CancellationToken.None); + + Assert.HasCount(1, summaries); + var summary = summaries[0]; + CollectionAssert.AreEqual(new[] { 3 }, summary.MissingEpisodes.ToArray()); + Assert.HasCount(1, summary.DuplicateEpisodes); + Assert.AreEqual(1, summary.DuplicateEpisodes[0].Episode); + Assert.AreEqual(1, summary.UnidentifiedReleaseCount); + Assert.HasCount(1, summary.UpgradeCandidates); + CollectionAssert.Contains( + summary.UpgradeCandidates[0].ScoreReasons.ToArray(), + "resolution:2160p:+400"); + } + + [TestMethod] + public async Task Migration_CreatesReleaseUniquenessAndSearchIndexes() + { + var indexes = await Fixture.GetLibraryIndexNamesAsync(CancellationToken.None); + + var expected = new[] + { + "UX_AnimationInfo_ReleaseIdentity", + "IX_AnimationInfo_Title_Trgm", + "IX_Animations_Name_Trgm", + "IX_Animations_OriginalName_Trgm", + "IX_AnimationGroups_Name_Trgm", + "IX_FileMappings_VirtualPath_Trgm", + "IX_AnimationInfo_ReleaseLanguages_Gin" + }; + Assert.IsTrue(expected.All(indexes.Contains), + $"Missing indexes: {string.Join(", ", expected.Except(indexes))}"); + } + + [TestMethod] + public async Task UpgradeRace_ClaimsOnce_AtomicallySwapsMappings_AndRollsBack() + { + var scenario = await Fixture.SeedUpgradeScenarioAsync(CancellationToken.None); + var claims = await Task.WhenAll( + Fixture.BeginUpgradeAsync(scenario.Candidate, CancellationToken.None), + Fixture.BeginUpgradeAsync(scenario.Candidate, CancellationToken.None)); + var operation = claims.Single(claim => claim is not null)!; + var recoverableCandidates = await Fixture.GetReadyUpgradeCandidateIdsAsync(CancellationToken.None); + CollectionAssert.Contains(recoverableCandidates.ToArray(), scenario.Candidate.CandidateReleaseId); + + var beforeCurrent = await Fixture.GetMappingsAsync( + scenario.Candidate.CurrentReleaseId, CancellationToken.None); + Assert.HasCount(1, beforeCurrent); + Assert.AreEqual(scenario.CanonicalPath, beforeCurrent[0].VirtualPath); + + var applied = await Fixture.ActivateUpgradeAsync(operation.Id, CancellationToken.None); + Assert.IsTrue(applied.IsSuccess); + Assert.AreEqual(ReleaseUpgradeStatus.Applied, applied.Operation!.Status); + Assert.IsEmpty(await Fixture.GetMappingsAsync( + scenario.Candidate.CurrentReleaseId, CancellationToken.None)); + var activeCandidate = await Fixture.GetMappingsAsync( + scenario.Candidate.CandidateReleaseId, CancellationToken.None); + Assert.HasCount(1, activeCandidate); + Assert.AreEqual(scenario.CanonicalPath, activeCandidate[0].VirtualPath); + + var rolledBack = await Fixture.RollbackUpgradeAsync(operation.Id, CancellationToken.None); + Assert.IsTrue(rolledBack.IsSuccess); + Assert.AreEqual(ReleaseUpgradeStatus.RolledBack, rolledBack.Operation!.Status); + var restored = await Fixture.GetMappingsAsync( + scenario.Candidate.CurrentReleaseId, CancellationToken.None); + Assert.HasCount(1, restored); + Assert.AreEqual(scenario.CanonicalPath, restored[0].VirtualPath); + Assert.IsEmpty(await Fixture.GetMappingsAsync( + scenario.Candidate.CandidateReleaseId, CancellationToken.None)); + } + private static FileMapping Mapping(Guid animationInfoId, string virtualPath) => new(Guid.NewGuid(), animationInfoId, virtualPath, "/physical/" + Guid.NewGuid(), "local"); + + private static LibrarySearchRequest AnySearch(Guid userId, string? cursor, int take) => + new(null, null, null, null, null, null, null, + LibraryDownloadState.Any, + LibraryWatchState.Any, + null, + LibrarySourceKind.Any, + LibrarySearchSort.PublishedDescending, + cursor, + take, + userId); } diff --git a/SecondDimensionWatcherReDive.Test/ReleaseUpgradeCoordinatorTests.cs b/SecondDimensionWatcherReDive.Test/ReleaseUpgradeCoordinatorTests.cs new file mode 100644 index 0000000..e048ecb --- /dev/null +++ b/SecondDimensionWatcherReDive.Test/ReleaseUpgradeCoordinatorTests.cs @@ -0,0 +1,122 @@ +using Microsoft.Extensions.Logging; +using Moq; +using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Framework.FileDownload; +using SecondDimensionWatcherReDive.Framework.FileStore; +using SecondDimensionWatcherReDive.Utils.Incidents; +using SecondDimensionWatcherReDive.Utils.ReleaseUpgrades; + +namespace SecondDimensionWatcherReDive.Test; + +[TestClass] +public sealed class ReleaseUpgradeCoordinatorTests +{ + [TestMethod] + public async Task CandidateValidationFailure_DoesNotInvokeAtomicSwap_AndRecordsFailure() + { + var fixture = new CoordinatorFixture(fileExists: false); + + var result = await fixture.Coordinator.TryActivateCandidateAsync( + fixture.Operation.CandidateReleaseId, + CancellationToken.None); + + Assert.IsNotNull(result); + Assert.IsFalse(result.IsSuccess); + fixture.UpgradeRepository.Verify(repository => repository.ActivateAsync( + It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny()), Times.Never); + fixture.UpgradeRepository.Verify(repository => repository.MarkFailedAsync( + fixture.Operation.Id, + It.Is(summary => summary.Contains("missing", StringComparison.OrdinalIgnoreCase)), + It.IsAny()), Times.Once); + fixture.IncidentReporter.Verify(reporter => reporter.ReportAsync( + It.Is(report => report.Title == "Release upgrade failed"), + It.IsAny()), Times.Once); + } + + [TestMethod] + public async Task ValidCandidate_InvokesAtomicSwapOnlyAfterEveryFilePasses() + { + var fixture = new CoordinatorFixture(fileExists: true); + + var result = await fixture.Coordinator.TryActivateCandidateAsync( + fixture.Operation.CandidateReleaseId, + CancellationToken.None); + + Assert.IsNotNull(result); + Assert.IsTrue(result.IsSuccess); + fixture.FileStore.Verify(store => store.ExistAsync( + "/store/new.mkv", It.IsAny()), Times.Once); + fixture.FileStore.Verify(store => store.FileInfoAsync( + "/store/new.mkv", It.IsAny()), Times.Once); + fixture.UpgradeRepository.Verify(repository => repository.ActivateAsync( + fixture.Operation.Id, + It.IsAny(), + It.Is(until => until > DateTimeOffset.UtcNow.AddHours(71)), + It.IsAny()), Times.Once); + } + + private sealed class CoordinatorFixture + { + public Mock UpgradeRepository { get; } = new(); + public Mock FileStore { get; } = new(); + public Mock IncidentReporter { get; } = new(); + public ReleaseUpgradeOperation Operation { get; } + public IReleaseUpgradeCoordinator Coordinator { get; } + + public CoordinatorFixture(bool fileExists) + { + Operation = new ReleaseUpgradeOperation( + Guid.NewGuid(), Guid.NewGuid(), Guid.NewGuid(), + ReleaseUpgradeStatus.Verifying, 200, 500, + DateTimeOffset.UtcNow, null, null, null, null, null); + var previous = new FileMapping( + Guid.NewGuid(), Operation.CurrentReleaseId, "/show/e01.mkv", "/store/old.mkv", "local"); + var candidate = new FileMapping( + Guid.NewGuid(), Operation.CandidateReleaseId, "/show/e01 (2).mkv", "/store/new.mkv", "local"); + UpgradeRepository.Setup(repository => repository.FindActiveByCandidateAsync( + Operation.CandidateReleaseId, It.IsAny())) + .ReturnsAsync(Operation); + UpgradeRepository.Setup(repository => repository.GetActivationAsync( + Operation.CandidateReleaseId, It.IsAny())) + .ReturnsAsync(new ReleaseUpgradeActivation(Operation, [previous], [candidate])); + UpgradeRepository.Setup(repository => repository.MarkFailedAsync( + Operation.Id, It.IsAny(), It.IsAny())) + .ReturnsAsync(new ReleaseUpgradeMutationResult(true, "failed", Operation)); + UpgradeRepository.Setup(repository => repository.ActivateAsync( + Operation.Id, It.IsAny(), It.IsAny(), + It.IsAny())) + .ReturnsAsync(new ReleaseUpgradeMutationResult(true, "applied", + Operation with { Status = ReleaseUpgradeStatus.Applied })); + + FileStore.Setup(store => store.ExistAsync( + candidate.PhysicalPath, It.IsAny())) + .ReturnsAsync(fileExists); + FileStore.Setup(store => store.FileInfoAsync( + candidate.PhysicalPath, It.IsAny())) + .ReturnsAsync(new FileStoreInfo(false, candidate.PhysicalPath, "new.mkv", 1024)); + var storeProvider = new Mock(); + storeProvider.Setup(provider => provider.GetRequiredClient("local")) + .Returns(FileStore.Object); + + var animationRepository = new Mock(); + animationRepository.Setup(repository => repository.FindByIdAsync( + Operation.CandidateReleaseId, It.IsAny())) + .ReturnsAsync(new AnimationInfo( + Operation.CandidateReleaseId, "candidate", "", DateTimeOffset.UtcNow, + "https://example.test/new", FileDownloadTypes.TorrentDownload, [], "", + true, default, default, true, "local", "/store/new", 1, 1, + null, null, true, 0)); + + Coordinator = new ReleaseUpgradeCoordinator( + UpgradeRepository.Object, + animationRepository.Object, + Mock.Of(), + Mock.Of(), + Mock.Of(), + storeProvider.Object, + IncidentReporter.Object, + Mock.Of>()); + } + } +} diff --git a/SecondDimensionWatcherReDive.Test/SubscriptionAutomationMatcherTests.cs b/SecondDimensionWatcherReDive.Test/SubscriptionAutomationMatcherTests.cs index ef29419..e318e6a 100644 --- a/SecondDimensionWatcherReDive.Test/SubscriptionAutomationMatcherTests.cs +++ b/SecondDimensionWatcherReDive.Test/SubscriptionAutomationMatcherTests.cs @@ -200,6 +200,30 @@ public void Evaluate_ExcludedKeywordOnlyAppliesToReleaseTitle() Assert.IsTrue(Explanation(result, "excludedKeywords").Passed); } + [TestMethod] + public void Score_CombinesQualityPreferencesAndReturnsExplainableReasons() + { + var scorer = new ReleaseScoringService(); + var metadata = new SubscriptionReleaseMetadata( + "LoliHouse", + "2160p", + "AV1", + ["ja", "zh-CN"], + 3L * 1024 * 1024 * 1024); + var policy = Policy( + subtitleGroups: ["Other", "LoliHouse"], + languages: ["ja"]); + + var result = scorer.Score(metadata, policy); + + Assert.AreEqual(575, result.Value); + CollectionAssert.Contains(result.Reasons.ToArray(), "resolution:2160p:+400"); + CollectionAssert.Contains(result.Reasons.ToArray(), "codec:AV1:+80"); + CollectionAssert.Contains(result.Reasons.ToArray(), "subtitleGroup:LoliHouse:+45"); + CollectionAssert.Contains(result.Reasons.ToArray(), "language:zh-CN:+5"); + CollectionAssert.Contains(result.Reasons.ToArray(), "size:3.00GiB:+25"); + } + private static SubscriptionAutomationExplanation Explanation( SubscriptionAutomationEvaluation result, string field) diff --git a/SecondDimensionWatcherReDive.Test/SyncFeedTests.cs b/SecondDimensionWatcherReDive.Test/SyncFeedTests.cs index 599c1eb..94c1087 100644 --- a/SecondDimensionWatcherReDive.Test/SyncFeedTests.cs +++ b/SecondDimensionWatcherReDive.Test/SyncFeedTests.cs @@ -29,9 +29,9 @@ public void Setup() _mockPolicyRepo = new Mock(); _mockDownloadProvider = new Mock(); _mockDownloadClient = new Mock(); - _mockRepo.Setup(repository => repository.AddAsync( + _mockRepo.Setup(repository => repository.TryAddReleaseAsync( It.IsAny(), It.IsAny())) - .Returns(Task.CompletedTask); + .ReturnsAsync(true); _mockRepo.Setup(repository => repository.UpdateAsync( It.IsAny(), It.IsAny())) .Returns(Task.CompletedTask); @@ -68,7 +68,7 @@ public void Setup() } [TestMethod] - public async Task ProcessSingle_ExistingTitle_SkipsAdd() + public async Task ProcessSingle_ExistingReleaseIdentity_SkipsDuplicate() { var request = new AnimationAddRequest( DateTimeOffset.UtcNow, @@ -78,23 +78,16 @@ public async Task ProcessSingle_ExistingTitle_SkipsAdd() FileDownloadTypes.HttpDownload, ""); - _mockRepo - .Setup(r => r.FindByTitleAsync("Existing Title", It.IsAny())) - .ReturnsAsync(new AnimationInfo( - Guid.NewGuid(), "Existing Title", "Description", - DateTimeOffset.UtcNow, "https://example.com/download", - FileDownloadTypes.HttpDownload, - Array.Empty(), "", - false, default, default, false, - null, null, null, null, null, null, - false, 0)); + _mockRepo.Setup(repository => repository.TryAddReleaseAsync( + It.IsAny(), It.IsAny())) + .ReturnsAsync(false); await (Task)_processSingleMethod.Invoke( _syncFeed, new object[] { request, CancellationToken.None })!; _mockRepo.Verify( - r => r.AddAsync(It.IsAny(), It.IsAny()), - Times.Never); + r => r.TryAddReleaseAsync(It.IsAny(), It.IsAny()), + Times.Once); } [TestMethod] @@ -117,7 +110,7 @@ public async Task ProcessSingle_NewTitle_AddsRecord() _syncFeed, new object[] { request, CancellationToken.None })!; _mockRepo.Verify( - r => r.AddAsync( + r => r.TryAddReleaseAsync( It.Is(info => info.Title == "New Title" && info.Description == "New Description" && @@ -142,7 +135,7 @@ public async Task ProcessSingle_PolicyDoesNotMatch_SkipsRecordAndDownload() await InvokeProcessSingleAsync(request); - _mockRepo.Verify(repository => repository.AddAsync( + _mockRepo.Verify(repository => repository.TryAddReleaseAsync( It.IsAny(), It.IsAny()), Times.Never); _mockDownloadClient.Verify(client => client.SubmitDownloadTaskAsync( It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), @@ -156,10 +149,10 @@ public async Task ProcessSingle_NotifyOnly_PersistsNotifiedOutcomeAndExplanation var request = PolicyRequest(feedId, "[Group] Anime [1080p HEVC][CHS]"); SetupNewPolicyRequest(request, Policy(feedId, SubscriptionAutomationMode.NotifyOnly)); AnimationInfo? added = null; - _mockRepo.Setup(repository => repository.AddAsync( + _mockRepo.Setup(repository => repository.TryAddReleaseAsync( It.IsAny(), It.IsAny())) .Callback((info, _) => added = info) - .Returns(Task.CompletedTask); + .ReturnsAsync(true); await InvokeProcessSingleAsync(request); @@ -182,7 +175,7 @@ public async Task ProcessSingle_ManualConfirm_PersistsPendingConfirmationWithout await InvokeProcessSingleAsync(request); - _mockRepo.Verify(repository => repository.AddAsync( + _mockRepo.Verify(repository => repository.TryAddReleaseAsync( It.Is(info => info.AutomationDisposition == SubscriptionAutomationDisposition.PendingConfirmation && !info.IsDownloadTracked), diff --git a/SecondDimensionWatcherReDive/Controllers/Converter.cs b/SecondDimensionWatcherReDive/Controllers/Converter.cs index 742d740..bcd3678 100644 --- a/SecondDimensionWatcherReDive/Controllers/Converter.cs +++ b/SecondDimensionWatcherReDive/Controllers/Converter.cs @@ -62,7 +62,10 @@ public static External.SubscriptionAutomationPolicy ToExternal( record.ExcludedKeywords, record.Mode.ToString(), record.CreatedAt, - record.UpdatedAt); + record.UpdatedAt, + record.EnableVersionUpgrade, + record.MinimumUpgradeScore, + record.UpgradeRollbackHours); public static External.SubscriptionAutomationSimulationResult ToExternal( this Framework.Feed.SubscriptionAutomationSimulationResult result) => @@ -82,6 +85,84 @@ public static External.SubscriptionAutomationSimulationResult ToExternal( explanation.Expected, explanation.Message)).ToList())).ToList()); + public static External.LibrarySearchItemResponse ToExternal(this LibrarySearchItem item) => + new(item.AnimationInfoId, + item.Title, + item.AnimationName, + item.AnimationOriginalName, + item.TmdbId, + item.Season, + item.Episode, + item.SubtitleGroup, + item.Resolution, + item.Codec, + item.Languages, + item.IsDownloadTracked, + item.IsDownloadFinished, + item.IsMediaLibraryImport, + item.IsWatched, + item.PlaybackPositionSeconds, + item.VirtualPaths, + item.ReleaseScore, + item.ScoreReasons, + item.PublishedAt); + + public static External.LibrarySearchResponse ToExternal(this LibrarySearchResult result) => + new(result.Items.Select(item => item.ToExternal()).ToList(), result.NextCursor); + + public static External.ReleaseUpgradeCandidateResponse ToExternal( + this ReleaseUpgradeCandidate candidate) => + new(candidate.CurrentReleaseId, + candidate.CandidateReleaseId, + candidate.AnimationName, + candidate.Season, + candidate.Episode, + candidate.CurrentScore, + candidate.CandidateScore, + candidate.ScoreReasons, + candidate.Automatic); + + public static External.LibraryIntegritySummaryResponse ToExternal( + this LibraryIntegritySummary summary) => + new(summary.TmdbId, + summary.AnimationName, + summary.Season, + summary.ExpectedEpisodeCount, + summary.MissingEpisodes, + summary.DuplicateEpisodes.Select(duplicate => new External.EpisodeDuplicateResponse( + duplicate.Episode, + duplicate.ReleaseIds)).ToList(), + summary.UnidentifiedReleaseCount, + summary.UpgradeCandidates.Select(candidate => candidate.ToExternal()).ToList()); + + public static External.ReleaseUpgradeOperationResponse ToExternal( + this ReleaseUpgradeOperation operation) => + new(operation.Id, + operation.CurrentReleaseId, + operation.CandidateReleaseId, + operation.Status.ToString(), + operation.CurrentScore, + operation.CandidateScore, + operation.CreatedAt, + operation.VerifiedAt, + operation.AppliedAt, + operation.RollbackUntil, + operation.CompletedAt, + operation.FailureSummary); + + public static External.ReleaseUpgradeMutationResponse ToExternal( + this ReleaseUpgradeMutationResult result) => + new(result.IsSuccess, result.Outcome, result.Operation?.ToExternal()); + + public static External.ReleaseUpgradeExecutionResponse ToExternal( + this Utils.ReleaseUpgrades.ReleaseUpgradeExecutionResult result) => + new(result.IsSuccess, + result.Outcome, + result.DryRun, + result.RequiresDownload, + result.Operation?.ToExternal(), + result.ValidationErrors); + public static External.WebDavTokenSummary ToExternal(this WebDavToken record) => new(record.Id, record.Username, record.Description, record.CreatedAt); diff --git a/SecondDimensionWatcherReDive/Controllers/External/AppJsonSerializerContext.cs b/SecondDimensionWatcherReDive/Controllers/External/AppJsonSerializerContext.cs index 0f20309..4e0efb5 100644 --- a/SecondDimensionWatcherReDive/Controllers/External/AppJsonSerializerContext.cs +++ b/SecondDimensionWatcherReDive/Controllers/External/AppJsonSerializerContext.cs @@ -65,4 +65,11 @@ namespace SecondDimensionWatcherReDive.Controllers.External; [JsonSerializable(typeof(QueueMediaLibraryScanResponse))] [JsonSerializable(typeof(ApplicationSettingsResponse))] [JsonSerializable(typeof(PatchApplicationSettingsRequest))] +[JsonSerializable(typeof(ExecuteReleaseUpgradeRequest))] +[JsonSerializable(typeof(LibrarySearchResponse))] +[JsonSerializable(typeof(List))] +[JsonSerializable(typeof(List))] +[JsonSerializable(typeof(List))] +[JsonSerializable(typeof(ReleaseUpgradeMutationResponse))] +[JsonSerializable(typeof(ReleaseUpgradeExecutionResponse))] internal partial class AppJsonSerializerContext : JsonSerializerContext; diff --git a/SecondDimensionWatcherReDive/Controllers/External/Library.cs b/SecondDimensionWatcherReDive/Controllers/External/Library.cs new file mode 100644 index 0000000..999e287 --- /dev/null +++ b/SecondDimensionWatcherReDive/Controllers/External/Library.cs @@ -0,0 +1,84 @@ +namespace SecondDimensionWatcherReDive.Controllers.External; + +internal sealed record ExecuteReleaseUpgradeRequest( + Guid CurrentReleaseId, + Guid CandidateReleaseId, + bool DryRun = true); + +internal sealed record LibrarySearchItemResponse( + Guid AnimationInfoId, + string Title, + string? AnimationName, + string? AnimationOriginalName, + string? TmdbId, + int? Season, + int? Episode, + string? SubtitleGroup, + string? Resolution, + string? Codec, + IReadOnlyList Languages, + bool IsDownloadTracked, + bool IsDownloadFinished, + bool IsMediaLibraryImport, + bool IsWatched, + double? PlaybackPositionSeconds, + IReadOnlyList VirtualPaths, + int ReleaseScore, + IReadOnlyList ScoreReasons, + DateTimeOffset PublishedAt); + +internal sealed record LibrarySearchResponse( + IReadOnlyList Items, + string? NextCursor); + +internal sealed record EpisodeDuplicateResponse( + int Episode, + IReadOnlyList ReleaseIds); + +internal sealed record ReleaseUpgradeCandidateResponse( + Guid CurrentReleaseId, + Guid CandidateReleaseId, + string AnimationName, + int Season, + int Episode, + int CurrentScore, + int CandidateScore, + IReadOnlyList ScoreReasons, + bool Automatic); + +internal sealed record LibraryIntegritySummaryResponse( + string TmdbId, + string AnimationName, + int Season, + int? ExpectedEpisodeCount, + IReadOnlyList MissingEpisodes, + IReadOnlyList DuplicateEpisodes, + int UnidentifiedReleaseCount, + IReadOnlyList UpgradeCandidates); + +internal sealed record ReleaseUpgradeOperationResponse( + Guid Id, + Guid CurrentReleaseId, + Guid CandidateReleaseId, + string Status, + int CurrentScore, + int CandidateScore, + DateTimeOffset CreatedAt, + DateTimeOffset? VerifiedAt, + DateTimeOffset? AppliedAt, + DateTimeOffset? RollbackUntil, + DateTimeOffset? CompletedAt, + string? FailureSummary); + +internal sealed record ReleaseUpgradeMutationResponse( + bool IsSuccess, + string Outcome, + ReleaseUpgradeOperationResponse? Operation); + +internal sealed record ReleaseUpgradeExecutionResponse( + bool IsSuccess, + string Outcome, + bool DryRun, + bool RequiresDownload, + ReleaseUpgradeOperationResponse? Operation, + IReadOnlyList ValidationErrors); diff --git a/SecondDimensionWatcherReDive/Controllers/External/SubscriptionAutomationPolicy.cs b/SecondDimensionWatcherReDive/Controllers/External/SubscriptionAutomationPolicy.cs index d27d33a..2bc1fa9 100644 --- a/SecondDimensionWatcherReDive/Controllers/External/SubscriptionAutomationPolicy.cs +++ b/SecondDimensionWatcherReDive/Controllers/External/SubscriptionAutomationPolicy.cs @@ -8,7 +8,10 @@ internal sealed record UpsertSubscriptionAutomationPolicyRequest( long? MinSizeBytes, long? MaxSizeBytes, IReadOnlyList? ExcludedKeywords, - string Mode); + string Mode, + bool EnableVersionUpgrade = false, + int? MinimumUpgradeScore = null, + int? UpgradeRollbackHours = null); internal sealed record SubscriptionAutomationPolicy( Guid FeedId, @@ -21,7 +24,10 @@ internal sealed record SubscriptionAutomationPolicy( IReadOnlyList ExcludedKeywords, string Mode, DateTimeOffset CreatedAt, - DateTimeOffset UpdatedAt); + DateTimeOffset UpdatedAt, + bool EnableVersionUpgrade, + int MinimumUpgradeScore, + int UpgradeRollbackHours); internal sealed record SubscriptionAutomationExplanation( string Field, diff --git a/SecondDimensionWatcherReDive/Controllers/LibraryController.cs b/SecondDimensionWatcherReDive/Controllers/LibraryController.cs new file mode 100644 index 0000000..d2dbfeb --- /dev/null +++ b/SecondDimensionWatcherReDive/Controllers/LibraryController.cs @@ -0,0 +1,153 @@ +using System.Security.Claims; +using Microsoft.AspNetCore.Authentication.JwtBearer; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Utils.ReleaseUpgrades; + +namespace SecondDimensionWatcherReDive.Controllers; + +[ApiController] +[Route("api/library")] +[Authorize(AuthenticationSchemes = JwtBearerDefaults.AuthenticationScheme)] +internal sealed class LibraryController( + ILibrarySearchRepository searchRepository, + IReleaseUpgradeRepository upgradeRepository, + IReleaseUpgradeCoordinator upgradeCoordinator) : ControllerBase +{ + [HttpGet("search")] + public async Task SearchAsync( + [FromQuery(Name = "q")] string? query, + [FromQuery] int? season, + [FromQuery] int? episode, + [FromQuery] string? subtitleGroup, + [FromQuery] string? resolution, + [FromQuery] string? codec, + [FromQuery] string? language, + [FromQuery] string? downloadState, + [FromQuery] string? watchState, + [FromQuery(Name = "path")] string? virtualPath, + [FromQuery] string? source, + [FromQuery] string? sort, + [FromQuery] string? cursor, + [FromQuery] int take = 30, + CancellationToken cancellationToken = default) + { + if (!TryGetUserId(out var userId)) return Unauthorized(); + if (take is < 1 or > 100 || season is < 0 or > 100 || episode is < 0 or > 100000) + return BadRequest(new { message = "Invalid pagination, season, or episode value." }); + if (!TryParse(downloadState, LibraryDownloadState.Any, out LibraryDownloadState parsedDownload) || + !TryParse(watchState, LibraryWatchState.Any, out LibraryWatchState parsedWatch) || + !TryParse(source, LibrarySourceKind.Any, out LibrarySourceKind parsedSource) || + !TryParse(sort, LibrarySearchSort.PublishedDescending, out LibrarySearchSort parsedSort)) + return BadRequest(new { message = "One or more search enum values are invalid." }); + + try + { + var result = await searchRepository.SearchAsync(new LibrarySearchRequest( + Normalize(query), + season, + episode, + Normalize(subtitleGroup), + Normalize(resolution), + Normalize(codec), + Normalize(language), + parsedDownload, + parsedWatch, + Normalize(virtualPath), + parsedSource, + parsedSort, + Normalize(cursor), + take, + userId), + cancellationToken); + return Ok(result.ToExternal()); + } + catch (ArgumentException exception) + { + return BadRequest(new { message = exception.Message }); + } + } + + [HttpGet("integrity")] + public async Task IntegrityAsync( + [FromQuery] string? tmdbId, + [FromQuery] int? season, + CancellationToken cancellationToken) + { + if (season is < 0 or > 100) return BadRequest(new { message = "Invalid season." }); + var result = await searchRepository.GetIntegrityAsync(Normalize(tmdbId), season, cancellationToken); + return Ok(result.Select(item => item.ToExternal()).ToList()); + } + + [HttpGet("upgrades")] + public async Task UpgradesAsync( + [FromQuery] bool automaticOnly = false, + [FromQuery] int take = 50, + CancellationToken cancellationToken = default) + { + if (take is < 1 or > 200) return BadRequest(new { message = "take must be between 1 and 200." }); + var result = await upgradeRepository.GetCandidatesAsync(automaticOnly, take, cancellationToken); + return Ok(result.Select(item => item.ToExternal()).ToList()); + } + + [HttpPost("upgrades/execute")] + public async Task ExecuteUpgradeAsync( + [FromBody] External.ExecuteReleaseUpgradeRequest request, + CancellationToken cancellationToken) + { + var candidate = (await upgradeRepository.GetCandidatesAsync(false, 200, cancellationToken)) + .SingleOrDefault(item => item.CurrentReleaseId == request.CurrentReleaseId && + item.CandidateReleaseId == request.CandidateReleaseId); + if (candidate is null) + return Conflict(new { message = "The requested release is no longer an available upgrade." }); + + var result = await upgradeCoordinator.ExecuteAsync(candidate, request.DryRun, cancellationToken); + var response = result.ToExternal(); + return result.IsSuccess ? Ok(response) : UnprocessableEntity(response); + } + + [HttpPost("upgrades/{operationId:guid}/rollback")] + public async Task RollbackUpgradeAsync( + [FromRoute] Guid operationId, + CancellationToken cancellationToken) + { + var result = await upgradeCoordinator.RollbackAsync(operationId, cancellationToken); + var response = result.ToExternal(); + if (result.Outcome == "not_found") return NotFound(response); + return result.IsSuccess ? Ok(response) : Conflict(response); + } + + [HttpGet("upgrade-history")] + public async Task UpgradeHistoryAsync( + [FromQuery] int take = 50, + CancellationToken cancellationToken = default) + { + if (take is < 1 or > 200) return BadRequest(new { message = "take must be between 1 and 200." }); + var result = await upgradeRepository.GetHistoryAsync(take, cancellationToken); + return Ok(result.Select(item => item.ToExternal()).ToList()); + } + + private bool TryGetUserId(out Guid userId) + { + var raw = User.FindFirstValue("Id") + ?? User.FindFirstValue(ClaimTypes.NameIdentifier) + ?? User.FindFirstValue("sub"); + return Guid.TryParse(raw, out userId); + } + + private static string? Normalize(string? value) => + string.IsNullOrWhiteSpace(value) ? null : value.Trim(); + + private static bool TryParse(string? value, T defaultValue, out T parsed) + where T : struct, Enum + { + if (string.IsNullOrWhiteSpace(value)) + { + parsed = defaultValue; + return true; + } + + return Enum.TryParse(value, ignoreCase: true, out parsed) && Enum.IsDefined(parsed); + } +} diff --git a/SecondDimensionWatcherReDive/Controllers/SubscriptionPoliciesController.cs b/SecondDimensionWatcherReDive/Controllers/SubscriptionPoliciesController.cs index cf1a204..53ffc87 100644 --- a/SecondDimensionWatcherReDive/Controllers/SubscriptionPoliciesController.cs +++ b/SecondDimensionWatcherReDive/Controllers/SubscriptionPoliciesController.cs @@ -128,6 +128,20 @@ private static bool TryCreatePolicy( return false; } + var minimumUpgradeScore = request.MinimumUpgradeScore ?? 25; + if (minimumUpgradeScore is < 1 or > 1000) + { + error = "minimumUpgradeScore must be between 1 and 1000."; + return false; + } + + var rollbackHours = request.UpgradeRollbackHours ?? 72; + if (rollbackHours is < 1 or > 720) + { + error = "upgradeRollbackHours must be between 1 and 720."; + return false; + } + policy = new SubscriptionAutomationPolicy( feedId, subtitleGroups, @@ -139,7 +153,10 @@ private static bool TryCreatePolicy( excludedKeywords, mode, timestamp, - timestamp); + timestamp, + request.EnableVersionUpgrade, + minimumUpgradeScore, + rollbackHours); error = null; return true; } diff --git a/SecondDimensionWatcherReDive/Migrations/20260829151303_AddLibrarySearchAndReleaseUpgrades.Designer.cs b/SecondDimensionWatcherReDive/Migrations/20260829151303_AddLibrarySearchAndReleaseUpgrades.Designer.cs new file mode 100644 index 0000000..c4f0658 --- /dev/null +++ b/SecondDimensionWatcherReDive/Migrations/20260829151303_AddLibrarySearchAndReleaseUpgrades.Designer.cs @@ -0,0 +1,1193 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using SecondDimensionWatcherReDive.Models; + +#nullable disable + +namespace SecondDimensionWatcherReDive.Migrations +{ + [DbContext(typeof(ApplicationContext))] + [Migration("20260829151303_AddLibrarySearchAndReleaseUpgrades")] + partial class AddLibrarySearchAndReleaseUpgrades + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.Animation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("OriginalName") + .IsRequired() + .HasColumnType("text"); + + b.Property("PosterPath") + .HasColumnType("text"); + + b.Property("TmdbId") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("TmdbId") + .IsUnique(); + + b.ToTable("Animations"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.AnimationGroup", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("AnimationGroups"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.AnimationInfo", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AdditionalDownloadInfo") + .IsRequired() + .HasColumnType("text"); + + b.Property("AiRetryCount") + .HasColumnType("integer"); + + b.Property("AnimationId") + .HasColumnType("uuid"); + + b.Property("AutomationDisposition") + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("AutomationExplanationJson") + .HasColumnType("text"); + + b.Property("CachedDownloadData") + .IsRequired() + .HasColumnType("bytea"); + + b.Property("CurrentMetadataReviewOperationId") + .HasColumnType("uuid"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text"); + + b.Property("DownloadAttemptId") + .HasColumnType("uuid"); + + b.Property("DownloadCancellationId") + .HasColumnType("uuid"); + + b.Property("DownloadEndTime") + .HasColumnType("timestamp with time zone"); + + b.Property("DownloadStartTime") + .HasColumnType("timestamp with time zone"); + + b.Property("DownloadType") + .IsRequired() + .HasColumnType("text"); + + b.Property("DownloadUrl") + .IsRequired() + .HasColumnType("text"); + + b.Property("EnclosureId") + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("Episode") + .HasColumnType("integer"); + + b.Property("ExpectedEpisodeCount") + .HasColumnType("integer"); + + b.Property("FeedItemGuid") + .HasMaxLength(1024) + .HasColumnType("character varying(1024)"); + + b.Property("FileStore") + .HasColumnType("text"); + + b.Property("GroupId") + .HasColumnType("uuid"); + + b.Property("IngestedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("IsActiveRelease") + .HasColumnType("boolean"); + + b.Property("IsAiProcessed") + .HasColumnType("boolean"); + + b.Property("IsDownloadFinished") + .HasColumnType("boolean"); + + b.Property("IsDownloadTracked") + .HasColumnType("boolean"); + + b.Property("MediaLibraryMissingSince") + .HasColumnType("timestamp with time zone"); + + b.Property("MediaLibrarySourceId") + .HasColumnType("uuid"); + + b.Property("MetadataConfidence") + .HasColumnType("double precision"); + + b.Property("MetadataLastError") + .HasMaxLength(1024) + .HasColumnType("character varying(1024)"); + + b.Property("MetadataReviewedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("MetadataStatus") + .HasColumnType("integer"); + + b.Property("PublishTime") + .HasColumnType("timestamp with time zone"); + + b.Property("ReleaseCodec") + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("ReleaseIdentity") + .HasMaxLength(192) + .HasColumnType("character varying(192)"); + + b.PrimitiveCollection("ReleaseLanguages") + .IsRequired() + .HasColumnType("text[]"); + + b.Property("ReleaseResolution") + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("ReleaseScore") + .HasColumnType("integer"); + + b.Property("ReleaseScoreReasonsJson") + .HasColumnType("text"); + + b.Property("ReleaseSizeBytes") + .HasColumnType("bigint"); + + b.Property("ReleaseSubtitleGroup") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("Season") + .HasColumnType("integer"); + + b.Property("SourceFeedId") + .HasColumnType("uuid"); + + b.Property("StateVersion") + .IsConcurrencyToken() + .HasColumnType("bigint"); + + b.Property("StorePath") + .HasColumnType("text"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text"); + + b.Property("TorrentInfoHash") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.HasKey("Id"); + + b.HasIndex("AnimationId"); + + b.HasIndex("CurrentMetadataReviewOperationId") + .IsUnique(); + + b.HasIndex("GroupId"); + + b.HasIndex("MediaLibrarySourceId"); + + b.HasIndex("ReleaseIdentity") + .IsUnique() + .HasDatabaseName("UX_AnimationInfo_ReleaseIdentity") + .HasFilter("\"ReleaseIdentity\" IS NOT NULL"); + + b.HasIndex("SourceFeedId"); + + b.HasIndex("FileStore", "StorePath") + .IsUnique() + .HasFilter("\"DownloadType\" = 'http://schemas.hcgstudio.com/ws/2023/06/sdw/downloadtype/media-library-import'"); + + b.HasIndex("MetadataStatus", "PublishTime"); + + b.HasIndex("Season", "Episode", "ReleaseScore"); + + b.ToTable("AnimationInfo", t => + { + t.HasCheckConstraint("CK_AnimationInfo_ExpectedEpisodeCount_Positive", "\"ExpectedEpisodeCount\" IS NULL OR \"ExpectedEpisodeCount\" > 0"); + + t.HasCheckConstraint("CK_AnimationInfo_MetadataConfidence_Range", "\"MetadataConfidence\" IS NULL OR (\"MetadataConfidence\" >= 0 AND \"MetadataConfidence\" <= 1)"); + + t.HasCheckConstraint("CK_AnimationInfo_ReleaseScore_NonNegative", "\"ReleaseScore\" >= 0"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ApplicationSettings", b => + { + b.Property("Id") + .HasColumnType("integer"); + + b.Property("ProtectedSecrets") + .HasColumnType("text"); + + b.Property("Revision") + .IsConcurrencyToken() + .HasColumnType("bigint"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("ValuesJson") + .IsRequired() + .HasColumnType("jsonb"); + + b.HasKey("Id"); + + b.ToTable("ApplicationSettings", t => + { + t.HasCheckConstraint("CK_ApplicationSettings_Revision_Positive", "\"Revision\" > 0"); + + t.HasCheckConstraint("CK_ApplicationSettings_Singleton", "\"Id\" = 1"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.BangumiSubgroup", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("MikanSubgroupId") + .HasColumnType("integer"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("ScrapedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("SeasonBangumiId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("SeasonBangumiId", "MikanSubgroupId") + .IsUnique(); + + b.ToTable("BangumiSubgroups"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ChatConversation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Title") + .HasColumnType("text"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.ToTable("ChatConversations"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ChatMessage", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Content") + .HasColumnType("text"); + + b.Property("ConversationId") + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Order") + .HasColumnType("integer"); + + b.Property("Role") + .IsRequired() + .HasColumnType("text"); + + b.Property("ToolCallId") + .HasColumnType("text"); + + b.Property("ToolCallsJson") + .HasColumnType("text"); + + b.Property("ToolName") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("ConversationId"); + + b.ToTable("ChatMessages"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.Feed", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Name") + .HasColumnType("text"); + + b.Property("Url") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("Feeds"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.FileMapping", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AnimationInfoId") + .HasColumnType("uuid"); + + b.Property("FileStore") + .IsRequired() + .HasColumnType("text"); + + b.Property("PhysicalPath") + .IsRequired() + .HasColumnType("text"); + + b.Property("VirtualPath") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("AnimationInfoId"); + + b.HasIndex("VirtualPath") + .IsUnique(); + + b.ToTable("FileMappings"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.FileNameRegexRule", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AnimationId") + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("Pattern") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("character varying(512)"); + + b.HasKey("Id"); + + b.HasIndex("AnimationId", "CreatedAt"); + + b.HasIndex("AnimationId", "Pattern") + .IsUnique(); + + b.ToTable("FileNameRegexRules"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.Incident", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Detail") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("DetectedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Fingerprint") + .IsRequired() + .HasMaxLength(96) + .HasColumnType("character varying(96)"); + + b.Property("LastRetryAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastRetryError") + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("ResolvedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("RetryCount") + .HasColumnType("integer"); + + b.Property("Severity") + .HasColumnType("integer"); + + b.Property("SourceId") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("Title") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("Type") + .HasColumnType("integer"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("Fingerprint") + .IsUnique(); + + b.HasIndex("ResolvedAt", "Type", "UpdatedAt"); + + b.ToTable("Incidents"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MediaLibrarySource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("IsMonitoring") + .HasColumnType("boolean"); + + b.Property("LastError") + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("LastImportedCount") + .HasColumnType("integer"); + + b.Property("LastRemovedCount") + .HasColumnType("integer"); + + b.Property("LastScanAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastSkippedCount") + .HasColumnType("integer"); + + b.Property("LastUpdatedCount") + .HasColumnType("integer"); + + b.Property("Path") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Path") + .IsUnique(); + + b.ToTable("MediaLibrarySources"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewMappingSnapshot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("FileStore") + .IsRequired() + .HasColumnType("text"); + + b.Property("Kind") + .HasColumnType("integer"); + + b.Property("OperationId") + .HasColumnType("uuid"); + + b.Property("PhysicalPath") + .IsRequired() + .HasColumnType("text"); + + b.Property("VirtualPath") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("OperationId", "Kind", "VirtualPath") + .IsUnique(); + + b.ToTable("MetadataReviewMappingSnapshots"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AnimationInfoId") + .HasColumnType("uuid"); + + b.Property("AppliedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("AppliedVersion") + .HasColumnType("bigint"); + + b.Property("BaseFileStore") + .HasColumnType("text"); + + b.Property("BaseIsDownloadFinished") + .HasColumnType("boolean"); + + b.Property("BaseStorePath") + .HasColumnType("text"); + + b.Property("BaseVersion") + .HasColumnType("bigint"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("ExpiresAt") + .HasColumnType("timestamp with time zone"); + + b.Property("PreviousAiRetryCount") + .HasColumnType("integer"); + + b.Property("PreviousAnimationId") + .HasColumnType("uuid"); + + b.Property("PreviousConfidence") + .HasColumnType("double precision"); + + b.Property("PreviousCurrentOperationId") + .HasColumnType("uuid"); + + b.Property("PreviousDescription") + .HasColumnType("text"); + + b.Property("PreviousEpisode") + .HasColumnType("integer"); + + b.Property("PreviousGroupId") + .HasColumnType("uuid"); + + b.Property("PreviousIsAiProcessed") + .HasColumnType("boolean"); + + b.Property("PreviousLastError") + .HasColumnType("text"); + + b.Property("PreviousMetadataStatus") + .HasColumnType("integer"); + + b.Property("PreviousReviewedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("PreviousSeason") + .HasColumnType("integer"); + + b.Property("ProposedAnimationName") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProposedAnimationOriginalName") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProposedAnimationPosterPath") + .HasColumnType("text"); + + b.Property("ProposedAnimationTmdbId") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProposedDescription") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProposedEpisode") + .HasColumnType("integer"); + + b.Property("ProposedGroupName") + .HasColumnType("text"); + + b.Property("ProposedSeason") + .HasColumnType("integer"); + + b.Property("State") + .HasColumnType("integer"); + + b.Property("UndoneAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("AnimationInfoId", "AppliedVersion") + .IsUnique(); + + b.HasIndex("AnimationInfoId", "State"); + + b.HasIndex("State", "ExpiresAt"); + + b.ToTable("MetadataReviewOperations", t => + { + t.HasCheckConstraint("CK_MetadataReviewOperations_Expiry", "\"ExpiresAt\" > \"CreatedAt\""); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MigrationMarker", b => + { + b.Property("Key") + .HasColumnType("text"); + + b.Property("AppliedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Key"); + + b.ToTable("MigrationMarkers"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.PlaybackPreference", b => + { + b.Property("UserId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AudioLanguage") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("AudioTrackLabel") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("AutoPlayNext") + .HasColumnType("boolean"); + + b.Property("SubtitleLanguage") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("SubtitleTrackLabel") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("UserId"); + + b.ToTable("PlaybackPreferences"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.PlaybackProgress", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AnimationInfoId") + .HasColumnType("uuid"); + + b.Property("DurationSeconds") + .HasColumnType("double precision"); + + b.Property("IsWatched") + .HasColumnType("boolean"); + + b.Property("PositionSeconds") + .HasColumnType("double precision"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.Property("VirtualPath") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("WatchedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("AnimationInfoId"); + + b.HasIndex("UserId", "AnimationInfoId", "VirtualPath") + .IsUnique(); + + b.HasIndex("UserId", "IsWatched", "UpdatedAt"); + + b.ToTable("PlaybackProgresses", t => + { + t.HasCheckConstraint("CK_PlaybackProgresses_Duration_NonNegative", "\"DurationSeconds\" >= 0"); + + t.HasCheckConstraint("CK_PlaybackProgresses_Position_NonNegative", "\"PositionSeconds\" >= 0"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ReleaseUpgradeMappingSnapshot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AnimationInfoId") + .HasColumnType("uuid"); + + b.Property("FileStore") + .IsRequired() + .HasColumnType("text"); + + b.Property("Kind") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("OperationId") + .HasColumnType("uuid"); + + b.Property("OriginalMappingId") + .HasColumnType("uuid"); + + b.Property("PhysicalPath") + .IsRequired() + .HasColumnType("text"); + + b.Property("VirtualPath") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.HasKey("Id"); + + b.HasIndex("OperationId", "Kind", "OriginalMappingId") + .IsUnique(); + + b.ToTable("ReleaseUpgradeMappingSnapshots"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ReleaseUpgradeOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AppliedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CandidateReleaseId") + .HasColumnType("uuid"); + + b.Property("CandidateScore") + .HasColumnType("integer"); + + b.Property("CompletedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CurrentReleaseId") + .HasColumnType("uuid"); + + b.Property("CurrentScore") + .HasColumnType("integer"); + + b.Property("FailureSummary") + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("RollbackUntil") + .HasColumnType("timestamp with time zone"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("VerifiedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("CandidateReleaseId") + .IsUnique(); + + b.HasIndex("CurrentReleaseId") + .IsUnique() + .HasDatabaseName("UX_ReleaseUpgradeOperations_ActiveCurrentRelease") + .HasFilter("\"Status\" IN ('Downloading', 'Verifying', 'Applied')"); + + b.HasIndex("Status", "CreatedAt"); + + b.ToTable("ReleaseUpgradeOperations", t => + { + t.HasCheckConstraint("CK_ReleaseUpgradeOperations_ScoreIncrease", "\"CandidateScore\" > \"CurrentScore\""); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SeasonBangumi", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("DayOfWeek") + .HasColumnType("integer"); + + b.Property("ImageUrl") + .HasColumnType("text"); + + b.Property("MikanId") + .HasColumnType("integer"); + + b.Property("ScrapedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("MikanId") + .IsUnique(); + + b.ToTable("SeasonBangumis"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SubscriptionAutomationPolicy", b => + { + b.Property("FeedId") + .HasColumnType("uuid"); + + b.PrimitiveCollection("Codecs") + .IsRequired() + .HasColumnType("text[]"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("EnableVersionUpgrade") + .HasColumnType("boolean"); + + b.PrimitiveCollection("ExcludedKeywords") + .IsRequired() + .HasColumnType("text[]"); + + b.PrimitiveCollection("Languages") + .IsRequired() + .HasColumnType("text[]"); + + b.Property("MaxSizeBytes") + .HasColumnType("bigint"); + + b.Property("MinSizeBytes") + .HasColumnType("bigint"); + + b.Property("MinimumUpgradeScore") + .HasColumnType("integer"); + + b.Property("Mode") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.PrimitiveCollection("Resolutions") + .IsRequired() + .HasColumnType("text[]"); + + b.PrimitiveCollection("SubtitleGroups") + .IsRequired() + .HasColumnType("text[]"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("UpgradeRollbackHours") + .HasColumnType("integer"); + + b.HasKey("FeedId"); + + b.HasIndex("UpdatedAt"); + + b.ToTable("SubscriptionAutomationPolicies", t => + { + t.HasCheckConstraint("CK_SubscriptionAutomationPolicies_MinimumUpgradeScore", "\"MinimumUpgradeScore\" >= 1 AND \"MinimumUpgradeScore\" <= 1000"); + + t.HasCheckConstraint("CK_SubscriptionAutomationPolicies_UpgradeRollbackHours", "\"UpgradeRollbackHours\" >= 1 AND \"UpgradeRollbackHours\" <= 720"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.WebDavToken", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("TokenHash") + .IsRequired() + .HasColumnType("text"); + + b.Property("Username") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Username") + .IsUnique(); + + b.ToTable("WebDavTokens"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.AnimationInfo", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.Animation", "Animation") + .WithMany() + .HasForeignKey("AnimationId"); + + b.HasOne("SecondDimensionWatcherReDive.Models.AnimationGroup", "Group") + .WithMany() + .HasForeignKey("GroupId"); + + b.HasOne("SecondDimensionWatcherReDive.Models.MediaLibrarySource", null) + .WithMany() + .HasForeignKey("MediaLibrarySourceId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("SecondDimensionWatcherReDive.Models.Feed", null) + .WithMany() + .HasForeignKey("SourceFeedId") + .OnDelete(DeleteBehavior.SetNull); + + b.Navigation("Animation"); + + b.Navigation("Group"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.BangumiSubgroup", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.SeasonBangumi", "SeasonBangumi") + .WithMany("Subgroups") + .HasForeignKey("SeasonBangumiId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("SeasonBangumi"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ChatMessage", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.ChatConversation", "Conversation") + .WithMany("Messages") + .HasForeignKey("ConversationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Conversation"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.FileNameRegexRule", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.Animation", null) + .WithMany() + .HasForeignKey("AnimationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewMappingSnapshot", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.MetadataReviewOperation", "Operation") + .WithMany("MappingSnapshots") + .HasForeignKey("OperationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Operation"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewOperation", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.AnimationInfo", "AnimationInfo") + .WithMany() + .HasForeignKey("AnimationInfoId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AnimationInfo"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.PlaybackProgress", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.AnimationInfo", "AnimationInfo") + .WithMany() + .HasForeignKey("AnimationInfoId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AnimationInfo"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ReleaseUpgradeMappingSnapshot", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.ReleaseUpgradeOperation", "Operation") + .WithMany("MappingSnapshots") + .HasForeignKey("OperationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Operation"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ReleaseUpgradeOperation", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.AnimationInfo", "CandidateRelease") + .WithMany() + .HasForeignKey("CandidateReleaseId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("SecondDimensionWatcherReDive.Models.AnimationInfo", "CurrentRelease") + .WithMany() + .HasForeignKey("CurrentReleaseId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("CandidateRelease"); + + b.Navigation("CurrentRelease"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SubscriptionAutomationPolicy", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.Feed", "Feed") + .WithOne() + .HasForeignKey("SecondDimensionWatcherReDive.Models.SubscriptionAutomationPolicy", "FeedId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Feed"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ChatConversation", b => + { + b.Navigation("Messages"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewOperation", b => + { + b.Navigation("MappingSnapshots"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ReleaseUpgradeOperation", b => + { + b.Navigation("MappingSnapshots"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SeasonBangumi", b => + { + b.Navigation("Subgroups"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/SecondDimensionWatcherReDive/Migrations/20260829151303_AddLibrarySearchAndReleaseUpgrades.cs b/SecondDimensionWatcherReDive/Migrations/20260829151303_AddLibrarySearchAndReleaseUpgrades.cs new file mode 100644 index 0000000..b4ab9af --- /dev/null +++ b/SecondDimensionWatcherReDive/Migrations/20260829151303_AddLibrarySearchAndReleaseUpgrades.cs @@ -0,0 +1,401 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace SecondDimensionWatcherReDive.Migrations +{ + /// + public partial class AddLibrarySearchAndReleaseUpgrades : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "EnableVersionUpgrade", + table: "SubscriptionAutomationPolicies", + type: "boolean", + nullable: false, + defaultValue: false); + + migrationBuilder.AddColumn( + name: "MinimumUpgradeScore", + table: "SubscriptionAutomationPolicies", + type: "integer", + nullable: false, + defaultValue: 25); + + migrationBuilder.AddColumn( + name: "UpgradeRollbackHours", + table: "SubscriptionAutomationPolicies", + type: "integer", + nullable: false, + defaultValue: 72); + + migrationBuilder.AddColumn( + name: "EnclosureId", + table: "AnimationInfo", + type: "character varying(2048)", + maxLength: 2048, + nullable: true); + + migrationBuilder.AddColumn( + name: "ExpectedEpisodeCount", + table: "AnimationInfo", + type: "integer", + nullable: true); + + migrationBuilder.AddColumn( + name: "FeedItemGuid", + table: "AnimationInfo", + type: "character varying(1024)", + maxLength: 1024, + nullable: true); + + migrationBuilder.AddColumn( + name: "IngestedAt", + table: "AnimationInfo", + type: "timestamp with time zone", + nullable: false, + defaultValueSql: "CURRENT_TIMESTAMP"); + + migrationBuilder.AddColumn( + name: "IsActiveRelease", + table: "AnimationInfo", + type: "boolean", + nullable: false, + defaultValue: false); + + migrationBuilder.Sql( + """ + UPDATE "AnimationInfo" + SET "IsActiveRelease" = TRUE; + + WITH ranked_releases AS ( + SELECT info."Id", + row_number() OVER ( + PARTITION BY info."AnimationId", info."Season", info."Episode" + ORDER BY (info."IsDownloadFinished" AND EXISTS ( + SELECT 1 + FROM "FileMappings" mapping + WHERE mapping."AnimationInfoId" = info."Id" + )) DESC, + info."PublishTime" DESC, + info."Id" + ) AS rank + FROM "AnimationInfo" info + WHERE info."AnimationId" IS NOT NULL + AND info."Season" IS NOT NULL + AND info."Episode" IS NOT NULL + ) + UPDATE "AnimationInfo" info + SET "IsActiveRelease" = ranked.rank = 1 + FROM ranked_releases ranked + WHERE info."Id" = ranked."Id"; + """); + + migrationBuilder.AddColumn( + name: "ReleaseCodec", + table: "AnimationInfo", + type: "character varying(32)", + maxLength: 32, + nullable: true); + + migrationBuilder.AddColumn( + name: "ReleaseIdentity", + table: "AnimationInfo", + type: "character varying(192)", + maxLength: 192, + nullable: true); + + migrationBuilder.Sql( + """ + UPDATE "AnimationInfo" + SET "ReleaseIdentity" = 'legacy:' || replace(lower("Id"::text), '-', '') + WHERE "ReleaseIdentity" IS NULL; + """); + + migrationBuilder.Sql("CREATE EXTENSION IF NOT EXISTS pg_trgm;"); + + migrationBuilder.AddColumn( + name: "ReleaseLanguages", + table: "AnimationInfo", + type: "text[]", + nullable: false, + defaultValue: new string[0]); + + migrationBuilder.AddColumn( + name: "ReleaseResolution", + table: "AnimationInfo", + type: "character varying(32)", + maxLength: 32, + nullable: true); + + migrationBuilder.AddColumn( + name: "ReleaseScore", + table: "AnimationInfo", + type: "integer", + nullable: false, + defaultValue: 0); + + migrationBuilder.AddColumn( + name: "ReleaseScoreReasonsJson", + table: "AnimationInfo", + type: "text", + nullable: true); + + migrationBuilder.AddColumn( + name: "ReleaseSubtitleGroup", + table: "AnimationInfo", + type: "character varying(256)", + maxLength: 256, + nullable: true); + + migrationBuilder.AddColumn( + name: "TorrentInfoHash", + table: "AnimationInfo", + type: "character varying(64)", + maxLength: 64, + nullable: true); + + migrationBuilder.CreateTable( + name: "ReleaseUpgradeOperations", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + CurrentReleaseId = table.Column(type: "uuid", nullable: false), + CandidateReleaseId = table.Column(type: "uuid", nullable: false), + Status = table.Column(type: "character varying(32)", maxLength: 32, nullable: false), + CurrentScore = table.Column(type: "integer", nullable: false), + CandidateScore = table.Column(type: "integer", nullable: false), + CreatedAt = table.Column(type: "timestamp with time zone", nullable: false), + VerifiedAt = table.Column(type: "timestamp with time zone", nullable: true), + AppliedAt = table.Column(type: "timestamp with time zone", nullable: true), + RollbackUntil = table.Column(type: "timestamp with time zone", nullable: true), + CompletedAt = table.Column(type: "timestamp with time zone", nullable: true), + FailureSummary = table.Column(type: "character varying(2048)", maxLength: 2048, nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_ReleaseUpgradeOperations", x => x.Id); + table.CheckConstraint("CK_ReleaseUpgradeOperations_ScoreIncrease", "\"CandidateScore\" > \"CurrentScore\""); + table.ForeignKey( + name: "FK_ReleaseUpgradeOperations_AnimationInfo_CandidateReleaseId", + column: x => x.CandidateReleaseId, + principalTable: "AnimationInfo", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + table.ForeignKey( + name: "FK_ReleaseUpgradeOperations_AnimationInfo_CurrentReleaseId", + column: x => x.CurrentReleaseId, + principalTable: "AnimationInfo", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + }); + + migrationBuilder.CreateTable( + name: "ReleaseUpgradeMappingSnapshots", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + OperationId = table.Column(type: "uuid", nullable: false), + Kind = table.Column(type: "character varying(32)", maxLength: 32, nullable: false), + OriginalMappingId = table.Column(type: "uuid", nullable: false), + AnimationInfoId = table.Column(type: "uuid", nullable: false), + VirtualPath = table.Column(type: "character varying(2048)", maxLength: 2048, nullable: false), + PhysicalPath = table.Column(type: "text", nullable: false), + FileStore = table.Column(type: "text", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_ReleaseUpgradeMappingSnapshots", x => x.Id); + table.ForeignKey( + name: "FK_ReleaseUpgradeMappingSnapshots_ReleaseUpgradeOperations_Ope~", + column: x => x.OperationId, + principalTable: "ReleaseUpgradeOperations", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.AddCheckConstraint( + name: "CK_SubscriptionAutomationPolicies_MinimumUpgradeScore", + table: "SubscriptionAutomationPolicies", + sql: "\"MinimumUpgradeScore\" >= 1 AND \"MinimumUpgradeScore\" <= 1000"); + + migrationBuilder.AddCheckConstraint( + name: "CK_SubscriptionAutomationPolicies_UpgradeRollbackHours", + table: "SubscriptionAutomationPolicies", + sql: "\"UpgradeRollbackHours\" >= 1 AND \"UpgradeRollbackHours\" <= 720"); + + migrationBuilder.CreateIndex( + name: "IX_AnimationInfo_Season_Episode_ReleaseScore", + table: "AnimationInfo", + columns: new[] { "Season", "Episode", "ReleaseScore" }); + + migrationBuilder.CreateIndex( + name: "UX_AnimationInfo_ReleaseIdentity", + table: "AnimationInfo", + column: "ReleaseIdentity", + unique: true, + filter: "\"ReleaseIdentity\" IS NOT NULL"); + + migrationBuilder.AddCheckConstraint( + name: "CK_AnimationInfo_ExpectedEpisodeCount_Positive", + table: "AnimationInfo", + sql: "\"ExpectedEpisodeCount\" IS NULL OR \"ExpectedEpisodeCount\" > 0"); + + migrationBuilder.AddCheckConstraint( + name: "CK_AnimationInfo_ReleaseScore_NonNegative", + table: "AnimationInfo", + sql: "\"ReleaseScore\" >= 0"); + + migrationBuilder.CreateIndex( + name: "IX_ReleaseUpgradeMappingSnapshots_OperationId_Kind_OriginalMap~", + table: "ReleaseUpgradeMappingSnapshots", + columns: new[] { "OperationId", "Kind", "OriginalMappingId" }, + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_ReleaseUpgradeOperations_CandidateReleaseId", + table: "ReleaseUpgradeOperations", + column: "CandidateReleaseId", + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_ReleaseUpgradeOperations_Status_CreatedAt", + table: "ReleaseUpgradeOperations", + columns: new[] { "Status", "CreatedAt" }); + + migrationBuilder.CreateIndex( + name: "UX_ReleaseUpgradeOperations_ActiveCurrentRelease", + table: "ReleaseUpgradeOperations", + column: "CurrentReleaseId", + unique: true, + filter: "\"Status\" IN ('Downloading', 'Verifying', 'Applied')"); + + migrationBuilder.Sql( + """ + CREATE INDEX "IX_AnimationInfo_Title_Trgm" + ON "AnimationInfo" USING GIN ("Title" gin_trgm_ops); + CREATE INDEX "IX_Animations_Name_Trgm" + ON "Animations" USING GIN ("Name" gin_trgm_ops); + CREATE INDEX "IX_Animations_OriginalName_Trgm" + ON "Animations" USING GIN ("OriginalName" gin_trgm_ops); + CREATE INDEX "IX_AnimationGroups_Name_Trgm" + ON "AnimationGroups" USING GIN ("Name" gin_trgm_ops); + CREATE INDEX "IX_FileMappings_VirtualPath_Trgm" + ON "FileMappings" USING GIN ("VirtualPath" gin_trgm_ops); + CREATE INDEX "IX_AnimationInfo_ReleaseLanguages_Gin" + ON "AnimationInfo" USING GIN ("ReleaseLanguages"); + """); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql( + """ + DROP INDEX IF EXISTS "IX_AnimationInfo_Title_Trgm"; + DROP INDEX IF EXISTS "IX_Animations_Name_Trgm"; + DROP INDEX IF EXISTS "IX_Animations_OriginalName_Trgm"; + DROP INDEX IF EXISTS "IX_AnimationGroups_Name_Trgm"; + DROP INDEX IF EXISTS "IX_FileMappings_VirtualPath_Trgm"; + DROP INDEX IF EXISTS "IX_AnimationInfo_ReleaseLanguages_Gin"; + """); + + migrationBuilder.DropTable( + name: "ReleaseUpgradeMappingSnapshots"); + + migrationBuilder.DropTable( + name: "ReleaseUpgradeOperations"); + + migrationBuilder.DropCheckConstraint( + name: "CK_SubscriptionAutomationPolicies_MinimumUpgradeScore", + table: "SubscriptionAutomationPolicies"); + + migrationBuilder.DropCheckConstraint( + name: "CK_SubscriptionAutomationPolicies_UpgradeRollbackHours", + table: "SubscriptionAutomationPolicies"); + + migrationBuilder.DropIndex( + name: "IX_AnimationInfo_Season_Episode_ReleaseScore", + table: "AnimationInfo"); + + migrationBuilder.DropIndex( + name: "UX_AnimationInfo_ReleaseIdentity", + table: "AnimationInfo"); + + migrationBuilder.DropCheckConstraint( + name: "CK_AnimationInfo_ExpectedEpisodeCount_Positive", + table: "AnimationInfo"); + + migrationBuilder.DropCheckConstraint( + name: "CK_AnimationInfo_ReleaseScore_NonNegative", + table: "AnimationInfo"); + + migrationBuilder.DropColumn( + name: "EnableVersionUpgrade", + table: "SubscriptionAutomationPolicies"); + + migrationBuilder.DropColumn( + name: "MinimumUpgradeScore", + table: "SubscriptionAutomationPolicies"); + + migrationBuilder.DropColumn( + name: "UpgradeRollbackHours", + table: "SubscriptionAutomationPolicies"); + + migrationBuilder.DropColumn( + name: "EnclosureId", + table: "AnimationInfo"); + + migrationBuilder.DropColumn( + name: "ExpectedEpisodeCount", + table: "AnimationInfo"); + + migrationBuilder.DropColumn( + name: "FeedItemGuid", + table: "AnimationInfo"); + + migrationBuilder.DropColumn( + name: "IngestedAt", + table: "AnimationInfo"); + + migrationBuilder.DropColumn( + name: "IsActiveRelease", + table: "AnimationInfo"); + + migrationBuilder.DropColumn( + name: "ReleaseCodec", + table: "AnimationInfo"); + + migrationBuilder.DropColumn( + name: "ReleaseIdentity", + table: "AnimationInfo"); + + migrationBuilder.DropColumn( + name: "ReleaseLanguages", + table: "AnimationInfo"); + + migrationBuilder.DropColumn( + name: "ReleaseResolution", + table: "AnimationInfo"); + + migrationBuilder.DropColumn( + name: "ReleaseScore", + table: "AnimationInfo"); + + migrationBuilder.DropColumn( + name: "ReleaseScoreReasonsJson", + table: "AnimationInfo"); + + migrationBuilder.DropColumn( + name: "ReleaseSubtitleGroup", + table: "AnimationInfo"); + + migrationBuilder.DropColumn( + name: "TorrentInfoHash", + table: "AnimationInfo"); + } + } +} diff --git a/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs b/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs index 8126b9e..78612ac 100644 --- a/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs +++ b/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs @@ -123,15 +123,34 @@ protected override void BuildModel(ModelBuilder modelBuilder) .IsRequired() .HasColumnType("text"); + b.Property("EnclosureId") + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + b.Property("Episode") .HasColumnType("integer"); + b.Property("ExpectedEpisodeCount") + .HasColumnType("integer"); + + b.Property("FeedItemGuid") + .HasMaxLength(1024) + .HasColumnType("character varying(1024)"); + b.Property("FileStore") .HasColumnType("text"); b.Property("GroupId") .HasColumnType("uuid"); + b.Property("IngestedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("IsActiveRelease") + .HasColumnType("boolean"); + b.Property("IsAiProcessed") .HasColumnType("boolean"); @@ -163,9 +182,35 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("PublishTime") .HasColumnType("timestamp with time zone"); + b.Property("ReleaseCodec") + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("ReleaseIdentity") + .HasMaxLength(192) + .HasColumnType("character varying(192)"); + + b.PrimitiveCollection("ReleaseLanguages") + .IsRequired() + .HasColumnType("text[]"); + + b.Property("ReleaseResolution") + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("ReleaseScore") + .HasColumnType("integer"); + + b.Property("ReleaseScoreReasonsJson") + .HasColumnType("text"); + b.Property("ReleaseSizeBytes") .HasColumnType("bigint"); + b.Property("ReleaseSubtitleGroup") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + b.Property("Season") .HasColumnType("integer"); @@ -183,6 +228,10 @@ protected override void BuildModel(ModelBuilder modelBuilder) .IsRequired() .HasColumnType("text"); + b.Property("TorrentInfoHash") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + b.HasKey("Id"); b.HasIndex("AnimationId"); @@ -194,6 +243,11 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasIndex("MediaLibrarySourceId"); + b.HasIndex("ReleaseIdentity") + .IsUnique() + .HasDatabaseName("UX_AnimationInfo_ReleaseIdentity") + .HasFilter("\"ReleaseIdentity\" IS NOT NULL"); + b.HasIndex("SourceFeedId"); b.HasIndex("FileStore", "StorePath") @@ -202,9 +256,15 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasIndex("MetadataStatus", "PublishTime"); + b.HasIndex("Season", "Episode", "ReleaseScore"); + b.ToTable("AnimationInfo", t => { + t.HasCheckConstraint("CK_AnimationInfo_ExpectedEpisodeCount_Positive", "\"ExpectedEpisodeCount\" IS NULL OR \"ExpectedEpisodeCount\" > 0"); + t.HasCheckConstraint("CK_AnimationInfo_MetadataConfidence_Range", "\"MetadataConfidence\" IS NULL OR (\"MetadataConfidence\" >= 0 AND \"MetadataConfidence\" <= 1)"); + + t.HasCheckConstraint("CK_AnimationInfo_ReleaseScore_NonNegative", "\"ReleaseScore\" >= 0"); }); }); @@ -753,6 +813,107 @@ protected override void BuildModel(ModelBuilder modelBuilder) }); }); + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ReleaseUpgradeMappingSnapshot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AnimationInfoId") + .HasColumnType("uuid"); + + b.Property("FileStore") + .IsRequired() + .HasColumnType("text"); + + b.Property("Kind") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("OperationId") + .HasColumnType("uuid"); + + b.Property("OriginalMappingId") + .HasColumnType("uuid"); + + b.Property("PhysicalPath") + .IsRequired() + .HasColumnType("text"); + + b.Property("VirtualPath") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.HasKey("Id"); + + b.HasIndex("OperationId", "Kind", "OriginalMappingId") + .IsUnique(); + + b.ToTable("ReleaseUpgradeMappingSnapshots"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ReleaseUpgradeOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AppliedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CandidateReleaseId") + .HasColumnType("uuid"); + + b.Property("CandidateScore") + .HasColumnType("integer"); + + b.Property("CompletedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CurrentReleaseId") + .HasColumnType("uuid"); + + b.Property("CurrentScore") + .HasColumnType("integer"); + + b.Property("FailureSummary") + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("RollbackUntil") + .HasColumnType("timestamp with time zone"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("VerifiedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("CandidateReleaseId") + .IsUnique(); + + b.HasIndex("CurrentReleaseId") + .IsUnique() + .HasDatabaseName("UX_ReleaseUpgradeOperations_ActiveCurrentRelease") + .HasFilter("\"Status\" IN ('Downloading', 'Verifying', 'Applied')"); + + b.HasIndex("Status", "CreatedAt"); + + b.ToTable("ReleaseUpgradeOperations", t => + { + t.HasCheckConstraint("CK_ReleaseUpgradeOperations_ScoreIncrease", "\"CandidateScore\" > \"CurrentScore\""); + }); + }); + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SeasonBangumi", b => { b.Property("Id") @@ -795,6 +956,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("CreatedAt") .HasColumnType("timestamp with time zone"); + b.Property("EnableVersionUpgrade") + .HasColumnType("boolean"); + b.PrimitiveCollection("ExcludedKeywords") .IsRequired() .HasColumnType("text[]"); @@ -809,6 +973,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("MinSizeBytes") .HasColumnType("bigint"); + b.Property("MinimumUpgradeScore") + .HasColumnType("integer"); + b.Property("Mode") .IsRequired() .HasMaxLength(32) @@ -825,11 +992,19 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("UpdatedAt") .HasColumnType("timestamp with time zone"); + b.Property("UpgradeRollbackHours") + .HasColumnType("integer"); + b.HasKey("FeedId"); b.HasIndex("UpdatedAt"); - b.ToTable("SubscriptionAutomationPolicies"); + b.ToTable("SubscriptionAutomationPolicies", t => + { + t.HasCheckConstraint("CK_SubscriptionAutomationPolicies_MinimumUpgradeScore", "\"MinimumUpgradeScore\" >= 1 AND \"MinimumUpgradeScore\" <= 1000"); + + t.HasCheckConstraint("CK_SubscriptionAutomationPolicies_UpgradeRollbackHours", "\"UpgradeRollbackHours\" >= 1 AND \"UpgradeRollbackHours\" <= 720"); + }); }); modelBuilder.Entity("SecondDimensionWatcherReDive.Models.WebDavToken", b => @@ -949,6 +1124,36 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Navigation("AnimationInfo"); }); + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ReleaseUpgradeMappingSnapshot", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.ReleaseUpgradeOperation", "Operation") + .WithMany("MappingSnapshots") + .HasForeignKey("OperationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Operation"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ReleaseUpgradeOperation", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.AnimationInfo", "CandidateRelease") + .WithMany() + .HasForeignKey("CandidateReleaseId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("SecondDimensionWatcherReDive.Models.AnimationInfo", "CurrentRelease") + .WithMany() + .HasForeignKey("CurrentReleaseId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("CandidateRelease"); + + b.Navigation("CurrentRelease"); + }); + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SubscriptionAutomationPolicy", b => { b.HasOne("SecondDimensionWatcherReDive.Models.Feed", "Feed") @@ -970,6 +1175,11 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Navigation("MappingSnapshots"); }); + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ReleaseUpgradeOperation", b => + { + b.Navigation("MappingSnapshots"); + }); + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SeasonBangumi", b => { b.Navigation("Subgroups"); diff --git a/SecondDimensionWatcherReDive/Models/AnimationInfo.cs b/SecondDimensionWatcherReDive/Models/AnimationInfo.cs index 976702e..f929279 100644 --- a/SecondDimensionWatcherReDive/Models/AnimationInfo.cs +++ b/SecondDimensionWatcherReDive/Models/AnimationInfo.cs @@ -5,6 +5,7 @@ namespace SecondDimensionWatcherReDive.Models; public class AnimationInfo { public Guid Id { get; set; } + public DateTimeOffset IngestedAt { get; set; } = DateTimeOffset.UtcNow; public string Title { get; set; } = string.Empty; public string Description { get; set; } = string.Empty; @@ -69,4 +70,28 @@ public class AnimationInfo public Guid? MediaLibrarySourceId { get; set; } public DateTimeOffset? MediaLibraryMissingSince { get; set; } + + public string? ReleaseIdentity { get; set; } + + public string? FeedItemGuid { get; set; } + + public string? EnclosureId { get; set; } + + public string? TorrentInfoHash { get; set; } + + public string? ReleaseSubtitleGroup { get; set; } + + public string? ReleaseResolution { get; set; } + + public string? ReleaseCodec { get; set; } + + public string[] ReleaseLanguages { get; set; } = []; + + public int ReleaseScore { get; set; } + + public string? ReleaseScoreReasonsJson { get; set; } + + public int? ExpectedEpisodeCount { get; set; } + + public bool IsActiveRelease { get; set; } = true; } diff --git a/SecondDimensionWatcherReDive/Models/ApplicationContext.cs b/SecondDimensionWatcherReDive/Models/ApplicationContext.cs index 59764ac..dab05a9 100644 --- a/SecondDimensionWatcherReDive/Models/ApplicationContext.cs +++ b/SecondDimensionWatcherReDive/Models/ApplicationContext.cs @@ -33,6 +33,8 @@ public ApplicationContext(DbContextOptions options) public DbSet PlaybackPreferences { get; set; } public DbSet MediaLibrarySources { get; set; } public DbSet ApplicationSettings { get; set; } + public DbSet ReleaseUpgradeOperations { get; set; } + public DbSet ReleaseUpgradeMappingSnapshots { get; set; } protected override void OnModelCreating(ModelBuilder modelBuilder) { @@ -71,6 +73,10 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) .Property(info => info.StateVersion) .IsConcurrencyToken(); + modelBuilder.Entity() + .Property(info => info.IngestedAt) + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + modelBuilder.Entity() .Property(info => info.MetadataLastError) .HasMaxLength(1024); @@ -83,6 +89,54 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) modelBuilder.Entity() .HasIndex(info => new { info.MetadataStatus, info.PublishTime }); + modelBuilder.Entity() + .HasIndex(info => info.ReleaseIdentity) + .IsUnique() + .HasFilter("\"ReleaseIdentity\" IS NOT NULL") + .HasDatabaseName("UX_AnimationInfo_ReleaseIdentity"); + + modelBuilder.Entity() + .HasIndex(info => new { info.Season, info.Episode, info.ReleaseScore }); + + modelBuilder.Entity() + .Property(info => info.ReleaseIdentity) + .HasMaxLength(192); + + modelBuilder.Entity() + .Property(info => info.FeedItemGuid) + .HasMaxLength(1024); + + modelBuilder.Entity() + .Property(info => info.EnclosureId) + .HasMaxLength(2048); + + modelBuilder.Entity() + .Property(info => info.TorrentInfoHash) + .HasMaxLength(64); + + modelBuilder.Entity() + .Property(info => info.ReleaseSubtitleGroup) + .HasMaxLength(256); + + modelBuilder.Entity() + .Property(info => info.ReleaseResolution) + .HasMaxLength(32); + + modelBuilder.Entity() + .Property(info => info.ReleaseCodec) + .HasMaxLength(32); + + modelBuilder.Entity() + .ToTable(table => + { + table.HasCheckConstraint( + "CK_AnimationInfo_ReleaseScore_NonNegative", + "\"ReleaseScore\" >= 0"); + table.HasCheckConstraint( + "CK_AnimationInfo_ExpectedEpisodeCount_Positive", + "\"ExpectedEpisodeCount\" IS NULL OR \"ExpectedEpisodeCount\" > 0"); + }); + modelBuilder.Entity() .HasIndex(info => info.CurrentMetadataReviewOperationId) .IsUnique(); @@ -282,6 +336,75 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) modelBuilder.Entity() .HasIndex(policy => policy.UpdatedAt); + modelBuilder.Entity() + .ToTable(table => + { + table.HasCheckConstraint( + "CK_SubscriptionAutomationPolicies_MinimumUpgradeScore", + "\"MinimumUpgradeScore\" >= 1 AND \"MinimumUpgradeScore\" <= 1000"); + table.HasCheckConstraint( + "CK_SubscriptionAutomationPolicies_UpgradeRollbackHours", + "\"UpgradeRollbackHours\" >= 1 AND \"UpgradeRollbackHours\" <= 720"); + }); + + modelBuilder.Entity() + .Property(operation => operation.Status) + .HasConversion() + .HasMaxLength(32); + + modelBuilder.Entity() + .Property(operation => operation.FailureSummary) + .HasMaxLength(2048); + + modelBuilder.Entity() + .HasOne(operation => operation.CurrentRelease) + .WithMany() + .HasForeignKey(operation => operation.CurrentReleaseId) + .OnDelete(DeleteBehavior.Restrict); + + modelBuilder.Entity() + .HasOne(operation => operation.CandidateRelease) + .WithMany() + .HasForeignKey(operation => operation.CandidateReleaseId) + .OnDelete(DeleteBehavior.Restrict); + + modelBuilder.Entity() + .HasIndex(operation => operation.CandidateReleaseId) + .IsUnique(); + + modelBuilder.Entity() + .HasIndex(operation => operation.CurrentReleaseId) + .IsUnique() + .HasFilter("\"Status\" IN ('Downloading', 'Verifying', 'Applied')") + .HasDatabaseName("UX_ReleaseUpgradeOperations_ActiveCurrentRelease"); + + modelBuilder.Entity() + .HasIndex(operation => new { operation.Status, operation.CreatedAt }); + + modelBuilder.Entity() + .ToTable(table => table.HasCheckConstraint( + "CK_ReleaseUpgradeOperations_ScoreIncrease", + "\"CandidateScore\" > \"CurrentScore\"")); + + modelBuilder.Entity() + .Property(snapshot => snapshot.Kind) + .HasConversion() + .HasMaxLength(32); + + modelBuilder.Entity() + .Property(snapshot => snapshot.VirtualPath) + .HasMaxLength(2048); + + modelBuilder.Entity() + .HasOne(snapshot => snapshot.Operation) + .WithMany(operation => operation.MappingSnapshots) + .HasForeignKey(snapshot => snapshot.OperationId) + .OnDelete(DeleteBehavior.Cascade); + + modelBuilder.Entity() + .HasIndex(snapshot => new { snapshot.OperationId, snapshot.Kind, snapshot.OriginalMappingId }) + .IsUnique(); + modelBuilder.Entity() .HasIndex(s => new { s.SeasonBangumiId, s.MikanSubgroupId }) .IsUnique(); diff --git a/SecondDimensionWatcherReDive/Models/ReleaseUpgradeOperation.cs b/SecondDimensionWatcherReDive/Models/ReleaseUpgradeOperation.cs new file mode 100644 index 0000000..df6fd62 --- /dev/null +++ b/SecondDimensionWatcherReDive/Models/ReleaseUpgradeOperation.cs @@ -0,0 +1,35 @@ +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Models; + +public class ReleaseUpgradeOperation +{ + public Guid Id { get; set; } + public Guid CurrentReleaseId { get; set; } + public AnimationInfo CurrentRelease { get; set; } = null!; + public Guid CandidateReleaseId { get; set; } + public AnimationInfo CandidateRelease { get; set; } = null!; + public ReleaseUpgradeStatus Status { get; set; } + public int CurrentScore { get; set; } + public int CandidateScore { get; set; } + public DateTimeOffset CreatedAt { get; set; } + public DateTimeOffset? VerifiedAt { get; set; } + public DateTimeOffset? AppliedAt { get; set; } + public DateTimeOffset? RollbackUntil { get; set; } + public DateTimeOffset? CompletedAt { get; set; } + public string? FailureSummary { get; set; } + public ICollection MappingSnapshots { get; set; } = []; +} + +public class ReleaseUpgradeMappingSnapshot +{ + public Guid Id { get; set; } + public Guid OperationId { get; set; } + public ReleaseUpgradeOperation Operation { get; set; } = null!; + public ReleaseUpgradeMappingKind Kind { get; set; } + public Guid OriginalMappingId { get; set; } + public Guid AnimationInfoId { get; set; } + public string VirtualPath { get; set; } = string.Empty; + public string PhysicalPath { get; set; } = string.Empty; + public string FileStore { get; set; } = string.Empty; +} diff --git a/SecondDimensionWatcherReDive/Models/SubscriptionAutomationPolicy.cs b/SecondDimensionWatcherReDive/Models/SubscriptionAutomationPolicy.cs index aec6c49..472d4bf 100644 --- a/SecondDimensionWatcherReDive/Models/SubscriptionAutomationPolicy.cs +++ b/SecondDimensionWatcherReDive/Models/SubscriptionAutomationPolicy.cs @@ -27,4 +27,10 @@ public class SubscriptionAutomationPolicy public DateTimeOffset CreatedAt { get; set; } public DateTimeOffset UpdatedAt { get; set; } + + public bool EnableVersionUpgrade { get; set; } + + public int MinimumUpgradeScore { get; set; } = 25; + + public int UpgradeRollbackHours { get; set; } = 72; } diff --git a/SecondDimensionWatcherReDive/Program.cs b/SecondDimensionWatcherReDive/Program.cs index 80f5f19..77870e7 100644 --- a/SecondDimensionWatcherReDive/Program.cs +++ b/SecondDimensionWatcherReDive/Program.cs @@ -33,6 +33,7 @@ using SecondDimensionWatcherReDive.Utils.FileStore; using SecondDimensionWatcherReDive.Utils.MetadataReview; using SecondDimensionWatcherReDive.Utils.Incidents; +using SecondDimensionWatcherReDive.Utils.ReleaseUpgrades; using SecondDimensionWatcherReDive.Utils.Scraper; var builder = WebApplication.CreateBuilder(args); @@ -226,6 +227,7 @@ builder.Services.AddSingleton(sp => sp.GetRequiredService()); builder.Services.AddHostedService(); +builder.Services.AddHostedService(); //Add scheduled tasks builder.Services.AddSingleton(); @@ -257,6 +259,7 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddScoped(); builder.Services.AddTransient(); @@ -277,9 +280,12 @@ builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); builder.Services.AddSingleton(); builder.Services.AddScoped(); builder.Services.AddScoped(); +builder.Services.AddScoped(); //Add AI Inference // Register all engines even when initially unconfigured. Runtime settings can then enable or diff --git a/SecondDimensionWatcherReDive/Repositories/AnimationInfoRepository.cs b/SecondDimensionWatcherReDive/Repositories/AnimationInfoRepository.cs index d90f6b6..2bfa7e0 100644 --- a/SecondDimensionWatcherReDive/Repositories/AnimationInfoRepository.cs +++ b/SecondDimensionWatcherReDive/Repositories/AnimationInfoRepository.cs @@ -1,5 +1,6 @@ using System.Runtime.CompilerServices; using Microsoft.EntityFrameworkCore; +using Npgsql; using SecondDimensionWatcherReDive.Framework.DataRepository; using SecondDimensionWatcherReDive.Framework.FileDownload; @@ -322,6 +323,31 @@ public async Task AddAsync(AnimationInfo info, CancellationToken cancellationTok await context.SaveChangesAsync(cancellationToken); } + public async Task TryAddReleaseAsync( + AnimationInfo info, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(info.ReleaseIdentity)) + throw new ArgumentException("A stable release identity is required.", nameof(info)); + + var entity = info.ToEntity(); + await context.AnimationInfo.AddAsync(entity, cancellationToken); + try + { + await context.SaveChangesAsync(cancellationToken); + return true; + } + catch (DbUpdateException exception) when (exception.InnerException is PostgresException + { + SqlState: PostgresErrorCodes.UniqueViolation, + ConstraintName: "UX_AnimationInfo_ReleaseIdentity" + }) + { + context.Entry(entity).State = EntityState.Detached; + return false; + } + } + public async Task UpdateAsync(AnimationInfo info, CancellationToken cancellationToken) { var entity = await context.AnimationInfo.FindAsync([info.Id], cancellationToken) diff --git a/SecondDimensionWatcherReDive/Repositories/FileMappingRepositoryPostgreSqlTestFixture.cs b/SecondDimensionWatcherReDive/Repositories/FileMappingRepositoryPostgreSqlTestFixture.cs index 34b17f5..dd21bf3 100644 --- a/SecondDimensionWatcherReDive/Repositories/FileMappingRepositoryPostgreSqlTestFixture.cs +++ b/SecondDimensionWatcherReDive/Repositories/FileMappingRepositoryPostgreSqlTestFixture.cs @@ -1,5 +1,6 @@ using Microsoft.EntityFrameworkCore; using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Framework.FileDownload; namespace SecondDimensionWatcherReDive.Repositories; @@ -24,7 +25,7 @@ public async Task ResetAsync(CancellationToken cancellationToken) { await using var context = new Models.ApplicationContext(_contextOptions); await context.Database.ExecuteSqlRawAsync( - "TRUNCATE TABLE \"FileMappings\", \"AnimationInfo\" RESTART IDENTITY CASCADE", + "TRUNCATE TABLE \"FileMappings\", \"AnimationInfo\", \"Animations\", \"AnimationGroups\" RESTART IDENTITY CASCADE", cancellationToken); } @@ -83,4 +84,281 @@ public async Task GetAnimationInfoStateVersionsAsync(CancellationToken c .Select(info => info.StateVersion) .ToArrayAsync(cancellationToken); } + + public async Task TryAddReleaseAsync( + string releaseIdentity, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + var repository = new AnimationInfoRepository(context, _contextOptions); + return await repository.TryAddReleaseAsync(new AnimationInfo( + Guid.NewGuid(), + "concurrent release", + string.Empty, + DateTimeOffset.UtcNow, + "https://example.test/" + Guid.NewGuid().ToString("N"), + FileDownloadTypes.HttpDownload, + [], + string.Empty, + false, + default, + default, + false, + null, + null, + null, + null, + null, + null, + false, + 0, + ReleaseIdentity: releaseIdentity), + cancellationToken); + } + + public async Task CountReleaseIdentityAsync( + string releaseIdentity, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + return await context.AnimationInfo.CountAsync( + info => info.ReleaseIdentity == releaseIdentity, + cancellationToken); + } + + public async Task SeedLibraryScenarioAsync(CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + var now = DateTimeOffset.UtcNow; + var animation = new Models.Animation + { + Id = Guid.NewGuid(), + TmdbId = "tv-1399", + Name = "進撃の巨人", + OriginalName = "Attack on Titan" + }; + var group = new Models.AnimationGroup { Id = Guid.NewGuid(), Name = "LoliHouse" }; + var current = Release(animation, group, 1, 1, 320, now.AddMinutes(-10), + FileDownloadTypes.TorrentDownload, true, "torrent:current", 3); + current.ReleaseResolution = "1080p"; + current.ReleaseCodec = "HEVC"; + current.ReleaseLanguages = ["zh-CN"]; + current.ReleaseScoreReasonsJson = "[\"resolution:1080p:+200\",\"codec:HEVC:+60\"]"; + var duplicate = Release(animation, group, 1, 1, 250, now.AddMinutes(-9), + FileDownloadTypes.TorrentDownload, true, "torrent:duplicate", 3); + duplicate.IsActiveRelease = false; + var imported = Release(animation, group, 1, 2, 520, now.AddMinutes(-8), + FileDownloadTypes.MediaLibraryImport, true, "import:episode-2", 3); + imported.ReleaseResolution = "2160p"; + imported.ReleaseCodec = "AV1"; + imported.ReleaseLanguages = ["ja"]; + var upgrade = Release(animation, group, 1, 1, 480, now.AddMinutes(-7), + FileDownloadTypes.TorrentDownload, false, "torrent:upgrade", 3); + upgrade.IsActiveRelease = false; + upgrade.ReleaseResolution = "2160p"; + upgrade.ReleaseCodec = "AV1"; + upgrade.ReleaseScoreReasonsJson = "[\"resolution:2160p:+400\",\"codec:AV1:+80\"]"; + var unidentified = Release(animation, group, 1, null, 100, now.AddMinutes(-6), + FileDownloadTypes.TorrentDownload, false, "torrent:unknown", 3); + context.AnimationInfo.AddRange(current, duplicate, imported, upgrade, unidentified); + context.FileMappings.AddRange( + MappingEntity(current.Id, "/進撃の巨人/LoliHouse/進撃の巨人 S01E01.mkv"), + MappingEntity(duplicate.Id, "/進撃の巨人/LoliHouse/進撃の巨人 S01E01 (2).mkv"), + MappingEntity(imported.Id, "/進撃の巨人/Imported/進撃の巨人 S01E02.mkv")); + var userId = Guid.NewGuid(); + context.PlaybackProgresses.Add(new Models.PlaybackProgress + { + Id = Guid.NewGuid(), + UserId = userId, + AnimationInfoId = imported.Id, + VirtualPath = "/進撃の巨人/Imported/進撃の巨人 S01E02.mkv", + PositionSeconds = 120, + DurationSeconds = 1200, + IsWatched = false, + UpdatedAt = now + }); + await context.SaveChangesAsync(cancellationToken); + return new LibraryScenario(userId, current.Id, imported.Id, upgrade.Id); + } + + public async Task InsertConcurrentSearchReleaseAsync(CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + var info = new Models.AnimationInfo + { + Id = Guid.NewGuid(), + Title = "new concurrent release", + Description = string.Empty, + PublishTime = DateTimeOffset.UtcNow.AddHours(1), + IngestedAt = DateTimeOffset.UtcNow, + DownloadUrl = "https://example.test/new", + DownloadType = FileDownloadTypes.TorrentDownload, + ReleaseIdentity = "torrent:inserted-" + Guid.NewGuid().ToString("N") + }; + context.AnimationInfo.Add(info); + await context.SaveChangesAsync(cancellationToken); + return info.Id; + } + + public async Task SearchAsync( + LibrarySearchRequest request, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + return await new LibrarySearchRepository(context).SearchAsync(request, cancellationToken); + } + + public async Task> GetIntegrityAsync( + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + return await new LibrarySearchRepository(context) + .GetIntegrityAsync("tv-1399", 1, cancellationToken); + } + + public async Task> GetLibraryIndexNamesAsync(CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + return await context.Database.SqlQueryRaw( + """ + SELECT indexname AS "Value" + FROM pg_indexes + WHERE schemaname = 'public' + AND (indexname LIKE 'IX_%_Trgm' + OR indexname = 'IX_AnimationInfo_ReleaseLanguages_Gin' + OR indexname = 'UX_AnimationInfo_ReleaseIdentity') + ORDER BY indexname + """) + .ToListAsync(cancellationToken); + } + + public async Task SeedUpgradeScenarioAsync(CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + var animation = new Models.Animation + { + Id = Guid.NewGuid(), TmdbId = "upgrade-show", Name = "Upgrade Show", OriginalName = "Upgrade Show" + }; + var current = Release(animation, null, 1, 1, 200, DateTimeOffset.UtcNow.AddMinutes(-2), + FileDownloadTypes.TorrentDownload, true, "torrent:old-" + Guid.NewGuid().ToString("N"), 1); + var candidate = Release(animation, null, 1, 1, 500, DateTimeOffset.UtcNow.AddMinutes(-1), + FileDownloadTypes.TorrentDownload, true, "torrent:new-" + Guid.NewGuid().ToString("N"), 1); + candidate.IsActiveRelease = false; + context.AnimationInfo.AddRange(current, candidate); + context.FileMappings.AddRange( + new Models.FileMapping + { + Id = Guid.NewGuid(), AnimationInfoId = current.Id, + VirtualPath = "/Upgrade Show/Old/Upgrade Show S01E01.mkv", + PhysicalPath = "/store/old.mkv", FileStore = "local" + }, + new Models.FileMapping + { + Id = Guid.NewGuid(), AnimationInfoId = candidate.Id, + VirtualPath = "/Upgrade Show/New/Upgrade Show S01E01 (2).mkv", + PhysicalPath = "/store/new.mkv", FileStore = "local" + }); + await context.SaveChangesAsync(cancellationToken); + return new UpgradeScenario(new ReleaseUpgradeCandidate( + current.Id, candidate.Id, animation.Name, 1, 1, 200, 500, + ["resolution:2160p:+400"], false), + "/Upgrade Show/Old/Upgrade Show S01E01.mkv"); + } + + public async Task BeginUpgradeAsync( + ReleaseUpgradeCandidate candidate, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + return await new ReleaseUpgradeRepository(context, _contextOptions) + .TryBeginAsync(candidate, DateTimeOffset.UtcNow, cancellationToken); + } + + public async Task ActivateUpgradeAsync( + Guid operationId, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + return await new ReleaseUpgradeRepository(context, _contextOptions) + .ActivateAsync(operationId, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow.AddHours(24), cancellationToken); + } + + public async Task> GetReadyUpgradeCandidateIdsAsync( + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + return await new ReleaseUpgradeRepository(context, _contextOptions) + .GetReadyCandidateIdsAsync(20, cancellationToken); + } + + public async Task RollbackUpgradeAsync( + Guid operationId, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + return await new ReleaseUpgradeRepository(context, _contextOptions) + .RollbackAsync(operationId, DateTimeOffset.UtcNow, cancellationToken); + } + + public async Task> GetMappingsAsync( + Guid animationInfoId, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + return await new FileMappingRepository(context, _contextOptions) + .GetForAnimationInfoAsync(animationInfoId, cancellationToken); + } + + private static Models.AnimationInfo Release( + Models.Animation animation, + Models.AnimationGroup? group, + int season, + int? episode, + int score, + DateTimeOffset ingestedAt, + string downloadType, + bool downloaded, + string identity, + int expected) => new() + { + Id = Guid.NewGuid(), + Animation = animation, + Group = group, + Title = $"{animation.Name} S{season:D2}E{episode:D2}", + Description = animation.OriginalName, + PublishTime = ingestedAt, + IngestedAt = ingestedAt, + DownloadUrl = "https://example.test/" + Guid.NewGuid().ToString("N"), + DownloadType = downloadType, + IsDownloadTracked = downloaded, + IsDownloadFinished = downloaded, + FileStore = downloaded ? "local" : null, + StorePath = downloaded ? "/store/" + Guid.NewGuid().ToString("N") : null, + Season = season, + Episode = episode, + ReleaseIdentity = identity, + ReleaseSubtitleGroup = group?.Name, + ReleaseScore = score, + ExpectedEpisodeCount = expected, + IsAiProcessed = true + }; + + private static Models.FileMapping MappingEntity(Guid animationInfoId, string path) => new() + { + Id = Guid.NewGuid(), + AnimationInfoId = animationInfoId, + VirtualPath = path, + PhysicalPath = "/store/" + Guid.NewGuid().ToString("N") + ".mkv", + FileStore = "local" + }; } + +internal sealed record LibraryScenario( + Guid UserId, + Guid CurrentReleaseId, + Guid ImportedReleaseId, + Guid UpgradeReleaseId); + +internal sealed record UpgradeScenario( + ReleaseUpgradeCandidate Candidate, + string CanonicalPath); diff --git a/SecondDimensionWatcherReDive/Repositories/LibrarySearchRepository.cs b/SecondDimensionWatcherReDive/Repositories/LibrarySearchRepository.cs new file mode 100644 index 0000000..d54de7e --- /dev/null +++ b/SecondDimensionWatcherReDive/Repositories/LibrarySearchRepository.cs @@ -0,0 +1,322 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using Microsoft.EntityFrameworkCore; +using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Framework.FileDownload; + +namespace SecondDimensionWatcherReDive.Repositories; + +public sealed class LibrarySearchRepository(Models.ApplicationContext context) + : ILibrarySearchRepository +{ + private sealed record SearchCursor(int Offset, DateTimeOffset SnapshotUtc, string Signature); + + public async Task SearchAsync( + LibrarySearchRequest request, + CancellationToken cancellationToken) + { + if (request.Take is < 1 or > 100) + throw new ArgumentOutOfRangeException(nameof(request), "Take must be between 1 and 100."); + + var signature = Signature(request); + var cursor = DecodeCursor(request.Cursor); + if (cursor is not null && !string.Equals(cursor.Signature, signature, StringComparison.Ordinal)) + throw new ArgumentException("The search cursor does not match the active filters.", nameof(request)); + var snapshot = cursor?.SnapshotUtc ?? DateTimeOffset.UtcNow; + var offset = cursor?.Offset ?? 0; + if (offset is < 0 or > 1_000_000) + throw new ArgumentException("The search cursor is outside the supported range.", nameof(request)); + + var query = context.AnimationInfo + .AsNoTracking() + .Include(info => info.Animation) + .Include(info => info.Group) + .Where(info => info.MediaLibraryMissingSince == null && info.IngestedAt <= snapshot); + + if (!string.IsNullOrWhiteSpace(request.Query)) + { + var pattern = ContainsPattern(request.Query); + query = query.Where(info => + EF.Functions.ILike(info.Title, pattern, "\\") || + EF.Functions.ILike(info.Description, pattern, "\\") || + (info.Animation != null && + (EF.Functions.ILike(info.Animation.Name, pattern, "\\") || + EF.Functions.ILike(info.Animation.OriginalName, pattern, "\\") || + EF.Functions.ILike(info.Animation.TmdbId, pattern, "\\"))) || + (info.Group != null && EF.Functions.ILike(info.Group.Name, pattern, "\\")) || + context.FileMappings.Any(mapping => + mapping.AnimationInfoId == info.Id && + EF.Functions.ILike(mapping.VirtualPath, pattern, "\\"))); + } + + if (request.Season is { } season) query = query.Where(info => info.Season == season); + if (request.Episode is { } episode) query = query.Where(info => info.Episode == episode); + if (!string.IsNullOrWhiteSpace(request.SubtitleGroup)) + query = query.Where(info => info.ReleaseSubtitleGroup == request.SubtitleGroup); + if (!string.IsNullOrWhiteSpace(request.Resolution)) + query = query.Where(info => info.ReleaseResolution == request.Resolution); + if (!string.IsNullOrWhiteSpace(request.Codec)) + query = query.Where(info => info.ReleaseCodec == request.Codec); + if (!string.IsNullOrWhiteSpace(request.Language)) + query = query.Where(info => info.ReleaseLanguages.Contains(request.Language)); + if (!string.IsNullOrWhiteSpace(request.VirtualPath)) + { + var pathPattern = ContainsPattern(request.VirtualPath); + query = query.Where(info => context.FileMappings.Any(mapping => + mapping.AnimationInfoId == info.Id && + EF.Functions.ILike(mapping.VirtualPath, pathPattern, "\\"))); + } + + query = request.DownloadState switch + { + LibraryDownloadState.NotDownloaded => query.Where(info => !info.IsDownloadTracked), + LibraryDownloadState.Downloading => query.Where(info => + info.IsDownloadTracked && !info.IsDownloadFinished), + LibraryDownloadState.Downloaded => query.Where(info => info.IsDownloadFinished), + _ => query + }; + query = request.Source switch + { + LibrarySourceKind.Torrent => query.Where(info => + info.DownloadType == FileDownloadTypes.TorrentDownload), + LibrarySourceKind.MediaLibraryImport => query.Where(info => + info.DownloadType == FileDownloadTypes.MediaLibraryImport), + _ => query + }; + query = request.WatchState switch + { + LibraryWatchState.Watched => query.Where(info => context.PlaybackProgresses.Any(progress => + progress.UserId == request.UserId && progress.AnimationInfoId == info.Id && progress.IsWatched)), + LibraryWatchState.InProgress => query.Where(info => context.PlaybackProgresses.Any(progress => + progress.UserId == request.UserId && progress.AnimationInfoId == info.Id && + !progress.IsWatched && progress.PositionSeconds > 0)), + LibraryWatchState.Unwatched => query.Where(info => !context.PlaybackProgresses.Any(progress => + progress.UserId == request.UserId && progress.AnimationInfoId == info.Id && + (progress.IsWatched || progress.PositionSeconds > 0))), + _ => query + }; + + var ordered = request.Sort switch + { + LibrarySearchSort.TitleAscending => query + .OrderBy(info => info.Animation == null ? info.Title : info.Animation.Name) + .ThenBy(info => info.Season) + .ThenBy(info => info.Episode) + .ThenBy(info => info.Id), + LibrarySearchSort.EpisodeAscending => query + .OrderBy(info => info.Animation == null ? info.Title : info.Animation.Name) + .ThenBy(info => info.Season) + .ThenBy(info => info.Episode) + .ThenBy(info => info.Id), + LibrarySearchSort.ScoreDescending => query + .OrderByDescending(info => info.ReleaseScore) + .ThenByDescending(info => info.PublishTime) + .ThenBy(info => info.Id), + _ => query + .OrderByDescending(info => info.PublishTime) + .ThenBy(info => info.Id) + }; + + var page = await ordered.Skip(offset).Take(request.Take + 1).ToListAsync(cancellationToken); + var hasMore = page.Count > request.Take; + if (hasMore) page.RemoveAt(page.Count - 1); + var ids = page.Select(info => info.Id).ToArray(); + var mappings = await context.FileMappings.AsNoTracking() + .Where(mapping => ids.Contains(mapping.AnimationInfoId)) + .OrderBy(mapping => mapping.VirtualPath) + .ToListAsync(cancellationToken); + var mappingsByRelease = mappings + .GroupBy(mapping => mapping.AnimationInfoId) + .ToDictionary(group => group.Key, group => group.Select(item => item.VirtualPath).ToList()); + var progressByRelease = (await context.PlaybackProgresses.AsNoTracking() + .Where(progress => progress.UserId == request.UserId && ids.Contains(progress.AnimationInfoId)) + .OrderByDescending(progress => progress.UpdatedAt) + .ToListAsync(cancellationToken)) + .GroupBy(progress => progress.AnimationInfoId) + .ToDictionary(group => group.Key, group => group.ToList()); + + var items = page.Select(info => + { + var progress = progressByRelease.GetValueOrDefault(info.Id) ?? []; + return new LibrarySearchItem( + info.Id, + info.Title, + info.Animation?.Name, + info.Animation?.OriginalName, + info.Animation?.TmdbId, + info.Season, + info.Episode, + info.ReleaseSubtitleGroup ?? info.Group?.Name, + info.ReleaseResolution, + info.ReleaseCodec, + info.ReleaseLanguages, + info.IsDownloadTracked, + info.IsDownloadFinished, + info.DownloadType == FileDownloadTypes.MediaLibraryImport, + progress.Any(item => item.IsWatched), + progress.Count == 0 ? null : progress.Max(item => item.PositionSeconds), + mappingsByRelease.GetValueOrDefault(info.Id) ?? [], + info.ReleaseScore, + ParseReasons(info.ReleaseScoreReasonsJson), + info.PublishTime); + }).ToList(); + + var nextCursor = hasMore + ? EncodeCursor(new SearchCursor(offset + request.Take, snapshot, signature)) + : null; + return new LibrarySearchResult(items, nextCursor); + } + + public async Task> GetIntegrityAsync( + string? tmdbId, + int? season, + CancellationToken cancellationToken) + { + var query = context.AnimationInfo.AsNoTracking() + .Include(info => info.Animation) + .Where(info => info.Animation != null && info.MediaLibraryMissingSince == null); + if (!string.IsNullOrWhiteSpace(tmdbId)) + query = query.Where(info => info.Animation!.TmdbId == tmdbId); + if (season is not null) query = query.Where(info => info.Season == season); + + var releases = await query.ToListAsync(cancellationToken); + var releaseIds = releases.Select(info => info.Id).ToArray(); + var mappedIds = await context.FileMappings.AsNoTracking() + .Where(mapping => releaseIds.Contains(mapping.AnimationInfoId)) + .Select(mapping => mapping.AnimationInfoId) + .Distinct() + .ToHashSetAsync(cancellationToken); + var policies = await context.SubscriptionAutomationPolicies.AsNoTracking() + .ToDictionaryAsync(policy => policy.FeedId, cancellationToken); + + return releases + .Where(info => info.Season is > 0) + .GroupBy(info => new + { + info.Animation!.TmdbId, + info.Animation.Name, + Season = info.Season!.Value + }) + .Select(group => BuildIntegrity(group, mappedIds, policies)) + .OrderBy(item => item.AnimationName) + .ThenBy(item => item.Season) + .ToList(); + } + + private static LibraryIntegritySummary BuildIntegrity( + IEnumerable source, + IReadOnlySet mappedIds, + IReadOnlyDictionary policies) + { + var releases = source.ToList(); + var first = releases[0]; + var downloaded = releases + .Where(info => info.IsDownloadFinished && mappedIds.Contains(info.Id) && info.Episode is > 0) + .ToList(); + var expected = releases.Max(info => info.ExpectedEpisodeCount); + var present = downloaded.Select(info => info.Episode!.Value).ToHashSet(); + var missing = expected is { } count + ? Enumerable.Range(1, count).Where(episode => !present.Contains(episode)).ToList() + : []; + var duplicates = downloaded + .GroupBy(info => info.Episode!.Value) + .Where(group => group.Count() > 1) + .Select(group => new EpisodeDuplicate( + group.Key, + group.OrderByDescending(item => item.ReleaseScore).Select(item => item.Id).ToList())) + .OrderBy(item => item.Episode) + .ToList(); + var candidates = new List(); + foreach (var episodeGroup in releases.Where(info => info.Episode is > 0).GroupBy(info => info.Episode!.Value)) + { + var current = episodeGroup + .Where(info => info.IsActiveRelease && info.IsDownloadFinished && mappedIds.Contains(info.Id)) + .OrderByDescending(info => info.ReleaseScore) + .ThenByDescending(info => info.PublishTime) + .FirstOrDefault(); + if (current is null) continue; + var candidate = episodeGroup + .Where(info => info.Id != current.Id && info.ReleaseScore > current.ReleaseScore) + .OrderByDescending(info => info.ReleaseScore) + .ThenByDescending(info => info.PublishTime) + .FirstOrDefault(); + if (candidate is null) continue; + var automatic = candidate.SourceFeedId is { } feedId && + policies.TryGetValue(feedId, out var policy) && + policy.EnableVersionUpgrade && + candidate.ReleaseScore - current.ReleaseScore >= policy.MinimumUpgradeScore; + candidates.Add(new ReleaseUpgradeCandidate( + current.Id, + candidate.Id, + first.Animation!.Name, + first.Season!.Value, + episodeGroup.Key, + current.ReleaseScore, + candidate.ReleaseScore, + ParseReasons(candidate.ReleaseScoreReasonsJson), + automatic)); + } + + return new LibraryIntegritySummary( + first.Animation!.TmdbId, + first.Animation.Name, + first.Season!.Value, + expected, + missing, + duplicates, + releases.Count(info => info.Episode is null), + candidates.OrderBy(item => item.Episode).ToList()); + } + + private static string ContainsPattern(string value) => + $"%{value.Trim().Replace("\\", "\\\\", StringComparison.Ordinal) + .Replace("%", "\\%", StringComparison.Ordinal) + .Replace("_", "\\_", StringComparison.Ordinal)}%"; + + private static IReadOnlyList ParseReasons(string? json) + { + if (string.IsNullOrWhiteSpace(json)) return []; + try + { + return JsonSerializer.Deserialize(json) ?? []; + } + catch (JsonException) + { + return []; + } + } + + private static string Signature(LibrarySearchRequest request) + { + var value = string.Join('\n', + request.Query?.Trim(), request.Season, request.Episode, + request.SubtitleGroup, request.Resolution, request.Codec, request.Language, + request.DownloadState, request.WatchState, request.VirtualPath, + request.Source, request.Sort, request.Take, request.UserId); + return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(value)))[..24]; + } + + private static string EncodeCursor(SearchCursor cursor) + { + var bytes = JsonSerializer.SerializeToUtf8Bytes(cursor); + return Convert.ToBase64String(bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_'); + } + + private static SearchCursor? DecodeCursor(string? value) + { + if (string.IsNullOrWhiteSpace(value)) return null; + if (value.Length > 512) throw new ArgumentException("The search cursor is too long."); + try + { + var normalized = value.Replace('-', '+').Replace('_', '/'); + normalized = normalized.PadRight((normalized.Length + 3) / 4 * 4, '='); + return JsonSerializer.Deserialize(Convert.FromBase64String(normalized)) + ?? throw new ArgumentException("The search cursor is invalid."); + } + catch (Exception exception) when (exception is FormatException or JsonException) + { + throw new ArgumentException("The search cursor is invalid.", exception); + } + } +} diff --git a/SecondDimensionWatcherReDive/Repositories/ReleaseUpgradeRepository.cs b/SecondDimensionWatcherReDive/Repositories/ReleaseUpgradeRepository.cs new file mode 100644 index 0000000..cff38da --- /dev/null +++ b/SecondDimensionWatcherReDive/Repositories/ReleaseUpgradeRepository.cs @@ -0,0 +1,468 @@ +using Microsoft.EntityFrameworkCore; +using Npgsql; +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Repositories; + +public sealed class ReleaseUpgradeRepository( + Models.ApplicationContext context, + DbContextOptions contextOptions) : IReleaseUpgradeRepository +{ + public async Task> GetCandidatesAsync( + bool automaticOnly, + int take, + CancellationToken cancellationToken) + { + if (take is < 1 or > 200) + throw new ArgumentOutOfRangeException(nameof(take)); + + var releases = await context.AnimationInfo.AsNoTracking() + .Include(info => info.Animation) + .Where(info => info.Animation != null && info.Season != null && info.Episode != null && + info.MediaLibraryMissingSince == null) + .ToListAsync(cancellationToken); + var ids = releases.Select(info => info.Id).ToArray(); + var mapped = await context.FileMappings.AsNoTracking() + .Where(mapping => ids.Contains(mapping.AnimationInfoId)) + .Select(mapping => mapping.AnimationInfoId) + .Distinct() + .ToHashSetAsync(cancellationToken); + var policies = await context.SubscriptionAutomationPolicies.AsNoTracking() + .ToDictionaryAsync(policy => policy.FeedId, cancellationToken); + var attempted = await context.ReleaseUpgradeOperations.AsNoTracking() + .Select(operation => operation.CandidateReleaseId) + .ToHashSetAsync(cancellationToken); + + var candidates = new List(); + foreach (var episode in releases.GroupBy(info => new + { + AnimationId = info.Animation!.Id, + info.Season, + info.Episode + })) + { + var current = episode + .Where(info => info.IsActiveRelease && info.IsDownloadFinished && mapped.Contains(info.Id)) + .OrderByDescending(info => info.ReleaseScore) + .ThenByDescending(info => info.PublishTime) + .FirstOrDefault(); + if (current is null) continue; + + var candidate = episode + .Where(info => info.Id != current.Id && + info.ReleaseScore > current.ReleaseScore && + !attempted.Contains(info.Id)) + .OrderByDescending(info => info.ReleaseScore) + .ThenByDescending(info => info.PublishTime) + .FirstOrDefault(); + if (candidate is null) continue; + + var automatic = candidate.SourceFeedId is { } feedId && + policies.TryGetValue(feedId, out var policy) && + policy.EnableVersionUpgrade && + candidate.ReleaseScore - current.ReleaseScore >= policy.MinimumUpgradeScore; + if (automaticOnly && !automatic) continue; + + candidates.Add(new ReleaseUpgradeCandidate( + current.Id, + candidate.Id, + current.Animation!.Name, + current.Season!.Value, + current.Episode!.Value, + current.ReleaseScore, + candidate.ReleaseScore, + ParseReasons(candidate.ReleaseScoreReasonsJson), + automatic)); + } + + return candidates + .OrderByDescending(candidate => candidate.CandidateScore - candidate.CurrentScore) + .ThenBy(candidate => candidate.AnimationName) + .ThenBy(candidate => candidate.Season) + .ThenBy(candidate => candidate.Episode) + .Take(take) + .ToList(); + } + + public async Task TryBeginAsync( + ReleaseUpgradeCandidate candidate, + DateTimeOffset createdAt, + CancellationToken cancellationToken) + { + var strategy = context.Database.CreateExecutionStrategy(); + return await strategy.ExecuteAsync(async () => + { + await using var writeContext = new Models.ApplicationContext(contextOptions); + await using var transaction = await writeContext.Database.BeginTransactionAsync(cancellationToken); + await MappingTransactionLock.AcquireAsync(writeContext, cancellationToken); + var releases = await MappingTransactionLock.LockAnimationInfosAsync( + writeContext, + [candidate.CurrentReleaseId, candidate.CandidateReleaseId], + cancellationToken); + if (!releases.TryGetValue(candidate.CurrentReleaseId, out var current) || + !releases.TryGetValue(candidate.CandidateReleaseId, out var next)) + return null; + await writeContext.Entry(current).Reference(info => info.Animation).LoadAsync(cancellationToken); + await writeContext.Entry(next).Reference(info => info.Animation).LoadAsync(cancellationToken); + if (current.Animation is null || next.Animation is null || + current.Animation.Id != next.Animation.Id || + current.Season != next.Season || + current.Episode != next.Episode || + next.ReleaseScore <= current.ReleaseScore || + !current.IsDownloadFinished || + !await writeContext.FileMappings.AnyAsync( + mapping => mapping.AnimationInfoId == current.Id, + cancellationToken)) + return null; + + if (await writeContext.ReleaseUpgradeOperations.AnyAsync( + operation => operation.CandidateReleaseId == next.Id || + (operation.CurrentReleaseId == current.Id && + (operation.Status == ReleaseUpgradeStatus.Downloading || + operation.Status == ReleaseUpgradeStatus.Verifying || + operation.Status == ReleaseUpgradeStatus.Applied)), + cancellationToken)) + return null; + + var entity = new Models.ReleaseUpgradeOperation + { + Id = Guid.NewGuid(), + CurrentReleaseId = current.Id, + CandidateReleaseId = next.Id, + Status = next.IsDownloadFinished + ? ReleaseUpgradeStatus.Verifying + : ReleaseUpgradeStatus.Downloading, + CurrentScore = current.ReleaseScore, + CandidateScore = next.ReleaseScore, + CreatedAt = createdAt + }; + writeContext.ReleaseUpgradeOperations.Add(entity); + try + { + await writeContext.SaveChangesAsync(cancellationToken); + await transaction.CommitAsync(cancellationToken); + return entity.ToRecord(); + } + catch (DbUpdateException exception) when (exception.InnerException is PostgresException + { SqlState: PostgresErrorCodes.UniqueViolation }) + { + return null; + } + }); + } + + public async Task FindActiveByCandidateAsync( + Guid candidateReleaseId, + CancellationToken cancellationToken) + { + var operation = await context.ReleaseUpgradeOperations.AsNoTracking() + .FirstOrDefaultAsync(item => item.CandidateReleaseId == candidateReleaseId && + (item.Status == ReleaseUpgradeStatus.Downloading || + item.Status == ReleaseUpgradeStatus.Verifying), + cancellationToken); + return operation?.ToRecord(); + } + + public async Task> GetReadyCandidateIdsAsync( + int take, + CancellationToken cancellationToken) + { + if (take is < 1 or > 200) throw new ArgumentOutOfRangeException(nameof(take)); + return await context.ReleaseUpgradeOperations.AsNoTracking() + .Where(operation => + (operation.Status == ReleaseUpgradeStatus.Downloading || + operation.Status == ReleaseUpgradeStatus.Verifying) && + operation.CandidateRelease.IsDownloadFinished && + context.FileMappings.Any(mapping => + mapping.AnimationInfoId == operation.CandidateReleaseId)) + .OrderBy(operation => operation.CreatedAt) + .Select(operation => operation.CandidateReleaseId) + .Take(take) + .ToListAsync(cancellationToken); + } + + public async Task GetActivationAsync( + Guid candidateReleaseId, + CancellationToken cancellationToken) + { + var operation = await context.ReleaseUpgradeOperations.AsNoTracking() + .FirstOrDefaultAsync(item => item.CandidateReleaseId == candidateReleaseId && + (item.Status == ReleaseUpgradeStatus.Downloading || + item.Status == ReleaseUpgradeStatus.Verifying), + cancellationToken); + if (operation is null) return null; + + var mappings = await context.FileMappings.AsNoTracking() + .Where(mapping => mapping.AnimationInfoId == operation.CurrentReleaseId || + mapping.AnimationInfoId == operation.CandidateReleaseId) + .OrderBy(mapping => mapping.VirtualPath) + .ToListAsync(cancellationToken); + return new ReleaseUpgradeActivation( + operation.ToRecord(), + mappings.Where(mapping => mapping.AnimationInfoId == operation.CurrentReleaseId) + .Select(mapping => mapping.ToRecord()).ToList(), + mappings.Where(mapping => mapping.AnimationInfoId == operation.CandidateReleaseId) + .Select(mapping => mapping.ToRecord()).ToList()); + } + + public async Task ActivateAsync( + Guid operationId, + DateTimeOffset verifiedAt, + DateTimeOffset rollbackUntil, + CancellationToken cancellationToken) + { + var strategy = context.Database.CreateExecutionStrategy(); + return await strategy.ExecuteAsync(async () => + { + await using var writeContext = new Models.ApplicationContext(contextOptions); + await using var transaction = await writeContext.Database.BeginTransactionAsync(cancellationToken); + await MappingTransactionLock.AcquireAsync(writeContext, cancellationToken); + var operation = await writeContext.ReleaseUpgradeOperations + .Include(item => item.MappingSnapshots) + .SingleOrDefaultAsync(item => item.Id == operationId, cancellationToken); + if (operation is null) + return new ReleaseUpgradeMutationResult(false, "not_found", null); + if (operation.Status == ReleaseUpgradeStatus.Applied) + return new ReleaseUpgradeMutationResult(true, "already_applied", operation.ToRecord()); + if (operation.Status is not (ReleaseUpgradeStatus.Downloading or ReleaseUpgradeStatus.Verifying)) + return new ReleaseUpgradeMutationResult(false, "invalid_state", operation.ToRecord()); + + var infos = await MappingTransactionLock.LockAnimationInfosAsync( + writeContext, + [operation.CurrentReleaseId, operation.CandidateReleaseId], + cancellationToken); + if (!infos.TryGetValue(operation.CurrentReleaseId, out var current) || + !infos.TryGetValue(operation.CandidateReleaseId, out var candidate) || + !candidate.IsDownloadFinished) + return new ReleaseUpgradeMutationResult(false, "candidate_not_ready", operation.ToRecord()); + + var mappings = await writeContext.FileMappings + .Where(mapping => mapping.AnimationInfoId == current.Id || + mapping.AnimationInfoId == candidate.Id) + .OrderBy(mapping => mapping.VirtualPath) + .ToListAsync(cancellationToken); + var previous = mappings.Where(mapping => mapping.AnimationInfoId == current.Id).ToList(); + var next = mappings.Where(mapping => mapping.AnimationInfoId == candidate.Id).ToList(); + if (previous.Count == 0 || next.Count == 0) + return new ReleaseUpgradeMutationResult(false, "mapping_missing", operation.ToRecord()); + + var snapshots = previous + .Select(mapping => ToSnapshot( + operation.Id, + mapping, + ReleaseUpgradeMappingKind.Previous)) + .Concat(next.Select(mapping => ToSnapshot( + operation.Id, + mapping, + ReleaseUpgradeMappingKind.Candidate))) + .ToList(); + await writeContext.ReleaseUpgradeMappingSnapshots.AddRangeAsync( + snapshots, + cancellationToken); + + writeContext.FileMappings.RemoveRange(mappings); + var replacement = BuildCandidateReplacement(previous, next, candidate.Id); + await writeContext.FileMappings.AddRangeAsync(replacement, cancellationToken); + await writeContext.AnimationInfo + .Where(info => info.Id == current.Id) + .ExecuteUpdateAsync(setters => setters + .SetProperty(info => info.StateVersion, info => info.StateVersion + 1) + .SetProperty(info => info.IsActiveRelease, false), + cancellationToken); + await writeContext.AnimationInfo + .Where(info => info.Id == candidate.Id) + .ExecuteUpdateAsync(setters => setters + .SetProperty(info => info.StateVersion, info => info.StateVersion + 1) + .SetProperty(info => info.IsActiveRelease, true), + cancellationToken); + operation.Status = ReleaseUpgradeStatus.Applied; + operation.VerifiedAt = verifiedAt; + operation.AppliedAt = verifiedAt; + operation.RollbackUntil = rollbackUntil; + await writeContext.SaveChangesAsync(cancellationToken); + await transaction.CommitAsync(cancellationToken); + return new ReleaseUpgradeMutationResult(true, "applied", operation.ToRecord()); + }); + } + + public async Task MarkFailedAsync( + Guid operationId, + string failureSummary, + CancellationToken cancellationToken) + { + var operation = await context.ReleaseUpgradeOperations + .SingleOrDefaultAsync(item => item.Id == operationId, cancellationToken); + if (operation is null) + return new ReleaseUpgradeMutationResult(false, "not_found", null); + if (operation.Status is ReleaseUpgradeStatus.Completed or ReleaseUpgradeStatus.RolledBack) + return new ReleaseUpgradeMutationResult(false, "invalid_state", operation.ToRecord()); + operation.Status = ReleaseUpgradeStatus.Failed; + operation.FailureSummary = failureSummary.Length <= 2048 + ? failureSummary + : failureSummary[..2048]; + operation.CompletedAt = DateTimeOffset.UtcNow; + await context.SaveChangesAsync(cancellationToken); + return new ReleaseUpgradeMutationResult(true, "failed", operation.ToRecord()); + } + + public async Task RollbackAsync( + Guid operationId, + DateTimeOffset rolledBackAt, + CancellationToken cancellationToken) + { + var strategy = context.Database.CreateExecutionStrategy(); + return await strategy.ExecuteAsync(async () => + { + await using var writeContext = new Models.ApplicationContext(contextOptions); + await using var transaction = await writeContext.Database.BeginTransactionAsync(cancellationToken); + await MappingTransactionLock.AcquireAsync(writeContext, cancellationToken); + var operation = await writeContext.ReleaseUpgradeOperations + .Include(item => item.MappingSnapshots) + .SingleOrDefaultAsync(item => item.Id == operationId, cancellationToken); + if (operation is null) + return new ReleaseUpgradeMutationResult(false, "not_found", null); + if (operation.Status == ReleaseUpgradeStatus.RolledBack) + return new ReleaseUpgradeMutationResult(true, "already_rolled_back", operation.ToRecord()); + if (operation.Status != ReleaseUpgradeStatus.Applied || operation.RollbackUntil < rolledBackAt) + return new ReleaseUpgradeMutationResult(false, "rollback_unavailable", operation.ToRecord()); + + var infos = await MappingTransactionLock.LockAnimationInfosAsync( + writeContext, + [operation.CurrentReleaseId, operation.CandidateReleaseId], + cancellationToken); + var previous = operation.MappingSnapshots + .Where(snapshot => snapshot.Kind == ReleaseUpgradeMappingKind.Previous) + .ToList(); + if (previous.Count == 0) + return new ReleaseUpgradeMutationResult(false, "snapshot_missing", operation.ToRecord()); + + await writeContext.FileMappings + .Where(mapping => mapping.AnimationInfoId == operation.CandidateReleaseId) + .ExecuteDeleteAsync(cancellationToken); + await writeContext.FileMappings.AddRangeAsync(previous.Select(snapshot => new Models.FileMapping + { + Id = snapshot.OriginalMappingId, + AnimationInfoId = snapshot.AnimationInfoId, + VirtualPath = snapshot.VirtualPath, + PhysicalPath = snapshot.PhysicalPath, + FileStore = snapshot.FileStore + }), cancellationToken); + await writeContext.AnimationInfo + .Where(info => info.Id == operation.CurrentReleaseId) + .ExecuteUpdateAsync(setters => setters + .SetProperty(info => info.StateVersion, info => info.StateVersion + 1) + .SetProperty(info => info.IsActiveRelease, true), + cancellationToken); + await writeContext.AnimationInfo + .Where(info => info.Id == operation.CandidateReleaseId) + .ExecuteUpdateAsync(setters => setters + .SetProperty(info => info.StateVersion, info => info.StateVersion + 1) + .SetProperty(info => info.IsActiveRelease, false), + cancellationToken); + operation.Status = ReleaseUpgradeStatus.RolledBack; + operation.CompletedAt = rolledBackAt; + await writeContext.SaveChangesAsync(cancellationToken); + await transaction.CommitAsync(cancellationToken); + return new ReleaseUpgradeMutationResult(true, "rolled_back", operation.ToRecord()); + }); + } + + public async Task> GetHistoryAsync( + int take, + CancellationToken cancellationToken) + { + if (take is < 1 or > 200) throw new ArgumentOutOfRangeException(nameof(take)); + return (await context.ReleaseUpgradeOperations.AsNoTracking() + .OrderByDescending(operation => operation.CreatedAt) + .Take(take) + .ToListAsync(cancellationToken)) + .Select(operation => operation.ToRecord()) + .ToList(); + } + + public Task CompleteExpiredAsync( + DateTimeOffset completedAt, + CancellationToken cancellationToken) => + context.ReleaseUpgradeOperations + .Where(operation => operation.Status == ReleaseUpgradeStatus.Applied && + operation.RollbackUntil <= completedAt) + .ExecuteUpdateAsync(setters => setters + .SetProperty(operation => operation.Status, ReleaseUpgradeStatus.Completed) + .SetProperty(operation => operation.CompletedAt, completedAt), + cancellationToken); + + private static Models.ReleaseUpgradeMappingSnapshot ToSnapshot( + Guid operationId, + Models.FileMapping mapping, + ReleaseUpgradeMappingKind kind) => new() + { + Id = Guid.NewGuid(), + OperationId = operationId, + Kind = kind, + OriginalMappingId = mapping.Id, + AnimationInfoId = mapping.AnimationInfoId, + VirtualPath = mapping.VirtualPath, + PhysicalPath = mapping.PhysicalPath, + FileStore = mapping.FileStore + }; + + private static IReadOnlyList BuildCandidateReplacement( + IReadOnlyList previous, + IReadOnlyList candidate, + Guid candidateReleaseId) + { + var remaining = new HashSet(candidate.Select(item => item.VirtualPath), StringComparer.Ordinal); + var used = new HashSet(StringComparer.Ordinal); + var result = new List(candidate.Count); + for (var index = 0; index < candidate.Count; index++) + { + var mapping = candidate[index]; + remaining.Remove(mapping.VirtualPath); + var preferred = index < previous.Count ? previous[index].VirtualPath : mapping.VirtualPath; + var virtualPath = !used.Contains(preferred) && !remaining.Contains(preferred) + ? preferred + : mapping.VirtualPath; + used.Add(virtualPath); + result.Add(new Models.FileMapping + { + Id = Guid.NewGuid(), + AnimationInfoId = candidateReleaseId, + VirtualPath = virtualPath, + PhysicalPath = mapping.PhysicalPath, + FileStore = mapping.FileStore + }); + } + + return result; + } + + private static IReadOnlyList ParseReasons(string? json) + { + if (string.IsNullOrWhiteSpace(json)) return []; + try + { + return System.Text.Json.JsonSerializer.Deserialize(json) ?? []; + } + catch (System.Text.Json.JsonException) + { + return []; + } + } +} + +internal static class ReleaseUpgradeRepositoryConverters +{ + public static ReleaseUpgradeOperation ToRecord(this Models.ReleaseUpgradeOperation operation) => + new(operation.Id, + operation.CurrentReleaseId, + operation.CandidateReleaseId, + operation.Status, + operation.CurrentScore, + operation.CandidateScore, + operation.CreatedAt, + operation.VerifiedAt, + operation.AppliedAt, + operation.RollbackUntil, + operation.CompletedAt, + operation.FailureSummary); +} diff --git a/SecondDimensionWatcherReDive/Repositories/RepositoryConverter.cs b/SecondDimensionWatcherReDive/Repositories/RepositoryConverter.cs index 260e342..4b264d0 100644 --- a/SecondDimensionWatcherReDive/Repositories/RepositoryConverter.cs +++ b/SecondDimensionWatcherReDive/Repositories/RepositoryConverter.cs @@ -47,7 +47,20 @@ public static DataRepo.AnimationInfo ToRecord(this Models.AnimationInfo entity) entity.DownloadAttemptId, entity.DownloadCancellationId, entity.MediaLibrarySourceId, - entity.MediaLibraryMissingSince); + entity.MediaLibraryMissingSince, + entity.ReleaseIdentity, + entity.FeedItemGuid, + entity.EnclosureId, + entity.TorrentInfoHash, + entity.ReleaseSubtitleGroup, + entity.ReleaseResolution, + entity.ReleaseCodec, + entity.ReleaseLanguages, + entity.ReleaseScore, + entity.ReleaseScoreReasonsJson, + entity.ExpectedEpisodeCount, + entity.IngestedAt, + entity.IsActiveRelease); public static DataRepo.Animation ToRecord(this Models.Animation entity) => new(entity.Id, @@ -103,7 +116,10 @@ public static DataRepo.SubscriptionAutomationPolicy ToRecord( entity.ExcludedKeywords, entity.Mode, entity.CreatedAt, - entity.UpdatedAt); + entity.UpdatedAt, + entity.EnableVersionUpgrade, + entity.MinimumUpgradeScore, + entity.UpgradeRollbackHours); public static DataRepo.WebDavToken ToRecord(this Models.WebDavToken entity) => new(entity.Id, @@ -168,7 +184,20 @@ public static Models.AnimationInfo ToEntity(this DataRepo.AnimationInfo record) DownloadAttemptId = record.DownloadAttemptId, DownloadCancellationId = record.DownloadCancellationId, MediaLibrarySourceId = record.MediaLibrarySourceId, - MediaLibraryMissingSince = record.MediaLibraryMissingSince + MediaLibraryMissingSince = record.MediaLibraryMissingSince, + ReleaseIdentity = record.ReleaseIdentity, + FeedItemGuid = record.FeedItemGuid, + EnclosureId = record.EnclosureId, + TorrentInfoHash = record.TorrentInfoHash, + ReleaseSubtitleGroup = record.ReleaseSubtitleGroup, + ReleaseResolution = record.ReleaseResolution, + ReleaseCodec = record.ReleaseCodec, + ReleaseLanguages = record.ReleaseLanguages?.ToArray() ?? [], + ReleaseScore = record.ReleaseScore, + ReleaseScoreReasonsJson = record.ReleaseScoreReasonsJson, + ExpectedEpisodeCount = record.ExpectedEpisodeCount, + IngestedAt = record.IngestedAt ?? DateTimeOffset.UtcNow, + IsActiveRelease = record.IsActiveRelease }; public static Models.Animation ToEntity(this DataRepo.Animation record) => @@ -252,7 +281,10 @@ public static Models.SubscriptionAutomationPolicy ToEntity( ExcludedKeywords = record.ExcludedKeywords.ToArray(), Mode = record.Mode, CreatedAt = record.CreatedAt, - UpdatedAt = record.UpdatedAt + UpdatedAt = record.UpdatedAt, + EnableVersionUpgrade = record.EnableVersionUpgrade, + MinimumUpgradeScore = record.MinimumUpgradeScore, + UpgradeRollbackHours = record.UpgradeRollbackHours }; public static Models.WebDavToken ToEntity(this DataRepo.WebDavToken record) => @@ -326,6 +358,17 @@ public static void ApplyTo(this DataRepo.AnimationInfo record, Models.AnimationI entity.DownloadCancellationId = record.DownloadCancellationId; entity.MediaLibrarySourceId = record.MediaLibrarySourceId; entity.MediaLibraryMissingSince = record.MediaLibraryMissingSince; + entity.ReleaseIdentity = record.ReleaseIdentity; + entity.FeedItemGuid = record.FeedItemGuid; + entity.EnclosureId = record.EnclosureId; + entity.TorrentInfoHash = record.TorrentInfoHash; + entity.ReleaseSubtitleGroup = record.ReleaseSubtitleGroup; + entity.ReleaseResolution = record.ReleaseResolution; + entity.ReleaseCodec = record.ReleaseCodec; + entity.ReleaseLanguages = record.ReleaseLanguages?.ToArray() ?? []; + entity.ReleaseScore = record.ReleaseScore; + entity.ReleaseScoreReasonsJson = record.ReleaseScoreReasonsJson; + entity.ExpectedEpisodeCount = record.ExpectedEpisodeCount; } public static void ApplyTo(this DataRepo.SeasonBangumi record, Models.SeasonBangumi entity) @@ -359,5 +402,8 @@ public static void ApplyTo( entity.Mode = record.Mode; entity.CreatedAt = record.CreatedAt; entity.UpdatedAt = record.UpdatedAt; + entity.EnableVersionUpgrade = record.EnableVersionUpgrade; + entity.MinimumUpgradeScore = record.MinimumUpgradeScore; + entity.UpgradeRollbackHours = record.UpgradeRollbackHours; } } diff --git a/SecondDimensionWatcherReDive/Services/CompleteDownloadBackgroundService.cs b/SecondDimensionWatcherReDive/Services/CompleteDownloadBackgroundService.cs index d6f2eb9..7116c8d 100644 --- a/SecondDimensionWatcherReDive/Services/CompleteDownloadBackgroundService.cs +++ b/SecondDimensionWatcherReDive/Services/CompleteDownloadBackgroundService.cs @@ -5,6 +5,7 @@ using SecondDimensionWatcherReDive.Plugin; using SecondDimensionWatcherReDive.Utils.FileStore; using SecondDimensionWatcherReDive.Utils.Incidents; +using SecondDimensionWatcherReDive.Utils.ReleaseUpgrades; namespace SecondDimensionWatcherReDive.Services; @@ -72,6 +73,9 @@ internal async Task ProcessRequestAsync( if (info is null) { + if (scope.ServiceProvider.GetService() is { } retryCoordinator && + await retryCoordinator.TryActivateCandidateAsync(request.ItemId, cancellationToken) is not null) + return; LogCompletionIgnored(logger, request.ItemId); return; } @@ -86,11 +90,13 @@ await incidentReporter.ResolveAsync( } // Build virtual-fs mappings for the downloaded files. + var mappingSucceeded = false; try { var fileMapper = scope.ServiceProvider.GetRequiredService(); if (!await fileMapper.MapDownloadAsync(request.ItemId, cancellationToken)) throw new InvalidOperationException("No file mapping could be produced."); + mappingSucceeded = true; if (incidentReporter is not null) { @@ -115,6 +121,11 @@ await incidentReporter.ReportAsync(new IncidentReport( } } + if (mappingSucceeded && scope.ServiceProvider.GetService() is { } coordinator) + { + await coordinator.TryActivateCandidateAsync(request.ItemId, cancellationToken); + } + // Fire plugin event try { diff --git a/SecondDimensionWatcherReDive/Services/InferAnimationMetadata.cs b/SecondDimensionWatcherReDive/Services/InferAnimationMetadata.cs index 517f322..36c8f66 100644 --- a/SecondDimensionWatcherReDive/Services/InferAnimationMetadata.cs +++ b/SecondDimensionWatcherReDive/Services/InferAnimationMetadata.cs @@ -89,6 +89,16 @@ private async Task ProcessItem( if (details != null && !string.IsNullOrEmpty(details.Overview)) item = item with { Description = details.Overview }; + if (result.Season is { } inferredSeason) + { + var expectedEpisodeCount = await tmdbTool.GetExpectedEpisodeCountAsync( + tmdbIdInt, + inferredSeason, + cancellationToken); + if (expectedEpisodeCount is not null) + item = item with { ExpectedEpisodeCount = expectedEpisodeCount }; + } + var animation = await animationRepository .FindByTmdbIdAsync(result.TmdbId, cancellationToken); diff --git a/SecondDimensionWatcherReDive/Services/MediaLibraryScanner.cs b/SecondDimensionWatcherReDive/Services/MediaLibraryScanner.cs index 8b17ee8..2e7d095 100644 --- a/SecondDimensionWatcherReDive/Services/MediaLibraryScanner.cs +++ b/SecondDimensionWatcherReDive/Services/MediaLibraryScanner.cs @@ -1,6 +1,7 @@ using Microsoft.Extensions.Options; using SecondDimensionWatcherReDive.Framework.DataRepository; using SecondDimensionWatcherReDive.Framework.FileDownload; +using SecondDimensionWatcherReDive.Framework.Feed; using SecondDimensionWatcherReDive.Framework.FileStore; using SecondDimensionWatcherReDive.Utils.FileStore; @@ -399,7 +400,11 @@ private static AnimationInfo CreateAnimationInfo( AiRetryCount: 0, ReleaseSizeBytes: candidate.TotalSize, MetadataStatus: MetadataReviewStatus.Pending, - MediaLibrarySourceId: source.Id); + MediaLibrarySourceId: source.Id, + ReleaseIdentity: ReleaseIdentity.CreateMediaImport( + source.Id, + FileStores.LocalDiskStore, + candidate.FullPath)); } private static string BuildDownloadUrl(Guid sourceId, string relativePath) => diff --git a/SecondDimensionWatcherReDive/Services/ReleaseUpgradeBackgroundService.cs b/SecondDimensionWatcherReDive/Services/ReleaseUpgradeBackgroundService.cs new file mode 100644 index 0000000..1ad96f2 --- /dev/null +++ b/SecondDimensionWatcherReDive/Services/ReleaseUpgradeBackgroundService.cs @@ -0,0 +1,55 @@ +using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Utils.ReleaseUpgrades; + +namespace SecondDimensionWatcherReDive.Services; + +public sealed class ReleaseUpgradeBackgroundService( + IServiceScopeFactory scopeFactory, + ILogger logger) : BackgroundService +{ + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + using var timer = new PeriodicTimer(TimeSpan.FromMinutes(1)); + do + { + try + { + await ProcessAsync(stoppingToken); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + catch (Exception exception) + { + logger.LogError(exception, "Automatic release-upgrade pass failed"); + } + } while (await timer.WaitForNextTickAsync(stoppingToken)); + } + + internal async Task ProcessAsync(CancellationToken cancellationToken) + { + await using var scope = scopeFactory.CreateAsyncScope(); + var repository = scope.ServiceProvider.GetRequiredService(); + await repository.CompleteExpiredAsync(DateTimeOffset.UtcNow, cancellationToken); + var coordinator = scope.ServiceProvider.GetRequiredService(); + var readyCandidates = await repository.GetReadyCandidateIdsAsync( + take: 20, + cancellationToken); + foreach (var candidateId in readyCandidates) + { + cancellationToken.ThrowIfCancellationRequested(); + await coordinator.TryActivateCandidateAsync(candidateId, cancellationToken); + } + + var candidates = await repository.GetCandidatesAsync( + automaticOnly: true, + take: 20, + cancellationToken); + foreach (var candidate in candidates) + { + cancellationToken.ThrowIfCancellationRequested(); + await coordinator.ExecuteAsync(candidate, dryRun: false, cancellationToken); + } + } +} diff --git a/SecondDimensionWatcherReDive/Services/SyncFeed.cs b/SecondDimensionWatcherReDive/Services/SyncFeed.cs index 6acd879..6ce3f57 100644 --- a/SecondDimensionWatcherReDive/Services/SyncFeed.cs +++ b/SecondDimensionWatcherReDive/Services/SyncFeed.cs @@ -20,7 +20,9 @@ public partial class SyncFeed( IHttpClientFactory httpClientFactory, IServiceScopeFactory scopeFactory, ISubscriptionAutomationMatcher automationMatcher, - IIncidentReporter? incidentReporter = null) + IIncidentReporter? incidentReporter = null, + ISubscriptionReleaseMetadataExtractor? metadataExtractor = null, + IReleaseScoringService? releaseScoringService = null) : ScheduledTaskBase { private readonly HttpClient _httpClient = httpClientFactory.CreateClient("Feed"); @@ -106,41 +108,44 @@ private async Task ProcessSingle(AnimationAddRequest request, CancellationToken await using var scope = scopeFactory.CreateAsyncScope(); var animationInfoRepository = scope.ServiceProvider.GetRequiredService(); - //Only process non-exist items - if (await animationInfoRepository.FindByTitleAsync(request.Title, cancellationToken) == null) + try { - try + SubscriptionAutomationPolicy? policy = null; + if (request.FeedId is { } feedId) { - SubscriptionAutomationPolicy? policy = null; - if (request.FeedId is { } feedId) - { - var policyRepository = scope.ServiceProvider - .GetRequiredService(); - policy = await policyRepository.FindByFeedIdAsync(feedId, cancellationToken); - } + var policyRepository = scope.ServiceProvider + .GetRequiredService(); + policy = await policyRepository.FindByFeedIdAsync(feedId, cancellationToken); + } - var torrentData = request.DownloadType switch - { - FileDownloadTypes.TorrentDownload => await DownloadTorrentData(request, cancellationToken), - _ => new TorrentData(Array.Empty(), request.AdditionalDownloadInfo, request.ContentLength) - }; + var torrentData = request.DownloadType switch + { + FileDownloadTypes.TorrentDownload => await DownloadTorrentData(request, cancellationToken), + _ => new TorrentData(Array.Empty(), request.AdditionalDownloadInfo, request.ContentLength) + }; - if (request.DownloadType == FileDownloadTypes.TorrentDownload && - request.ContentLength is { } advertisedSize && - advertisedSize != torrentData.PayloadSizeBytes) - throw new InvalidTorrentDataException(request.DownloadUrl, "advertised and declared payload sizes differ"); + if (request.DownloadType == FileDownloadTypes.TorrentDownload && + request.ContentLength is { } advertisedSize && + advertisedSize != torrentData.PayloadSizeBytes) + throw new InvalidTorrentDataException(request.DownloadUrl, "advertised and declared payload sizes differ"); - SubscriptionAutomationEvaluation? evaluation = null; - if (policy is not null) - { - evaluation = automationMatcher.Evaluate( - policy, - request with { ContentLength = torrentData.PayloadSizeBytes }); - if (!evaluation.Matched) - return; - } + var releaseWithSize = request with { ContentLength = torrentData.PayloadSizeBytes }; + SubscriptionAutomationEvaluation? evaluation = null; + if (policy is not null) + { + evaluation = automationMatcher.Evaluate(policy, releaseWithSize); + if (!evaluation.Matched) + return; + } - var info = new AnimationInfo( + var metadata = evaluation?.Metadata ?? metadataExtractor?.Extract(releaseWithSize) ?? + new SubscriptionReleaseMetadata(null, null, null, [], torrentData.PayloadSizeBytes); + var score = releaseScoringService?.Score(metadata, policy) ?? new ReleaseScore(0, []); + var torrentInfoHash = request.DownloadType == FileDownloadTypes.TorrentDownload + ? torrentData.Hash + : null; + + var info = new AnimationInfo( Guid.NewGuid(), request.Title, request.Description, @@ -174,35 +179,50 @@ request.ContentLength is { } advertisedSize && }, AutomationExplanationJson: evaluation is null ? null - : JsonSerializer.Serialize(evaluation.Explanations, ExplanationJsonOptions)); - await animationInfoRepository.AddAsync(info, cancellationToken); + : JsonSerializer.Serialize(evaluation.Explanations, ExplanationJsonOptions), + ReleaseIdentity: ReleaseIdentity.Create( + request.FeedId, + request.FeedItemGuid, + request.EnclosureId, + torrentInfoHash, + request.DownloadUrl), + FeedItemGuid: request.FeedItemGuid, + EnclosureId: request.EnclosureId, + TorrentInfoHash: torrentInfoHash, + ReleaseSubtitleGroup: metadata.SubtitleGroup, + ReleaseResolution: metadata.Resolution, + ReleaseCodec: metadata.Codec, + ReleaseLanguages: metadata.Languages, + ReleaseScore: score.Value, + ReleaseScoreReasonsJson: JsonSerializer.Serialize(score.Reasons, ExplanationJsonOptions)); + if (!await animationInfoRepository.TryAddReleaseAsync(info, cancellationToken)) + return; - if (incidentReporter is not null) - await incidentReporter.ResolveAsync( - IncidentType.FeedFailure, - request.DownloadUrl, - cancellationToken); + if (incidentReporter is not null) + await incidentReporter.ResolveAsync( + IncidentType.FeedFailure, + request.DownloadUrl, + cancellationToken); - if (policy?.Mode == SubscriptionAutomationMode.AutoDownload) - await QueueAutomaticDownloadAsync( - info, - animationInfoRepository, - scope.ServiceProvider.GetRequiredService(), - cancellationToken); - } - catch (InvalidTorrentDataException e) + if (policy?.Mode == SubscriptionAutomationMode.AutoDownload) + await QueueAutomaticDownloadAsync( + info, + animationInfoRepository, + scope.ServiceProvider.GetRequiredService(), + cancellationToken); + } + catch (InvalidTorrentDataException e) + { + LogSyncFeedWarning(logger, e.Message); + if (incidentReporter is not null) { - LogSyncFeedWarning(logger, e.Message); - if (incidentReporter is not null) - { - await incidentReporter.ReportAsync(new IncidentReport( - IncidentType.FeedFailure, - IncidentSeverity.Error, - "Feed item contains invalid torrent data", - e.Message, - request.DownloadUrl), - cancellationToken); - } + await incidentReporter.ReportAsync(new IncidentReport( + IncidentType.FeedFailure, + IncidentSeverity.Error, + "Feed item contains invalid torrent data", + e.Message, + request.DownloadUrl), + cancellationToken); } } } diff --git a/SecondDimensionWatcherReDive/Utils/Feed/MikananiSubscriptionFeedReader.cs b/SecondDimensionWatcherReDive/Utils/Feed/MikananiSubscriptionFeedReader.cs index 14f2cdc..ed06cad 100644 --- a/SecondDimensionWatcherReDive/Utils/Feed/MikananiSubscriptionFeedReader.cs +++ b/SecondDimensionWatcherReDive/Utils/Feed/MikananiSubscriptionFeedReader.cs @@ -50,7 +50,9 @@ item.Enclosure is null || FileDownloadTypes.TorrentDownload, string.Empty, feedId, - item.Torrent.ContentLength > 0 ? item.Torrent.ContentLength : null)); + item.Torrent.ContentLength > 0 ? item.Torrent.ContentLength : null, + item.Guid?.Text, + item.Enclosure.Url)); } return releases; diff --git a/SecondDimensionWatcherReDive/Utils/Feed/ReleaseScoringService.cs b/SecondDimensionWatcherReDive/Utils/Feed/ReleaseScoringService.cs new file mode 100644 index 0000000..1b405ac --- /dev/null +++ b/SecondDimensionWatcherReDive/Utils/Feed/ReleaseScoringService.cs @@ -0,0 +1,104 @@ +using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Framework.Feed; + +namespace SecondDimensionWatcherReDive.Utils.Feed; + +public sealed class ReleaseScoringService : IReleaseScoringService +{ + public ReleaseScore Score( + SubscriptionReleaseMetadata metadata, + SubscriptionAutomationPolicy? policy) + { + var reasons = new List(); + var score = ScoreResolution(metadata.Resolution, reasons) + + ScoreCodec(metadata.Codec, reasons) + + ScoreSubtitleGroup(metadata.SubtitleGroup, policy, reasons) + + ScoreLanguages(metadata.Languages, policy, reasons) + + ScoreSize(metadata.SizeBytes, reasons); + return new ReleaseScore(score, reasons); + } + + private static int ScoreResolution(string? resolution, ICollection reasons) + { + var value = resolution?.Trim().ToUpperInvariant() switch + { + "2160P" or "4K" or "UHD" => 400, + "1440P" => 300, + "1080P" => 200, + "720P" => 100, + "576P" => 60, + "480P" => 40, + _ => 0 + }; + if (value > 0) reasons.Add($"resolution:{resolution}:+{value}"); + return value; + } + + private static int ScoreCodec(string? codec, ICollection reasons) + { + var value = codec?.Trim().ToUpperInvariant() switch + { + "AV1" => 80, + "HEVC" or "H265" or "H.265" => 60, + "AVC" or "H264" or "H.264" => 40, + "VP9" => 30, + _ => 0 + }; + if (value > 0) reasons.Add($"codec:{codec}:+{value}"); + return value; + } + + private static int ScoreSubtitleGroup( + string? group, + SubscriptionAutomationPolicy? policy, + ICollection reasons) + { + if (string.IsNullOrWhiteSpace(group)) return 0; + var index = policy?.SubtitleGroups + .Select((value, position) => (value, position)) + .Where(item => string.Equals( + item.value.Trim('[', ']', '【', '】'), + group.Trim('[', ']', '【', '】'), + StringComparison.OrdinalIgnoreCase)) + .Select(item => (int?)item.position) + .FirstOrDefault(); + var value = policy is { SubtitleGroups.Count: > 0 } + ? index is { } position ? Math.Max(10, 50 - position * 5) : 0 + : 20; + reasons.Add($"subtitleGroup:{group}:+{value}"); + return value; + } + + private static int ScoreLanguages( + IReadOnlyList languages, + SubscriptionAutomationPolicy? policy, + ICollection reasons) + { + var preferred = policy?.Languages ?? []; + var score = 0; + foreach (var language in languages.Distinct(StringComparer.OrdinalIgnoreCase)) + { + var value = preferred.Count == 0 || preferred.Contains(language, StringComparer.OrdinalIgnoreCase) + ? 20 + : 5; + score += value; + reasons.Add($"language:{language}:+{value}"); + } + return score; + } + + private static int ScoreSize(long? sizeBytes, ICollection reasons) + { + if (sizeBytes is not > 0) return 0; + var gibibytes = sizeBytes.Value / (1024d * 1024 * 1024); + var value = gibibytes switch + { + >= 8 => 40, + >= 2 => 25, + >= 0.7 => 10, + _ => 5 + }; + reasons.Add($"size:{gibibytes:F2}GiB:+{value}"); + return value; + } +} diff --git a/SecondDimensionWatcherReDive/Utils/ReleaseUpgrades/IReleaseUpgradeCoordinator.cs b/SecondDimensionWatcherReDive/Utils/ReleaseUpgrades/IReleaseUpgradeCoordinator.cs new file mode 100644 index 0000000..6690e65 --- /dev/null +++ b/SecondDimensionWatcherReDive/Utils/ReleaseUpgrades/IReleaseUpgradeCoordinator.cs @@ -0,0 +1,27 @@ +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Utils.ReleaseUpgrades; + +public sealed record ReleaseUpgradeExecutionResult( + bool IsSuccess, + string Outcome, + bool DryRun, + bool RequiresDownload, + ReleaseUpgradeOperation? Operation, + IReadOnlyList ValidationErrors); + +public interface IReleaseUpgradeCoordinator +{ + Task ExecuteAsync( + ReleaseUpgradeCandidate candidate, + bool dryRun, + CancellationToken cancellationToken); + + Task TryActivateCandidateAsync( + Guid candidateReleaseId, + CancellationToken cancellationToken); + + Task RollbackAsync( + Guid operationId, + CancellationToken cancellationToken); +} diff --git a/SecondDimensionWatcherReDive/Utils/ReleaseUpgrades/ReleaseUpgradeCoordinator.cs b/SecondDimensionWatcherReDive/Utils/ReleaseUpgrades/ReleaseUpgradeCoordinator.cs new file mode 100644 index 0000000..2424349 --- /dev/null +++ b/SecondDimensionWatcherReDive/Utils/ReleaseUpgrades/ReleaseUpgradeCoordinator.cs @@ -0,0 +1,262 @@ +using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Framework.FileDownload; +using SecondDimensionWatcherReDive.Framework.FileStore; +using SecondDimensionWatcherReDive.Utils.Incidents; + +namespace SecondDimensionWatcherReDive.Utils.ReleaseUpgrades; + +public sealed class ReleaseUpgradeCoordinator( + IReleaseUpgradeRepository upgradeRepository, + IAnimationInfoRepository animationInfoRepository, + ISubscriptionAutomationPolicyRepository policyRepository, + IFileMappingRepository fileMappingRepository, + IFileDownloadClientProvider downloadClientProvider, + IFileStoreProvider fileStoreProvider, + IIncidentReporter incidentReporter, + ILogger logger) : IReleaseUpgradeCoordinator +{ + public async Task ExecuteAsync( + ReleaseUpgradeCandidate candidate, + bool dryRun, + CancellationToken cancellationToken) + { + var next = await animationInfoRepository.FindByIdAsync( + candidate.CandidateReleaseId, + cancellationToken); + if (next is null) + return Result(false, "candidate_not_found", dryRun, false, null, ["Candidate release does not exist."]); + + var requiresDownload = !next.IsDownloadFinished; + if (dryRun) + { + var validationErrors = requiresDownload + ? Array.Empty() + : await ValidateCandidateAsync(candidate, cancellationToken); + return Result(validationErrors.Count == 0, + validationErrors.Count == 0 ? "ready" : "validation_failed", + true, + requiresDownload, + null, + validationErrors); + } + + var operation = await upgradeRepository.TryBeginAsync( + candidate, + DateTimeOffset.UtcNow, + cancellationToken); + if (operation is null) + return Result(false, "upgrade_already_started", false, requiresDownload, null, + ["Another worker already claimed this upgrade."]); + + if (!requiresDownload) + return await ActivateAsync(operation.CandidateReleaseId, cancellationToken); + + var downloadAttemptId = Guid.NewGuid(); + try + { + if (!await animationInfoRepository.TryStartDownloadAsync( + next.Id, + downloadAttemptId, + DateTimeOffset.UtcNow, + SubscriptionAutomationDisposition.AutoDownloadQueued, + cancellationToken)) + return await FailAsync(operation, "Candidate download state changed.", cancellationToken); + + var client = downloadClientProvider.GetRequiredClient(next.DownloadType); + if (!await client.SubmitDownloadTaskAsync( + next.Id, + next.DownloadUrl, + next.CachedDownloadData, + next.AdditionalDownloadInfo, + cancellationToken)) + { + await animationInfoRepository.TryCancelDownloadAsync( + next.Id, + downloadAttemptId, + SubscriptionAutomationDisposition.AutoDownloadFailed, + cancellationToken); + return await FailAsync(operation, "Download client rejected the candidate.", cancellationToken); + } + + return Result(true, "download_queued", false, true, operation, []); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + logger.LogWarning(exception, "Failed to queue release upgrade {OperationId}", operation.Id); + return await FailAsync(operation, exception.Message, cancellationToken); + } + } + + public async Task TryActivateCandidateAsync( + Guid candidateReleaseId, + CancellationToken cancellationToken) + { + var operation = await upgradeRepository.FindActiveByCandidateAsync( + candidateReleaseId, + cancellationToken); + return operation is null + ? null + : await ActivateAsync(candidateReleaseId, cancellationToken); + } + + public async Task RollbackAsync( + Guid operationId, + CancellationToken cancellationToken) + { + var result = await upgradeRepository.RollbackAsync( + operationId, + DateTimeOffset.UtcNow, + cancellationToken); + if (result.IsSuccess) + { + await incidentReporter.ResolveAsync( + IncidentType.FileMappingFailure, + UpgradeSource(operationId), + cancellationToken); + } + + return result; + } + + private async Task ActivateAsync( + Guid candidateReleaseId, + CancellationToken cancellationToken) + { + var activation = await upgradeRepository.GetActivationAsync( + candidateReleaseId, + cancellationToken); + if (activation is null) + return Result(false, "operation_not_found", false, false, null, + ["No active upgrade was found for this candidate."]); + + // Validation is deliberately external to the mapping transaction. Until every + // candidate file passes, the old virtual paths and physical files remain untouched. + var validationErrors = await ValidateActivationAsync(activation, cancellationToken); + if (validationErrors.Count > 0) + return await FailAsync(activation.Operation, string.Join(" ", validationErrors), cancellationToken, + validationErrors); + + var candidate = await animationInfoRepository.FindByIdAsync(candidateReleaseId, cancellationToken); + var rollbackHours = 72; + if (candidate?.SourceFeedId is { } feedId) + { + var policy = await policyRepository.FindByFeedIdAsync(feedId, cancellationToken); + rollbackHours = policy?.UpgradeRollbackHours ?? rollbackHours; + } + + var now = DateTimeOffset.UtcNow; + var mutation = await upgradeRepository.ActivateAsync( + activation.Operation.Id, + now, + now.AddHours(rollbackHours), + cancellationToken); + if (!mutation.IsSuccess) + return await FailAsync(activation.Operation, + $"Atomic mapping swap failed: {mutation.Outcome}.", + cancellationToken); + + await incidentReporter.ResolveAsync( + IncidentType.FileMappingFailure, + UpgradeSource(activation.Operation.Id), + cancellationToken); + return Result(true, mutation.Outcome, false, false, mutation.Operation, []); + } + + private async Task> ValidateCandidateAsync( + ReleaseUpgradeCandidate candidate, + CancellationToken cancellationToken) + { + var previous = await fileMappingRepository.GetForAnimationInfoAsync( + candidate.CurrentReleaseId, + cancellationToken); + var next = await fileMappingRepository.GetForAnimationInfoAsync( + candidate.CandidateReleaseId, + cancellationToken); + return await ValidateMappingsAsync(previous, next, cancellationToken); + } + + private async Task> ValidateActivationAsync( + ReleaseUpgradeActivation activation, + CancellationToken cancellationToken) => + await ValidateMappingsAsync( + activation.PreviousMappings, + activation.CandidateMappings, + cancellationToken); + + private async Task> ValidateMappingsAsync( + IReadOnlyList previousMappings, + IReadOnlyList candidateMappings, + CancellationToken cancellationToken) + { + var errors = new List(); + if (previousMappings.Count == 0) + errors.Add("The current release has no mapping to preserve."); + if (candidateMappings.Count == 0) + errors.Add("The candidate release has no mapped files."); + + var readableFiles = 0; + foreach (var mapping in candidateMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + var store = fileStoreProvider.GetRequiredClient(mapping.FileStore); + if (!await store.ExistAsync(mapping.PhysicalPath, cancellationToken)) + { + errors.Add($"Candidate file is missing: {mapping.PhysicalPath}"); + continue; + } + + var info = await store.FileInfoAsync(mapping.PhysicalPath, cancellationToken); + if (info.IsDirectory || info.Length is <= 0) + errors.Add($"Candidate file is not a readable non-empty file: {mapping.PhysicalPath}"); + else + readableFiles++; + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + errors.Add($"Candidate file validation failed for {mapping.PhysicalPath}: {exception.Message}"); + } + } + + if (candidateMappings.Count > 0 && readableFiles == 0) + errors.Add("Candidate validation found no readable file."); + return errors; + } + + private async Task FailAsync( + ReleaseUpgradeOperation operation, + string summary, + CancellationToken cancellationToken, + IReadOnlyList? errors = null) + { + var mutation = await upgradeRepository.MarkFailedAsync( + operation.Id, + summary, + cancellationToken); + await incidentReporter.ReportAsync(new IncidentReport( + IncidentType.FileMappingFailure, + IncidentSeverity.Error, + "Release upgrade failed", + summary, + UpgradeSource(operation.Id)), + cancellationToken); + return Result(false, "failed", false, false, mutation.Operation ?? operation, + errors ?? [summary]); + } + + private static string UpgradeSource(Guid operationId) => $"release-upgrade:{operationId:N}"; + + private static ReleaseUpgradeExecutionResult Result( + bool success, + string outcome, + bool dryRun, + bool requiresDownload, + ReleaseUpgradeOperation? operation, + IReadOnlyList errors) => + new(success, outcome, dryRun, requiresDownload, operation, errors); +} From 055089304bf3213fbcb47ed7f0bd499229795fbe Mon Sep 17 00:00:00 2001 From: mahoshojoHCG Date: Sat, 29 Aug 2026 23:34:06 +0800 Subject: [PATCH 04/45] feat: add durable job runtime reliability --- .../Engines/CodexAppServerEngine.cs | 32 +- .../ChatController.cs | 8 +- README.md | 6 + .../mock-server.mjs | 43 + .../src/i18n/locales/en/tasks.json | 31 + .../src/i18n/locales/ja/tasks.json | 31 + .../src/i18n/locales/zh-CN/tasks.json | 31 + .../src/pages/TasksPage.tsx | 141 ++- .../src/tasks/hooks.ts | 7 +- .../src/tasks/types.ts | 19 + .../src/tasks/utils.ts | 11 + .../DataRepository/IDurableJobRepository.cs | 110 ++ .../DataRepository/IReadinessRepository.cs | 6 + .../IScheduledTaskLeaseRepository.cs | 28 + .../IDownloadCompletionNotifier.cs | 15 + .../PluginParams/FileDownloadCompleteParam.cs | 14 +- .../Tasks/IScheduledTaskLeaseManager.cs | 23 + .../Tasks/ScheduledTaskBase.cs | 172 ++- .../Health/HealthEndpointTests.cs | 47 + .../FileMappingRepositoryPostgreSqlTests.cs | 193 +++ .../WebDavWebApplicationFactory.cs | 8 + .../CodexAppServerEngineTests.cs | 26 + .../CompleteDownloadBackgroundServiceTests.cs | 310 +++-- .../DurableJobsControllerTests.cs | 72 ++ .../ObservabilityTests.cs | 96 ++ .../ScheduledTaskBaseTests.cs | 168 +++ .../Controllers/DurableJobsController.cs | 120 ++ .../External/AppJsonSerializerContext.cs | 4 + .../Controllers/External/DurableJob.cs | 22 + ...27_AddDurableJobsAndTaskLeases.Designer.cs | 1086 +++++++++++++++++ ...60829145227_AddDurableJobsAndTaskLeases.cs | 79 ++ .../ApplicationContextModelSnapshot.cs | 103 ++ .../Models/ApplicationContext.cs | 55 + .../Models/DurableJob.cs | 22 + .../Models/ScheduledTaskState.cs | 13 + .../DurableJobMetricsBackgroundService.cs | 37 + .../Observability/ReadinessHealthChecks.cs | 111 ++ .../Observability/RuntimeTelemetry.cs | 143 +++ .../SensitiveTagRedactionProcessor.cs | 49 + SecondDimensionWatcherReDive/Program.cs | 165 ++- .../Repositories/AnimationInfoRepository.cs | 31 +- .../Repositories/DurableJobRepository.cs | 218 ++++ ...eMappingRepositoryPostgreSqlTestFixture.cs | 198 ++- .../Repositories/ReadinessRepository.cs | 11 + .../ScheduledTaskLeaseRepository.cs | 95 ++ .../SecondDimensionWatcherReDive.csproj | 7 + .../CompleteDownloadBackgroundService.cs | 392 ++++-- .../FetchRemoteTorrentBackgroundService.cs | 111 +- .../Services/MediaLibraryScanQueue.cs | 10 +- .../NullDownloadCompletionNotifier.cs | 12 + .../PostgresScheduledTaskLeaseManager.cs | 146 +++ .../ScheduledTaskBackgroundService.cs | 56 +- .../RemoteTorrentDownloadClient.cs | 6 +- .../appsettings.example.json | 15 + docs/runtime-reliability.md | 66 + packaging/appsettings.yml | 11 + 56 files changed, 4751 insertions(+), 291 deletions(-) create mode 100644 SecondDimensionWatcherReDive.Framework/DataRepository/IDurableJobRepository.cs create mode 100644 SecondDimensionWatcherReDive.Framework/DataRepository/IReadinessRepository.cs create mode 100644 SecondDimensionWatcherReDive.Framework/DataRepository/IScheduledTaskLeaseRepository.cs create mode 100644 SecondDimensionWatcherReDive.Framework/Notifications/IDownloadCompletionNotifier.cs create mode 100644 SecondDimensionWatcherReDive.Framework/Tasks/IScheduledTaskLeaseManager.cs create mode 100644 SecondDimensionWatcherReDive.IntegrationTest/Health/HealthEndpointTests.cs create mode 100644 SecondDimensionWatcherReDive.Test/DurableJobsControllerTests.cs create mode 100644 SecondDimensionWatcherReDive.Test/ObservabilityTests.cs create mode 100644 SecondDimensionWatcherReDive.Test/ScheduledTaskBaseTests.cs create mode 100644 SecondDimensionWatcherReDive/Controllers/DurableJobsController.cs create mode 100644 SecondDimensionWatcherReDive/Controllers/External/DurableJob.cs create mode 100644 SecondDimensionWatcherReDive/Migrations/20260829145227_AddDurableJobsAndTaskLeases.Designer.cs create mode 100644 SecondDimensionWatcherReDive/Migrations/20260829145227_AddDurableJobsAndTaskLeases.cs create mode 100644 SecondDimensionWatcherReDive/Models/DurableJob.cs create mode 100644 SecondDimensionWatcherReDive/Models/ScheduledTaskState.cs create mode 100644 SecondDimensionWatcherReDive/Observability/DurableJobMetricsBackgroundService.cs create mode 100644 SecondDimensionWatcherReDive/Observability/ReadinessHealthChecks.cs create mode 100644 SecondDimensionWatcherReDive/Observability/RuntimeTelemetry.cs create mode 100644 SecondDimensionWatcherReDive/Observability/SensitiveTagRedactionProcessor.cs create mode 100644 SecondDimensionWatcherReDive/Repositories/DurableJobRepository.cs create mode 100644 SecondDimensionWatcherReDive/Repositories/ReadinessRepository.cs create mode 100644 SecondDimensionWatcherReDive/Repositories/ScheduledTaskLeaseRepository.cs create mode 100644 SecondDimensionWatcherReDive/Services/NullDownloadCompletionNotifier.cs create mode 100644 SecondDimensionWatcherReDive/Services/PostgresScheduledTaskLeaseManager.cs create mode 100644 docs/runtime-reliability.md diff --git a/Plugins/SecondDimensionWatcherReDive.AI/Engines/CodexAppServerEngine.cs b/Plugins/SecondDimensionWatcherReDive.AI/Engines/CodexAppServerEngine.cs index 2735fe5..63d0ccb 100644 --- a/Plugins/SecondDimensionWatcherReDive.AI/Engines/CodexAppServerEngine.cs +++ b/Plugins/SecondDimensionWatcherReDive.AI/Engines/CodexAppServerEngine.cs @@ -226,14 +226,14 @@ await rpc.SendRequestAsync( turnId = GetRequiredNestedString(turnResult, "turn", "id"); state.SetTurnId(turnId); - while (state.Updates.TryDequeue(out var bufferedUpdate)) + while (state.TryDequeueUpdate(out var bufferedUpdate)) yield return bufferedUpdate; while (!state.IsTerminal) { var message = await rpc.ReceiveAsync(cancellationToken); await ProcessTurnMessageAsync(rpc, state, message, cancellationToken); - while (state.Updates.TryDequeue(out var update)) + while (state.TryDequeueUpdate(out var update)) yield return update; } @@ -488,8 +488,8 @@ await rpc.SendErrorAsync(requestId, -32602, } var argumentsJson = arguments.GetRawText(); - state.Updates.Enqueue(new ToolCallBegin(callId, toolName)); - state.Updates.Enqueue(new ToolCallDelta(callId, argumentsJson)); + state.TryEnqueueUpdate(new ToolCallBegin(callId, toolName)); + state.TryEnqueueUpdate(new ToolCallDelta(callId, argumentsJson)); IToolResult toolResult; var executionCanceled = false; @@ -516,7 +516,7 @@ await rpc.SendErrorAsync(requestId, -32602, var serializedResult = JsonSerializer.SerializeToElement( toolResult, toolResult.GetType(), ToolJsonOptions.Options); - state.Updates.Enqueue(new ToolResultUpdate(callId, serializedResult)); + state.TryEnqueueUpdate(new ToolResultUpdate(callId, serializedResult)); if (executionCanceled || cancellationToken.IsCancellationRequested) { @@ -827,17 +827,18 @@ private sealed class TurnState( IToolExecutor? toolExecutor, int maxDynamicToolCalls) { + private const int MaxBufferedUpdates = 1024; private readonly Dictionary _finalAgentTexts = new(StringComparer.Ordinal); private readonly HashSet _completedFinalAgentItems = new(StringComparer.Ordinal); private readonly HashSet _toolCallIds = new(StringComparer.Ordinal); + private readonly Queue _updates = new(); private int _dynamicToolCallCount; public string ThreadId { get; } = threadId; public string? TurnId { get; private set; } public IToolExecutor? ToolExecutor { get; } = toolExecutor; - public Queue Updates { get; } = new(); public bool IsTerminal { get; private set; } public bool ServerTerminalReceived { get; private set; } public string? Status { get; private set; } @@ -854,7 +855,7 @@ public void AppendFinalAgentDelta(string itemId, string delta) _finalAgentTexts[itemId] = accumulated + delta; if (delta.Length > 0) - Updates.Enqueue(new TextDelta(delta)); + TryEnqueueUpdate(new TextDelta(delta)); } public void CompleteFinalAgentItem(string itemId, string authoritativeText) @@ -870,9 +871,24 @@ public void CompleteFinalAgentItem(string itemId, string authoritativeText) var suffix = authoritativeText[accumulated.Length..]; _finalAgentTexts[itemId] = authoritativeText; if (suffix.Length > 0) - Updates.Enqueue(new TextDelta(suffix)); + TryEnqueueUpdate(new TextDelta(suffix)); } + public bool TryEnqueueUpdate(IChatUpdate update) + { + if (_updates.Count < MaxBufferedUpdates) + { + _updates.Enqueue(update); + return true; + } + + Fail($"Codex app-server buffered update limit ({MaxBufferedUpdates}) was exceeded."); + return false; + } + + public bool TryDequeueUpdate(out IChatUpdate update) => + _updates.TryDequeue(out update!); + public bool TryBeginToolCall(string callId, out string rejectionReason) { if (!_toolCallIds.Add(callId)) diff --git a/Plugins/SecondDimensionWatcherReDive.Chat/ChatController.cs b/Plugins/SecondDimensionWatcherReDive.Chat/ChatController.cs index 203ffc1..7420380 100644 --- a/Plugins/SecondDimensionWatcherReDive.Chat/ChatController.cs +++ b/Plugins/SecondDimensionWatcherReDive.Chat/ChatController.cs @@ -190,7 +190,13 @@ private async IAsyncEnumerable> StreamChatEvents( string? model, [EnumeratorCancellation] CancellationToken cancellationToken) { - var channel = Channel.CreateUnbounded>(); + var channel = Channel.CreateBounded>( + new BoundedChannelOptions(256) + { + SingleReader = true, + SingleWriter = true, + FullMode = BoundedChannelFullMode.Wait + }); // Producer: runs AI chat streaming in background, writes SSE items to channel. // Keep the task and await it during iterator disposal so a disconnected request cannot diff --git a/README.md b/README.md index a55a5f4..b6e0b47 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,8 @@ - [x] 按动画分组的主页展示（卡片 + 剧集列表） - [x] 当季番组发现（mikanani.me 爬取）+ 一键订阅 - [x] 后台任务仪表盘（查看状态、手动触发） +- [x] PostgreSQL 持久任务 / Outbox（崩溃恢复、指数重试、死信处理、多实例租约） +- [x] 存活与就绪探针、Prometheus 指标及 OpenTelemetry 链路 - [x] 一次性数据迁移框架（`MigrationMarkers` 表幂等记录） - [x] 插件事件系统（下载前 / 下载完成后钩子） - [x] 多语言界面（简体中文 / English / 日本語） @@ -105,11 +107,15 @@ bash <(curl -fsSL https://raw.githubusercontent.com/HCGStudio/SecondDimensionWat | `AI:CodexAppServer:Endpoint` / `BearerToken` / `Model` / `PermissionProfile` / `TimeoutSeconds` | Codex app-server WebSocket 端点；空模型使用服务端默认模型；权限配置默认 `:read-only`，也可填写管理员定义的 profile id | | `Inference:RateLimitDelayMs` | 推断 API 调用最小间隔（毫秒，默认 1000） | | `Valkey:ConnectionString` | Valkey / Redis 连接（可选；为空则使用内存缓存） | +| `Health:ValkeyRequired` / `QbittorrentRequired` / `StorageRequired` / `AIRequired` | `/health/ready` 的依赖要求；PostgreSQL 始终必需，AI 默认不阻塞就绪 | +| `OpenTelemetry:OtlpEndpoint` | 可选 OTLP Collector 地址；Prometheus `/metrics` 无需配置即启用 | > 使用现有媒体库导入前，必须至少配置一个 `MediaLibrary:AllowedRoots`。导入源必须位于白名单内，且不能与 `FileStore:Local` 管理的下载目录相同、互为父目录或以其他方式重叠。导入与后续对账只会修改数据库中的媒体记录和虚拟路径映射；系统绝不会移动、重命名或删除原文件。短暂缺失的条目会先撤下映射并保留观看/审核记录，超过 `MissingGracePeriod`（默认 24 小时）后才清理数据库记录。 > 从 v2.2 之前升级：旧的 `Inference:ApiKey/Provider/Model` 已迁移到 `AI:` 前缀。运行 `deployments/migrate-config.sh` 自动迁移；包管理器安装时 `postinstall.sh` 会自动执行。 +运行探针、持久任务恢复、死信操作和遥测标签约束详见 [运行可靠性与可观测性](docs/runtime-reliability.md)。 + ### 网页运行时设置 登录后打开「设置」，可修改 AI 执行模式与 Provider、AI/TMDB 密钥、qBittorrent、媒体库扫描、异常检测和 NFS。保存值存入 PostgreSQL，并覆盖部署文件或环境变量中的默认值；密钥和密码使用持久化 Data Protection 密钥环加密，API 不会回显明文。可对单个敏感项选择保留、替换、清除或恢复部署默认值。 diff --git a/SecondDimensionWatcherReDive.Client/mock-server.mjs b/SecondDimensionWatcherReDive.Client/mock-server.mjs index f5ee4d4..9c7c57f 100644 --- a/SecondDimensionWatcherReDive.Client/mock-server.mjs +++ b/SecondDimensionWatcherReDive.Client/mock-server.mjs @@ -2589,10 +2589,53 @@ async function route(method, pathname, searchParams, req, res) { }, ]; + const mockDurableJobs = + globalThis._mockDurableJobs ?? + (globalThis._mockDurableJobs = [ + { + id: randomUUID(), + type: "downloadCompletion", + status: "deadLetter", + stage: "mapFiles", + attemptCount: 8, + createdAt: new Date(Date.now() - 3_600_000).toISOString(), + updatedAt: new Date(Date.now() - 60_000).toISOString(), + nextAttemptAt: new Date(Date.now() - 60_000).toISOString(), + lastAttemptAt: new Date(Date.now() - 60_000).toISOString(), + completedAt: null, + lastError: "InvalidOperationException: No file mapping could be produced.", + }, + ]); + if (method === "GET" && pathname === "/api/tasks") { return json(res, MOCK_TASKS); } + if (method === "GET" && pathname === "/api/jobs") { + const status = searchParams.get("status"); + const items = status + ? mockDurableJobs.filter( + (job) => job.status.toLowerCase() === status.toLowerCase(), + ) + : mockDurableJobs; + return json(res, { items, totalCount: items.length }); + } + + if ( + method === "POST" && + (pathname === "/api/jobs/retry" || pathname === "/api/jobs/resolve") + ) { + const body = await readBody(req); + const ids = new Set(Array.isArray(body.ids) ? body.ids : []); + let affectedCount = 0; + for (let index = mockDurableJobs.length - 1; index >= 0; index--) { + if (!ids.has(mockDurableJobs[index].id)) continue; + mockDurableJobs.splice(index, 1); + affectedCount++; + } + return json(res, { affectedCount }); + } + // POST /api/tasks/:id/run { const m = pathname.match(/^\/api\/tasks\/(.+)\/run$/); diff --git a/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/tasks.json b/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/tasks.json index b00d2b8..551e15c 100644 --- a/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/tasks.json +++ b/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/tasks.json @@ -29,6 +29,37 @@ "success": "Task \"{{name}}\" finished", "failure": "Task \"{{name}}\" failed" }, + "deadLetters": { + "title": "Failed durable jobs", + "description": "Jobs that exhausted automatic retries. Retry them after fixing the cause, or mark them handled.", + "empty": "No durable jobs require attention.", + "loadFailed": "Could not load failed durable jobs.", + "retry": "Retry", + "resolve": "Mark handled", + "columns": { + "type": "Job", + "stage": "Failed stage", + "attempts": "Attempts", + "updated": "Last attempt", + "error": "Last error", + "actions": "Actions" + }, + "types": { + "downloadCompletion": "Download completion" + }, + "stages": { + "mapFiles": "Map files", + "notify": "Notify", + "invokePlugins": "Invoke plugins", + "done": "Finalize" + }, + "toast": { + "retrySuccess": "Job queued for retry", + "retryFailure": "Could not retry the job", + "resolveSuccess": "Job marked handled", + "resolveFailure": "Could not mark the job handled" + } + }, "metadata": { "SyncFeed": { "name": "SyncFeed", diff --git a/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/tasks.json b/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/tasks.json index 1112f71..9ef237f 100644 --- a/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/tasks.json +++ b/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/tasks.json @@ -26,6 +26,37 @@ "success": "タスク「{{name}}」を完了しました", "failure": "タスク「{{name}}」が失敗しました" }, + "deadLetters": { + "title": "失敗した永続ジョブ", + "description": "自動再試行を使い切ったジョブです。原因を修正して再試行するか、処理済みにしてください。", + "empty": "対応が必要な永続ジョブはありません。", + "loadFailed": "失敗した永続ジョブを読み込めませんでした。", + "retry": "再試行", + "resolve": "処理済みにする", + "columns": { + "type": "ジョブ", + "stage": "失敗ステージ", + "attempts": "試行回数", + "updated": "最終試行", + "error": "最終エラー", + "actions": "操作" + }, + "types": { + "downloadCompletion": "ダウンロード完了処理" + }, + "stages": { + "mapFiles": "ファイルのマッピング", + "notify": "通知", + "invokePlugins": "プラグインの呼び出し", + "done": "完了" + }, + "toast": { + "retrySuccess": "ジョブを再試行キューに追加しました", + "retryFailure": "ジョブを再試行できませんでした", + "resolveSuccess": "ジョブを処理済みにしました", + "resolveFailure": "ジョブを処理済みにできませんでした" + } + }, "metadata": { "SyncFeed": { "name": "SyncFeed", diff --git a/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/tasks.json b/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/tasks.json index fad79a7..b0ab651 100644 --- a/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/tasks.json +++ b/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/tasks.json @@ -26,6 +26,37 @@ "success": "任务「{{name}}」执行完成", "failure": "任务「{{name}}」执行失败" }, + "deadLetters": { + "title": "失败的持久任务", + "description": "这些任务已耗尽自动重试次数。修复原因后可重新执行，或将其标记为已处理。", + "empty": "当前没有需要处理的持久任务。", + "loadFailed": "无法加载失败的持久任务。", + "retry": "重试", + "resolve": "标记已处理", + "columns": { + "type": "任务", + "stage": "失败阶段", + "attempts": "尝试次数", + "updated": "最后尝试", + "error": "最后错误", + "actions": "操作" + }, + "types": { + "downloadCompletion": "下载完成处理" + }, + "stages": { + "mapFiles": "建立文件映射", + "notify": "发送通知", + "invokePlugins": "调用插件", + "done": "完成" + }, + "toast": { + "retrySuccess": "任务已加入重试队列", + "retryFailure": "无法重试该任务", + "resolveSuccess": "任务已标记为处理完成", + "resolveFailure": "无法标记该任务" + } + }, "metadata": { "SyncFeed": { "name": "SyncFeed", diff --git a/SecondDimensionWatcherReDive.Client/src/pages/TasksPage.tsx b/SecondDimensionWatcherReDive.Client/src/pages/TasksPage.tsx index 8e0626b..e23ed81 100644 --- a/SecondDimensionWatcherReDive.Client/src/pages/TasksPage.tsx +++ b/SecondDimensionWatcherReDive.Client/src/pages/TasksPage.tsx @@ -1,16 +1,23 @@ -import { AlertTriangle, Loader2, Play } from "lucide-react"; import React from "react"; import { useTranslation } from "react-i18next"; +import { + AlertTriangle, + CheckCircle2, + Loader2, + Play, + RotateCcw, +} from "lucide-react"; + import { useToast } from "../components/ToastProvider"; import { Button } from "../components/ui/Button"; import { EmptyPrompt } from "../components/ui/EmptyPrompt"; import { Spinner } from "../components/ui/Spinner"; import { Table, type TableColumn } from "../components/ui/Table"; -import { useTasks } from "../tasks/hooks"; +import { useDeadLetterJobs, useTasks } from "../tasks/hooks"; import { useTaskMetadata } from "../tasks/taskMetadata"; -import { runTask } from "../tasks/utils"; -import { ITask } from "../tasks/types"; +import { IDurableJob, ITask } from "../tasks/types"; +import { resolveJobs, retryJobs, runTask } from "../tasks/utils"; import { PageTemplate } from "./PageTemplate"; function useFormatInterval(): (interval: string) => string { @@ -43,8 +50,18 @@ export const TasksPage: React.FC = () => { const getTaskMetadata = useTaskMetadata(); const formatInterval = useFormatInterval(); const { data: tasks, error, mutate } = useTasks(); + const { + data: deadLetters, + error: deadLetterError, + mutate: mutateDeadLetters, + } = useDeadLetterJobs(); const { addToast } = useToast(); - const [runningTasks, setRunningTasks] = React.useState>(new Set()); + const [runningTasks, setRunningTasks] = React.useState>( + new Set(), + ); + const [mutatingJobs, setMutatingJobs] = React.useState>( + new Set(), + ); const onRun = React.useCallback( async (id: string) => { @@ -72,6 +89,36 @@ export const TasksPage: React.FC = () => { [mutate, addToast, t, getTaskMetadata], ); + const mutateJob = React.useCallback( + async (job: IDurableJob, action: "retry" | "resolve") => { + setMutatingJobs((previous) => new Set(previous).add(job.id)); + try { + const result = + action === "retry" + ? await retryJobs([job.id]) + : await resolveJobs([job.id]); + if (result.affectedCount !== 1) throw new Error("job state changed"); + await mutateDeadLetters(); + addToast({ + title: t(`tasks:deadLetters.toast.${action}Success`), + color: "success", + }); + } catch { + addToast({ + title: t(`tasks:deadLetters.toast.${action}Failure`), + color: "danger", + }); + } finally { + setMutatingJobs((previous) => { + const next = new Set(previous); + next.delete(job.id); + return next; + }); + } + }, + [addToast, mutateDeadLetters, t], + ); + const columns: TableColumn[] = [ { name: t("tasks:columns.name"), @@ -79,7 +126,8 @@ export const TasksPage: React.FC = () => { }, { name: t("tasks:columns.description"), - render: (_value: any, item: ITask) => getTaskMetadata(item.id).description, + render: (_value: any, item: ITask) => + getTaskMetadata(item.id).description, }, { field: "interval", @@ -122,6 +170,65 @@ export const TasksPage: React.FC = () => { }, ]; + const deadLetterColumns: TableColumn[] = [ + { + field: "type", + name: t("tasks:deadLetters.columns.type"), + render: () => t("tasks:deadLetters.types.downloadCompletion"), + }, + { + field: "stage", + name: t("tasks:deadLetters.columns.stage"), + render: (value: IDurableJob["stage"]) => + t(`tasks:deadLetters.stages.${value}`), + }, + { + field: "attemptCount", + name: t("tasks:deadLetters.columns.attempts"), + }, + { + field: "updatedAt", + name: t("tasks:deadLetters.columns.updated"), + render: (value: string) => new Date(value).toLocaleString(), + }, + { + field: "lastError", + name: t("tasks:deadLetters.columns.error"), + render: (value: string | null) => value ?? "-", + truncateText: true, + }, + { + name: t("tasks:deadLetters.columns.actions"), + render: (_value: unknown, item: IDurableJob) => { + const disabled = mutatingJobs.has(item.id); + return ( +
+ + +
+ ); + }, + width: "230px", + }, + ]; + return (

@@ -145,6 +252,28 @@ export const TasksPage: React.FC = () => { body={

{t("tasks:empty.body")}

} /> )} + +

+ {t("tasks:deadLetters.title")} +

+

+ {t("tasks:deadLetters.description")} +

+ {deadLetterError ? ( +

+ {t("tasks:deadLetters.loadFailed")} +

+ ) : !deadLetters ? ( +
+ +
+ ) : deadLetters.items.length > 0 ? ( + + ) : ( +

+ {t("tasks:deadLetters.empty")} +

+ )} ); }; diff --git a/SecondDimensionWatcherReDive.Client/src/tasks/hooks.ts b/SecondDimensionWatcherReDive.Client/src/tasks/hooks.ts index cacb821..f8361d9 100644 --- a/SecondDimensionWatcherReDive.Client/src/tasks/hooks.ts +++ b/SecondDimensionWatcherReDive.Client/src/tasks/hooks.ts @@ -1,7 +1,12 @@ import useSWR from "swr"; import fetcher from "../auth/httpClient"; -import { ITask } from "./types"; +import { IDurableJobPage, ITask } from "./types"; export const useTasks = () => useSWR("/api/tasks", fetcher, { refreshInterval: 3000 }); + +export const useDeadLetterJobs = () => + useSWR("/api/jobs?status=deadLetter&take=100", fetcher, { + refreshInterval: 5000, + }); diff --git a/SecondDimensionWatcherReDive.Client/src/tasks/types.ts b/SecondDimensionWatcherReDive.Client/src/tasks/types.ts index f065b84..2f89d3f 100644 --- a/SecondDimensionWatcherReDive.Client/src/tasks/types.ts +++ b/SecondDimensionWatcherReDive.Client/src/tasks/types.ts @@ -5,3 +5,22 @@ export interface ITask { lastRunAt: string | null; isRunning: boolean; } + +export interface IDurableJob { + id: string; + type: "downloadCompletion"; + status: "deadLetter"; + stage: "mapFiles" | "notify" | "invokePlugins" | "done"; + attemptCount: number; + createdAt: string; + updatedAt: string; + nextAttemptAt: string; + lastAttemptAt: string | null; + completedAt: string | null; + lastError: string | null; +} + +export interface IDurableJobPage { + items: IDurableJob[]; + totalCount: number; +} diff --git a/SecondDimensionWatcherReDive.Client/src/tasks/utils.ts b/SecondDimensionWatcherReDive.Client/src/tasks/utils.ts index 0c18130..aa4ec30 100644 --- a/SecondDimensionWatcherReDive.Client/src/tasks/utils.ts +++ b/SecondDimensionWatcherReDive.Client/src/tasks/utils.ts @@ -3,3 +3,14 @@ import fetcher from "../auth/httpClient"; export const runTask = async (id: string) => { return await fetcher(`/api/tasks/${id}/run`, { method: "POST" }); }; + +const mutateJobs = async (action: "retry" | "resolve", ids: string[]) => + await fetcher<{ affectedCount: number }>(`/api/jobs/${action}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ ids }), + }); + +export const retryJobs = async (ids: string[]) => mutateJobs("retry", ids); + +export const resolveJobs = async (ids: string[]) => mutateJobs("resolve", ids); diff --git a/SecondDimensionWatcherReDive.Framework/DataRepository/IDurableJobRepository.cs b/SecondDimensionWatcherReDive.Framework/DataRepository/IDurableJobRepository.cs new file mode 100644 index 0000000..7a496f0 --- /dev/null +++ b/SecondDimensionWatcherReDive.Framework/DataRepository/IDurableJobRepository.cs @@ -0,0 +1,110 @@ +namespace SecondDimensionWatcherReDive.Framework.DataRepository; + +public enum DurableJobType +{ + DownloadCompletion +} + +public enum DurableJobStatus +{ + Pending, + Processing, + Completed, + DeadLetter, + Resolved +} + +public enum DurableJobStage +{ + MapFiles, + Notify, + InvokePlugins, + Done +} + +public sealed record DurableJob( + Guid Id, + string DeduplicationKey, + DurableJobType Type, + DurableJobStatus Status, + DurableJobStage Stage, + string PayloadJson, + int AttemptCount, + DateTimeOffset CreatedAt, + DateTimeOffset UpdatedAt, + DateTimeOffset NextAttemptAt, + DateTimeOffset? LastAttemptAt, + DateTimeOffset? CompletedAt, + string? LeaseOwner, + DateTimeOffset? LeaseExpiresAt, + string? LastError); + +public sealed record DurableJobPage( + IReadOnlyList Items, + int TotalCount); + +public sealed record DurableJobStatistics( + int PendingCount, + int ProcessingCount, + int DeadLetterCount, + double OldestPendingAgeSeconds); + +public sealed record DownloadCompletionJobPayload( + Guid ItemId, + string StorePath, + string FileStore, + Guid? DownloadAttemptId); + +public interface IDurableJobRepository +{ + Task> ClaimDueAsync( + string workerId, + DateTimeOffset now, + DateTimeOffset leaseUntil, + int take, + CancellationToken cancellationToken); + + Task AdvanceStageAsync( + Guid id, + string workerId, + DurableJobStage expectedStage, + DurableJobStage nextStage, + DateTimeOffset now, + CancellationToken cancellationToken); + + Task RenewLeaseAsync( + Guid id, + string workerId, + DateTimeOffset now, + DateTimeOffset leaseUntil, + CancellationToken cancellationToken); + + Task MarkFailedAsync( + Guid id, + string workerId, + int attemptCount, + DateTimeOffset attemptedAt, + DateTimeOffset? nextAttemptAt, + string error, + CancellationToken cancellationToken); + + Task GetPageAsync( + DurableJobStatus? status, + int skip, + int take, + CancellationToken cancellationToken); + + Task RetryAsync( + IReadOnlyCollection ids, + DateTimeOffset now, + CancellationToken cancellationToken); + + Task ResolveAsync( + IReadOnlyCollection ids, + DateTimeOffset now, + CancellationToken cancellationToken); + + Task GetStatisticsAsync( + DateTimeOffset now, + CancellationToken cancellationToken); +} diff --git a/SecondDimensionWatcherReDive.Framework/DataRepository/IReadinessRepository.cs b/SecondDimensionWatcherReDive.Framework/DataRepository/IReadinessRepository.cs new file mode 100644 index 0000000..b65eba4 --- /dev/null +++ b/SecondDimensionWatcherReDive.Framework/DataRepository/IReadinessRepository.cs @@ -0,0 +1,6 @@ +namespace SecondDimensionWatcherReDive.Framework.DataRepository; + +public interface IReadinessRepository +{ + Task CanConnectAsync(CancellationToken cancellationToken); +} diff --git a/SecondDimensionWatcherReDive.Framework/DataRepository/IScheduledTaskLeaseRepository.cs b/SecondDimensionWatcherReDive.Framework/DataRepository/IScheduledTaskLeaseRepository.cs new file mode 100644 index 0000000..34b6424 --- /dev/null +++ b/SecondDimensionWatcherReDive.Framework/DataRepository/IScheduledTaskLeaseRepository.cs @@ -0,0 +1,28 @@ +namespace SecondDimensionWatcherReDive.Framework.DataRepository; + +public interface IScheduledTaskLeaseRepository +{ + Task TryAcquireAsync( + string taskId, + string ownerId, + DateTimeOffset now, + DateTimeOffset leaseUntil, + bool force, + CancellationToken cancellationToken); + + Task RenewAsync( + string taskId, + string ownerId, + DateTimeOffset now, + DateTimeOffset leaseUntil, + CancellationToken cancellationToken); + + Task CompleteAsync( + string taskId, + string ownerId, + DateTimeOffset completedAt, + DateTimeOffset leaseUntil, + bool succeeded, + string? error, + CancellationToken cancellationToken); +} diff --git a/SecondDimensionWatcherReDive.Framework/Notifications/IDownloadCompletionNotifier.cs b/SecondDimensionWatcherReDive.Framework/Notifications/IDownloadCompletionNotifier.cs new file mode 100644 index 0000000..94d95ef --- /dev/null +++ b/SecondDimensionWatcherReDive.Framework/Notifications/IDownloadCompletionNotifier.cs @@ -0,0 +1,15 @@ +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Framework.Notifications; + +/// +/// Optional notification effect in the durable download-completion workflow. +/// Implementations must treat as an idempotency key. +/// +public interface IDownloadCompletionNotifier +{ + Task NotifyAsync( + Guid eventId, + DownloadCompletionJobPayload payload, + CancellationToken cancellationToken); +} diff --git a/SecondDimensionWatcherReDive.Framework/PluginParams/FileDownloadCompleteParam.cs b/SecondDimensionWatcherReDive.Framework/PluginParams/FileDownloadCompleteParam.cs index c0187bd..e0adc16 100644 --- a/SecondDimensionWatcherReDive.Framework/PluginParams/FileDownloadCompleteParam.cs +++ b/SecondDimensionWatcherReDive.Framework/PluginParams/FileDownloadCompleteParam.cs @@ -1,8 +1,18 @@ namespace SecondDimensionWatcherReDive.Framework.PluginParams; -public class FileDownloadCompleteParam(Guid itemId, string storePath, string fileStore) +public class FileDownloadCompleteParam( + Guid itemId, + string storePath, + string fileStore, + Guid? eventId = null) { + /// + /// Stable identifier for this completion workflow. Plugin handlers can persist + /// it as an idempotency key before performing externally visible work. + /// + public Guid EventId { get; } = eventId ?? itemId; + public Guid ItemId { get; set; } = itemId; public string StorePath { get; set; } = storePath; public string FileStore { get; set; } = fileStore; -} \ No newline at end of file +} diff --git a/SecondDimensionWatcherReDive.Framework/Tasks/IScheduledTaskLeaseManager.cs b/SecondDimensionWatcherReDive.Framework/Tasks/IScheduledTaskLeaseManager.cs new file mode 100644 index 0000000..2a0f1bd --- /dev/null +++ b/SecondDimensionWatcherReDive.Framework/Tasks/IScheduledTaskLeaseManager.cs @@ -0,0 +1,23 @@ +namespace SecondDimensionWatcherReDive.Framework.Tasks; + +public sealed class ScheduledTaskLeaseUnavailableException(Exception innerException) + : Exception("The scheduled-task lease store is unavailable.", innerException); + +public interface IScheduledTaskExecutionLease : IAsyncDisposable +{ + CancellationToken LeaseLostToken { get; } + + Task CompleteAsync( + bool succeeded, + string? error, + CancellationToken cancellationToken); +} + +public interface IScheduledTaskLeaseManager +{ + Task TryAcquireAsync( + string taskId, + TimeSpan interval, + bool force, + CancellationToken cancellationToken); +} diff --git a/SecondDimensionWatcherReDive.Framework/Tasks/ScheduledTaskBase.cs b/SecondDimensionWatcherReDive.Framework/Tasks/ScheduledTaskBase.cs index 7a9a2e0..fa025c7 100644 --- a/SecondDimensionWatcherReDive.Framework/Tasks/ScheduledTaskBase.cs +++ b/SecondDimensionWatcherReDive.Framework/Tasks/ScheduledTaskBase.cs @@ -4,10 +4,16 @@ namespace SecondDimensionWatcherReDive.Framework.Tasks; public abstract class ScheduledTaskBase : IScheduledTask { - private readonly Channel _runQueue = - Channel.CreateUnbounded( - new UnboundedChannelOptions { SingleReader = true }); - + private readonly Channel _runQueue = Channel.CreateBounded( + new BoundedChannelOptions(1) + { + SingleReader = true, + SingleWriter = false, + FullMode = BoundedChannelFullMode.DropWrite + }); + private readonly object _sync = new(); + private TaskCompletionSource? _pendingRun; + private bool _pendingForce; private volatile bool _isRunning; private DateTimeOffset? _lastRunAt; @@ -19,55 +25,157 @@ public abstract class ScheduledTaskBase : IScheduledTask public async Task RunNowAsync(CancellationToken cancellationToken) { - var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - - await using var registration = cancellationToken.Register( - () => tcs.TrySetCanceled(cancellationToken)); - - await _runQueue.Writer.WriteAsync(tcs, cancellationToken); - await tcs.Task; + var completion = QueueRun(force: true); + // Cancelling one HTTP request must not cancel the shared execution that + // other callers and the periodic scheduler are awaiting. + await completion.WaitAsync(cancellationToken); } /// - /// Enqueues a run request without waiting for completion. + /// Runs a periodic signal and reports whether this instance acquired the + /// distributed lease. Hosting services use the result to poll quickly + /// while another instance owns an unfinished run. /// - public void Enqueue() - { - var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - _runQueue.Writer.TryWrite(tcs); - } + public Task RunScheduledAsync(CancellationToken cancellationToken) => + QueueRun(force: false).WaitAsync(cancellationToken); /// - /// Sequentially processes queued run requests. Called by the hosting - /// BackgroundService; runs for the lifetime of the host. + /// Coalesces a run request without waiting for completion. At most one + /// pending signal exists while the current execution is in flight. /// - public async Task ProcessQueueAsync(CancellationToken cancellationToken) + public void Enqueue() => QueueRun(force: true); + + /// + /// Sequentially processes coalesced run requests. A PostgreSQL lease + /// ensures only one application instance executes a task at a time. + /// + public async Task ProcessQueueAsync( + IScheduledTaskLeaseManager leaseManager, + CancellationToken cancellationToken) { - await foreach (var tcs in _runQueue.Reader.ReadAllAsync(cancellationToken)) + await foreach (var _ in _runQueue.Reader.ReadAllAsync(cancellationToken)) { - if (tcs.Task.IsCanceled) continue; + TaskCompletionSource? completion; + bool force; + lock (_sync) + { + completion = _pendingRun; + force = _pendingForce; + } + if (completion is null) continue; - _isRunning = true; + IScheduledTaskExecutionLease? lease; try { - await ExecuteTaskAsync(cancellationToken); - _lastRunAt = DateTimeOffset.UtcNow; - tcs.TrySetResult(); + lease = await leaseManager.TryAcquireAsync( + Id, + Interval, + force, + cancellationToken); } - catch (OperationCanceledException ex) + catch (OperationCanceledException exception) when (cancellationToken.IsCancellationRequested) { - tcs.TrySetCanceled(ex.CancellationToken); + completion.TrySetCanceled(exception.CancellationToken); + FinishRun(completion); + throw; } - catch (Exception ex) + catch (Exception exception) { - tcs.TrySetException(ex); + completion.TrySetException( + new ScheduledTaskLeaseUnavailableException(exception)); + FinishRun(completion); + continue; } - finally + + if (lease is null) + { + // Another instance owns the same periodic task. Its local timer + // will drive the execution; this duplicate signal is complete. + completion.TrySetResult(false); + FinishRun(completion); + continue; + } + + await using (lease) { - _isRunning = false; + using var executionCancellation = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + lease.LeaseLostToken); + _isRunning = true; + try + { + await ExecuteTaskAsync(executionCancellation.Token); + _lastRunAt = DateTimeOffset.UtcNow; + await lease.CompleteAsync(true, null, cancellationToken); + completion.TrySetResult(true); + } + catch (OperationCanceledException exception) + { + // On host shutdown or lease loss, leave the lease to expire so + // another instance can resume without an overlapping run. + completion.TrySetCanceled(exception.CancellationToken); + if (cancellationToken.IsCancellationRequested) + throw; + } + catch (Exception exception) + { + try + { + await lease.CompleteAsync( + false, + exception.GetType().Name, + cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception leaseException) + { + completion.TrySetException( + new ScheduledTaskLeaseUnavailableException(leaseException)); + continue; + } + completion.TrySetException(exception); + } + finally + { + _isRunning = false; + FinishRun(completion); + } } } } protected abstract Task ExecuteTaskAsync(CancellationToken cancellationToken); + + private Task QueueRun(bool force) + { + lock (_sync) + { + if (_pendingRun is { Task.IsCompleted: false }) + { + _pendingForce |= force; + return _pendingRun.Task; + } + + _pendingForce = force; + _pendingRun = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + _runQueue.Writer.TryWrite(0); + return _pendingRun.Task; + } + } + + private void FinishRun(TaskCompletionSource completion) + { + lock (_sync) + { + if (ReferenceEquals(_pendingRun, completion)) + { + _pendingRun = null; + _pendingForce = false; + } + } + } } diff --git a/SecondDimensionWatcherReDive.IntegrationTest/Health/HealthEndpointTests.cs b/SecondDimensionWatcherReDive.IntegrationTest/Health/HealthEndpointTests.cs new file mode 100644 index 0000000..5297e13 --- /dev/null +++ b/SecondDimensionWatcherReDive.IntegrationTest/Health/HealthEndpointTests.cs @@ -0,0 +1,47 @@ +using System.Net; +using System.Text.Json; + +namespace SecondDimensionWatcherReDive.IntegrationTest.Health; + +[TestClass] +public sealed class HealthEndpointTests +{ + private WebDavWebApplicationFactory _factory = null!; + + [TestInitialize] + public void Setup() => _factory = new WebDavWebApplicationFactory(); + + [TestCleanup] + public void Cleanup() => _factory.Dispose(); + + [TestMethod] + public async Task Liveness_HasNoExternalChecks_WhenReadinessDependencyFails() + { + using var client = _factory.CreateUnauthenticatedClient(); + + using var live = await client.GetAsync("/health/live"); + using var ready = await client.GetAsync("/health/ready"); + + Assert.AreEqual(HttpStatusCode.OK, live.StatusCode); + using var liveBody = await JsonDocument.ParseAsync( + await live.Content.ReadAsStreamAsync()); + Assert.AreEqual(0, liveBody.RootElement.GetProperty("checks").EnumerateObject().Count()); + Assert.AreEqual(HttpStatusCode.ServiceUnavailable, ready.StatusCode); + } + + [TestMethod] + public async Task Metrics_ArePublicAndDoNotExposeRawPathLabels() + { + using var client = _factory.CreateUnauthenticatedClient(); + using var _ = await client.GetAsync("/health/live?private=value"); + + using var response = await client.GetAsync("/metrics"); + var body = await response.Content.ReadAsStringAsync(); + + Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); + StringAssert.Contains(body, "target_info"); + Assert.IsFalse(body.Contains("url.path", StringComparison.OrdinalIgnoreCase)); + Assert.IsFalse(body.Contains("url_path", StringComparison.OrdinalIgnoreCase)); + Assert.IsFalse(body.Contains("private=value", StringComparison.Ordinal)); + } +} diff --git a/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/FileMappingRepositoryPostgreSqlTests.cs b/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/FileMappingRepositoryPostgreSqlTests.cs index 1a06401..3e53a88 100644 --- a/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/FileMappingRepositoryPostgreSqlTests.cs +++ b/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/FileMappingRepositoryPostgreSqlTests.cs @@ -1,4 +1,5 @@ using Microsoft.EntityFrameworkCore; +using System.Text.Json; using Testcontainers.PostgreSql; using SecondDimensionWatcherReDive.Framework.DataRepository; using SecondDimensionWatcherReDive.Repositories; @@ -88,6 +89,198 @@ static async Task WriteAsync(FileMappingRepositoryPostgreSqlTestFixture fi CollectionAssert.AreEquivalent(new long[] { 0, 1 }, versions); } + [TestMethod] + public async Task DownloadCompletion_CommitsStateAndOneDurableJobTogether() + { + var seed = await Fixture.SeedTrackedAnimationAsync(CancellationToken.None); + const string StorePath = "/store/completed"; + + await Fixture.CompleteTrackedAnimationAsync( + seed.ItemId, seed.AttemptId, StorePath, CancellationToken.None); + await Fixture.CompleteTrackedAnimationAsync( + seed.ItemId, seed.AttemptId, StorePath, CancellationToken.None); + + var state = await Fixture.GetCompletionStateAsync( + seed.ItemId, CancellationToken.None); + Assert.IsTrue(state.IsFinished); + Assert.AreEqual(1, state.JobCount); + Assert.AreEqual(seed.ItemId, state.Payload.ItemId); + Assert.AreEqual(seed.AttemptId, state.Payload.DownloadAttemptId); + Assert.AreEqual(StorePath, state.Payload.StorePath); + } + + [TestMethod] + public async Task DurableJobClaim_AllowsOnlyOneWorker() + { + var now = DateTimeOffset.UtcNow; + var job = Job(DurableJobStatus.Pending, now); + await Fixture.SeedDurableJobAsync(job, CancellationToken.None); + + var claims = await Task.WhenAll( + Fixture.ClaimDueJobsAsync("worker-a", now, CancellationToken.None), + Fixture.ClaimDueJobsAsync("worker-b", now, CancellationToken.None)); + + Assert.AreEqual(1, claims.Sum(claim => claim.Count)); + Assert.AreEqual(job.Id, claims.SelectMany(claim => claim).Single().Id); + } + + [TestMethod] + public async Task DurableJobLease_RenewalProtectsSlowConsumerAndStillExpires() + { + var now = DateTimeOffset.UtcNow; + var job = Job(DurableJobStatus.Pending, now); + await Fixture.SeedDurableJobAsync(job, CancellationToken.None); + Assert.HasCount(1, await Fixture.ClaimDueJobsAsync( + "worker-a", now, CancellationToken.None)); + Assert.IsTrue(await Fixture.RenewDurableJobLeaseAsync( + job.Id, + "worker-a", + now.AddSeconds(30), + now.AddMinutes(2), + CancellationToken.None)); + + Assert.IsEmpty(await Fixture.ClaimDueJobsAsync( + "worker-b", now.AddSeconds(61), CancellationToken.None)); + Assert.HasCount(1, await Fixture.ClaimDueJobsAsync( + "worker-b", now.AddSeconds(121), CancellationToken.None)); + } + + [TestMethod] + public async Task DurableJobLease_ExpiredOwnerCannotAdvanceStage() + { + var now = DateTimeOffset.UtcNow; + var job = Job(DurableJobStatus.Pending, now); + await Fixture.SeedDurableJobAsync(job, CancellationToken.None); + Assert.HasCount(1, await Fixture.ClaimDueJobsAsync( + "worker-a", now, CancellationToken.None)); + + Assert.IsFalse(await Fixture.AdvanceDurableJobAsync( + job.Id, + "worker-a", + DurableJobStage.MapFiles, + DurableJobStage.Notify, + now.AddMinutes(2), + CancellationToken.None)); + } + + [TestMethod] + public async Task ScheduledTaskLease_HasSingleOwnerAndExpiresForTakeover() + { + var now = DateTimeOffset.UtcNow; + var first = await Fixture.TryAcquireTaskLeaseAsync( + "SyncFeed", "instance-a", now, now.AddSeconds(30), false, CancellationToken.None); + var overlapping = await Fixture.TryAcquireTaskLeaseAsync( + "SyncFeed", "instance-b", now.AddSeconds(1), now.AddSeconds(31), false, CancellationToken.None); + var takeover = await Fixture.TryAcquireTaskLeaseAsync( + "SyncFeed", "instance-b", now.AddSeconds(31), now.AddSeconds(61), false, CancellationToken.None); + + Assert.IsTrue(first); + Assert.IsFalse(overlapping); + Assert.IsTrue(takeover); + } + + [TestMethod] + public async Task ScheduledTaskLease_NormalCompletionPreventsDuplicateUntilNextDueTime() + { + var now = DateTimeOffset.UtcNow; + Assert.IsTrue(await Fixture.TryAcquireTaskLeaseAsync( + "ScrapeSeasonBangumi", + "instance-a", + now, + now.AddSeconds(30), + false, + CancellationToken.None)); + await Fixture.CompleteTaskLeaseAsync( + "ScrapeSeasonBangumi", + "instance-a", + now.AddSeconds(5), + now.AddMinutes(10), + CancellationToken.None); + + Assert.IsFalse(await Fixture.TryAcquireTaskLeaseAsync( + "ScrapeSeasonBangumi", + "instance-b", + now.AddSeconds(31), + now.AddMinutes(1), + false, + CancellationToken.None)); + Assert.IsTrue(await Fixture.TryAcquireTaskLeaseAsync( + "ScrapeSeasonBangumi", + "instance-b", + now.AddMinutes(11), + now.AddMinutes(12), + false, + CancellationToken.None)); + } + + [TestMethod] + public async Task ScheduledTaskLease_ManualRunCanOverrideCompletedCooldown() + { + var now = DateTimeOffset.UtcNow; + Assert.IsTrue(await Fixture.TryAcquireTaskLeaseAsync( + "InferAnimationMetadata", + "instance-a", + now, + now.AddSeconds(30), + false, + CancellationToken.None)); + await Fixture.CompleteTaskLeaseAsync( + "InferAnimationMetadata", + "instance-a", + now.AddSeconds(5), + now.AddMinutes(30), + CancellationToken.None); + + Assert.IsTrue(await Fixture.TryAcquireTaskLeaseAsync( + "InferAnimationMetadata", + "instance-b", + now.AddSeconds(31), + now.AddMinutes(1), + true, + CancellationToken.None)); + } + + [TestMethod] + public async Task DeadLetterJobs_CanBeRetriedOrMarkedHandled() + { + var now = DateTimeOffset.UtcNow; + var retried = Job(DurableJobStatus.DeadLetter, now); + var resolved = Job(DurableJobStatus.DeadLetter, now); + await Fixture.SeedDurableJobAsync(retried, CancellationToken.None); + await Fixture.SeedDurableJobAsync(resolved, CancellationToken.None); + + Assert.AreEqual(1, await Fixture.RetryJobsAsync( + [retried.Id], now, CancellationToken.None)); + Assert.AreEqual(1, await Fixture.ResolveJobsAsync( + [resolved.Id], now, CancellationToken.None)); + Assert.AreEqual(DurableJobStatus.Pending, await Fixture.GetJobStatusAsync( + retried.Id, CancellationToken.None)); + Assert.AreEqual(DurableJobStatus.Resolved, await Fixture.GetJobStatusAsync( + resolved.Id, CancellationToken.None)); + } + private static FileMapping Mapping(Guid animationInfoId, string virtualPath) => new(Guid.NewGuid(), animationInfoId, virtualPath, "/physical/" + Guid.NewGuid(), "local"); + + private static DurableJob Job(DurableJobStatus status, DateTimeOffset now) + { + var id = Guid.NewGuid(); + return new DurableJob( + id, + $"test:{id:N}", + DurableJobType.DownloadCompletion, + status, + DurableJobStage.MapFiles, + JsonSerializer.Serialize(new DownloadCompletionJobPayload( + Guid.NewGuid(), "/store", "local", Guid.NewGuid())), + status == DurableJobStatus.DeadLetter ? 8 : 0, + now, + now, + now, + now, + null, + null, + null, + status == DurableJobStatus.DeadLetter ? "failed" : null); + } } diff --git a/SecondDimensionWatcherReDive.IntegrationTest/WebDavWebApplicationFactory.cs b/SecondDimensionWatcherReDive.IntegrationTest/WebDavWebApplicationFactory.cs index ae5cf79..00dde62 100644 --- a/SecondDimensionWatcherReDive.IntegrationTest/WebDavWebApplicationFactory.cs +++ b/SecondDimensionWatcherReDive.IntegrationTest/WebDavWebApplicationFactory.cs @@ -122,6 +122,7 @@ protected override void ConfigureWebHost(IWebHostBuilder builder) services.RemoveAll(); services.RemoveAll(); services.RemoveAll(); + services.RemoveAll(); services.AddSingleton(FileStoreMock.Object); services.AddSingleton(FileStoreProviderMock.Object); @@ -130,6 +131,7 @@ protected override void ConfigureWebHost(IWebHostBuilder builder) services.AddSingleton(_ => new FakeWebDavTokenRepository(TestUserName, BCrypt.Net.BCrypt.HashPassword(TestPassword))); services.AddSingleton(); + services.AddSingleton(); }); } @@ -190,6 +192,12 @@ public string GenerateScript( public bool HasPendingModelChanges() => false; } + private sealed class UnavailableReadinessRepository : IReadinessRepository + { + public Task CanConnectAsync(CancellationToken cancellationToken) => + Task.FromResult(false); + } + private sealed class FakeApplicationSettingsRepository : IApplicationSettingsRepository { private readonly object _gate = new(); diff --git a/SecondDimensionWatcherReDive.Test/CodexAppServerEngineTests.cs b/SecondDimensionWatcherReDive.Test/CodexAppServerEngineTests.cs index 887fe92..7e146e1 100644 --- a/SecondDimensionWatcherReDive.Test/CodexAppServerEngineTests.cs +++ b/SecondDimensionWatcherReDive.Test/CodexAppServerEngineTests.cs @@ -253,6 +253,32 @@ [new UserMessage("use tools")], Assert.IsNotNull(transport.SingleSent("turn/interrupt")); } + [TestMethod] + public async Task ChatAsync_BufferedUpdateOverflowFailsClosed() + { + var messages = new List + { + Response(1, "{}"), + Response(2, PermissionProfiles()), + Response(3, SafeThread()), + """{"method":"item/started","params":{"threadId":"thread-1","turnId":"turn-1","item":{"id":"message-1","type":"agentMessage","phase":"final_answer","text":""}}}""" + }; + messages.AddRange(Enumerable.Range(0, 1025).Select(_ => + """{"method":"item/agentMessage/delta","params":{"threadId":"thread-1","turnId":"turn-1","itemId":"message-1","delta":"x"}}""")); + messages.Add(Response(4, """{"turn":{"id":"turn-1"}}""")); + var transport = new ScriptedTransport(messages.ToArray()); + var (engine, _) = CreateEngine(transport); + + var exception = await Assert.ThrowsExactlyAsync(() => + CollectAsync(engine.ChatAsync( + [new UserMessage("overflow")], + null, + CancellationToken.None))); + + StringAssert.Contains(exception.Message, "buffered update limit (1024)"); + Assert.IsNotNull(transport.SingleSent("turn/interrupt")); + } + [TestMethod] public async Task GetAvailableModelsAsync_FollowsModelListPagination() { diff --git a/SecondDimensionWatcherReDive.Test/CompleteDownloadBackgroundServiceTests.cs b/SecondDimensionWatcherReDive.Test/CompleteDownloadBackgroundServiceTests.cs index 7c402db..1f5ebe2 100644 --- a/SecondDimensionWatcherReDive.Test/CompleteDownloadBackgroundServiceTests.cs +++ b/SecondDimensionWatcherReDive.Test/CompleteDownloadBackgroundServiceTests.cs @@ -1,14 +1,15 @@ +using System.Text.Json; using System.Threading.Channels; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Moq; using SecondDimensionWatcherReDive.Data; using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Framework.Notifications; using SecondDimensionWatcherReDive.Framework.PluginParams; using SecondDimensionWatcherReDive.Plugin; using SecondDimensionWatcherReDive.Services; using SecondDimensionWatcherReDive.Utils.FileStore; -using SecondDimensionWatcherReDive.Utils.Incidents; namespace SecondDimensionWatcherReDive.Test; @@ -16,120 +17,261 @@ namespace SecondDimensionWatcherReDive.Test; public sealed class CompleteDownloadBackgroundServiceTests { [TestMethod] - public async Task ProcessRequestAsync_CancelledDownload_IgnoresLateCompletion() + public async Task ProcessClaimedJobAsync_ResumesAtPersistedStage() { - var request = new DownloadCompleteRequest( - Guid.NewGuid(), - "/downloads/item", - "local", - Guid.NewGuid()); - var repository = new Mock(); - repository.Setup(candidate => candidate.TryCompleteDownloadAsync( - request.ItemId, - request.DownloadAttemptId, - request.FileStore, - request.StorePath, + var job = CreateJob(DurableJobStage.Notify); + var payload = JsonSerializer.Deserialize(job.PayloadJson)!; + var repository = new Mock(); + repository.Setup(candidate => candidate.AdvanceStageAsync( + job.Id, + It.IsAny(), + It.IsAny(), + It.IsAny(), It.IsAny(), - CancellationToken.None)) - .ReturnsAsync((AnimationInfo?)null); + It.IsAny())) + .ReturnsAsync(true); var mapper = new Mock(); + var notifier = new Mock(); var plugin = new Mock>(); - var reporter = new Mock(); - using var provider = CreateProvider(repository.Object, mapper.Object, plugin.Object); - var service = new CompleteDownloadBackgroundService( - Channel.CreateUnbounded(), - provider.GetRequiredService(), - Mock.Of>(), - reporter.Object); + using var provider = CreateProvider( + repository.Object, + mapper.Object, + notifier.Object, + plugin.Object); + var service = CreateService(provider); - await service.ProcessRequestAsync(request, CancellationToken.None); + await service.ProcessClaimedJobAsync( + provider, + repository.Object, + job, + CancellationToken.None); mapper.Verify(candidate => candidate.MapDownloadAsync( It.IsAny(), It.IsAny()), Times.Never); + notifier.Verify(candidate => candidate.NotifyAsync( + job.Id, + It.Is(value => value == payload), + It.IsAny()), Times.Once); plugin.Verify(candidate => candidate.InvokeAsync( - It.IsAny(), It.IsAny()), Times.Never); - reporter.VerifyNoOtherCalls(); + It.Is(value => + value.EventId == job.Id + && value.ItemId == payload.ItemId + && value.StorePath == payload.StorePath + && value.FileStore == payload.FileStore), + It.IsAny()), Times.Once); + repository.Verify(candidate => candidate.AdvanceStageAsync( + job.Id, + It.IsAny(), + DurableJobStage.Notify, + DurableJobStage.InvokePlugins, + It.IsAny(), + It.IsAny()), Times.Once); + repository.Verify(candidate => candidate.AdvanceStageAsync( + job.Id, + It.IsAny(), + DurableJobStage.InvokePlugins, + DurableJobStage.Done, + It.IsAny(), + It.IsAny()), Times.Once); } [TestMethod] - public async Task ProcessRequestAsync_TrackedDownload_CompletesBeforeMappingAndPlugin() + public async Task ProcessClaimedJobAsync_PluginStageDoesNotReplayPriorEffects() { - var request = new DownloadCompleteRequest( - Guid.NewGuid(), - "/downloads/item", - "local", - Guid.NewGuid()); - var info = CreateInfo(request.ItemId); - var repository = new Mock(); - repository.Setup(candidate => candidate.TryCompleteDownloadAsync( - request.ItemId, - request.DownloadAttemptId, - request.FileStore, - request.StorePath, + var job = CreateJob(DurableJobStage.InvokePlugins); + var repository = new Mock(); + repository.Setup(candidate => candidate.AdvanceStageAsync( + job.Id, + It.IsAny(), + DurableJobStage.InvokePlugins, + DurableJobStage.Done, It.IsAny(), - CancellationToken.None)) - .ReturnsAsync(info); + It.IsAny())) + .ReturnsAsync(true); + var mapper = new Mock(); + var notifier = new Mock(); + var plugin = new Mock>(); + using var provider = CreateProvider( + repository.Object, + mapper.Object, + notifier.Object, + plugin.Object); + + await CreateService(provider).ProcessClaimedJobAsync( + provider, + repository.Object, + job, + CancellationToken.None); + + mapper.VerifyNoOtherCalls(); + notifier.VerifyNoOtherCalls(); + plugin.Verify(candidate => candidate.InvokeAsync( + It.Is(value => value.EventId == job.Id), + It.IsAny()), Times.Once); + } + + [TestMethod] + public async Task ProcessClaimedJobAsync_FailureSchedulesExponentialRetry() + { + var job = CreateJob(DurableJobStage.MapFiles, attemptCount: 2); + var repository = new Mock(); var mapper = new Mock(); mapper.Setup(candidate => candidate.MapDownloadAsync( - request.ItemId, - CancellationToken.None)) - .ReturnsAsync(true); + It.IsAny(), It.IsAny())) + .ReturnsAsync(false); + using var provider = CreateProvider( + repository.Object, + mapper.Object, + Mock.Of(), + Mock.Of>()); + var service = CreateService(provider); + + await service.ProcessClaimedJobAsync( + provider, + repository.Object, + job, + CancellationToken.None); + + repository.Verify(candidate => candidate.MarkFailedAsync( + job.Id, + It.IsAny(), + 3, + It.IsAny(), + It.Is(retry => retry.HasValue), + It.Is(error => error.Contains("InvalidOperationException")), + CancellationToken.None), Times.Once); + Assert.AreEqual(TimeSpan.FromSeconds(20), + CompleteDownloadBackgroundService.RetryDelay(3)); + } + + [TestMethod] + public async Task ProcessClaimedJobAsync_LastFailureEntersDeadLetter() + { + var job = CreateJob( + DurableJobStage.InvokePlugins, + CompleteDownloadBackgroundService.MaxAttempts - 1); + var repository = new Mock(); var plugin = new Mock>(); plugin.Setup(candidate => candidate.InvokeAsync( It.IsAny(), - CancellationToken.None)) - .Returns(Task.CompletedTask); - using var provider = CreateProvider(repository.Object, mapper.Object, plugin.Object); + It.IsAny())) + .ThrowsAsync(new InvalidOperationException("plugin unavailable")); + using var provider = CreateProvider( + repository.Object, + Mock.Of(), + Mock.Of(), + plugin.Object); + var service = CreateService(provider); + + await service.ProcessClaimedJobAsync( + provider, + repository.Object, + job, + CancellationToken.None); + + repository.Verify(candidate => candidate.MarkFailedAsync( + job.Id, + It.IsAny(), + CompleteDownloadBackgroundService.MaxAttempts, + It.IsAny(), + null, + It.IsAny(), + CancellationToken.None), Times.Once); + } + + [TestMethod] + public async Task ExecuteAsync_TemporaryRepositoryFailureDoesNotStopWorker() + { + var repository = new Mock(); + repository.SetupSequence(candidate => candidate.ClaimDueAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .ThrowsAsync(new InvalidOperationException("database unavailable")) + .ReturnsAsync([]); + using var provider = CreateProvider( + repository.Object, + Mock.Of(), + Mock.Of(), + Mock.Of>()); + var channel = Channel.CreateBounded(1); var service = new CompleteDownloadBackgroundService( - Channel.CreateUnbounded(), + channel, provider.GetRequiredService(), - Mock.Of>(), - Mock.Of()); + Mock.Of>()); - await service.ProcessRequestAsync(request, CancellationToken.None); + await service.StartAsync(CancellationToken.None); + channel.Writer.TryWrite(new DownloadCompleteRequest( + Guid.NewGuid(), "/store", "local", Guid.NewGuid())); + await WaitUntilAsync( + () => repository.Invocations.Count(invocation => + invocation.Method.Name == nameof(IDurableJobRepository.ClaimDueAsync)) >= 2, + TimeSpan.FromSeconds(2)); + await service.StopAsync(CancellationToken.None); - mapper.Verify(candidate => candidate.MapDownloadAsync( - request.ItemId, CancellationToken.None), Times.Once); - plugin.Verify(candidate => candidate.InvokeAsync( - It.Is(parameter => - parameter.ItemId == request.ItemId - && parameter.StorePath == request.StorePath - && parameter.FileStore == request.FileStore), - CancellationToken.None), Times.Once); + repository.Verify(candidate => candidate.ClaimDueAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), Times.AtLeast(2)); } + private static CompleteDownloadBackgroundService CreateService( + ServiceProvider provider) => new( + Channel.CreateBounded(1), + provider.GetRequiredService(), + Mock.Of>()); + private static ServiceProvider CreateProvider( - IAnimationInfoRepository repository, + IDurableJobRepository repository, IFileMapper mapper, - IPluginEventTrigger plugin) - { - return new ServiceCollection() + IDownloadCompletionNotifier notifier, + IPluginEventTrigger plugin) => + new ServiceCollection() .AddSingleton(repository) .AddSingleton(mapper) + .AddSingleton(notifier) .AddSingleton(plugin) .BuildServiceProvider(); + + private static DurableJob CreateJob( + DurableJobStage stage, + int attemptCount = 0) + { + var now = DateTimeOffset.UtcNow; + return new DurableJob( + Guid.NewGuid(), + $"completion:{Guid.NewGuid():N}", + DurableJobType.DownloadCompletion, + DurableJobStatus.Processing, + stage, + JsonSerializer.Serialize(new DownloadCompletionJobPayload( + Guid.NewGuid(), + "/downloads/item", + "local", + Guid.NewGuid())), + attemptCount, + now, + now, + now, + null, + null, + "worker", + now.AddMinutes(1), + null); } - private static AnimationInfo CreateInfo(Guid id) => new( - id, - "Title", - "Description", - DateTimeOffset.UtcNow, - "https://example.test/item.torrent", - "torrent", - [], - "hash", - true, - DateTimeOffset.UtcNow, - DateTimeOffset.UtcNow, - true, - "local", - "/downloads/item", - 1, - 1, - null, - null, - true, - 0, - AutomationDisposition: SubscriptionAutomationDisposition.DownloadCompleted); + private static async Task WaitUntilAsync(Func condition, TimeSpan timeout) + { + var deadline = DateTimeOffset.UtcNow + timeout; + while (!condition()) + { + if (DateTimeOffset.UtcNow >= deadline) + Assert.Fail("Timed out waiting for the worker to retry."); + await Task.Delay(10); + } + } } diff --git a/SecondDimensionWatcherReDive.Test/DurableJobsControllerTests.cs b/SecondDimensionWatcherReDive.Test/DurableJobsControllerTests.cs new file mode 100644 index 0000000..952e4d9 --- /dev/null +++ b/SecondDimensionWatcherReDive.Test/DurableJobsControllerTests.cs @@ -0,0 +1,72 @@ +using Microsoft.AspNetCore.Mvc; +using Moq; +using SecondDimensionWatcherReDive.Controllers; +using SecondDimensionWatcherReDive.Controllers.External; +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Test; + +[TestClass] +public sealed class DurableJobsControllerTests +{ + [TestMethod] + public async Task GetAsync_MapsDeadLettersWithoutPayloadOrLeaseData() + { + var now = DateTimeOffset.UtcNow; + var job = new DurableJob( + Guid.NewGuid(), + "secret-deduplication-key", + DurableJobType.DownloadCompletion, + DurableJobStatus.DeadLetter, + DurableJobStage.Notify, + "{\"storePath\":\"/secret/path\"}", + 8, + now, + now, + now, + now, + null, + "private-host:worker", + now, + "InvalidOperationException"); + var repository = new Mock(); + repository.Setup(candidate => candidate.GetPageAsync( + DurableJobStatus.DeadLetter, + 0, + 50, + CancellationToken.None)) + .ReturnsAsync(new DurableJobPage([job], 1)); + var controller = new DurableJobsController(repository.Object); + + var result = await controller.GetAsync( + "deadLetter", 0, 50, CancellationToken.None); + + var response = (DurableJobListResponse)((OkObjectResult)result).Value!; + Assert.HasCount(1, response.Items); + Assert.AreEqual(job.Id, response.Items[0].Id); + Assert.AreEqual("notify", response.Items[0].Stage); + Assert.IsNull(typeof(DurableJobItem).GetProperty("PayloadJson")); + Assert.IsNull(typeof(DurableJobItem).GetProperty("LeaseOwner")); + Assert.IsNull(typeof(DurableJobItem).GetProperty("DeduplicationKey")); + } + + [TestMethod] + public async Task RetryAsync_DeduplicatesIds() + { + var id = Guid.NewGuid(); + var repository = new Mock(); + repository.Setup(candidate => candidate.RetryAsync( + It.Is>(ids => ids.Count == 1 && ids.Contains(id)), + It.IsAny(), + CancellationToken.None)) + .ReturnsAsync(1); + var controller = new DurableJobsController(repository.Object); + + var result = await controller.RetryAsync( + new DurableJobMutationRequest([id, id]), + CancellationToken.None); + + var response = (DurableJobMutationResponse)((OkObjectResult)result).Value!; + Assert.AreEqual(1, response.AffectedCount); + } +} diff --git a/SecondDimensionWatcherReDive.Test/ObservabilityTests.cs b/SecondDimensionWatcherReDive.Test/ObservabilityTests.cs new file mode 100644 index 0000000..1be8fe3 --- /dev/null +++ b/SecondDimensionWatcherReDive.Test/ObservabilityTests.cs @@ -0,0 +1,96 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; +using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Observability; + +namespace SecondDimensionWatcherReDive.Test; + +[TestClass] +public sealed class ObservabilityTests +{ + [TestMethod] + public void SensitiveTagProcessor_RemovesPathsQueriesAndStatements() + { + using var activity = new Activity("request").Start(); + activity.SetTag("url.path", "/anime/private-title"); + activity.SetTag("url.query", "token=secret"); + activity.SetTag("db.statement", "SELECT * FROM users"); + activity.SetTag("tool.arguments", "{ secret: true }"); + activity.SetTag("http.route", "/anime/{id}"); + + new SensitiveTagRedactionProcessor().OnEnd(activity); + + Assert.IsNull(activity.GetTagItem("url.path")); + Assert.IsNull(activity.GetTagItem("url.query")); + Assert.IsNull(activity.GetTagItem("db.statement")); + Assert.IsNull(activity.GetTagItem("tool.arguments")); + Assert.AreEqual("/anime/{id}", activity.GetTagItem("http.route")); + } + + [TestMethod] + public void DurableJobMetrics_UseOnlyFixedLowCardinalityTags() + { + var observedKeys = new HashSet(StringComparer.Ordinal); + using var listener = new MeterListener(); + listener.InstrumentPublished = (instrument, candidate) => + { + if (instrument.Meter.Name == RuntimeTelemetry.MeterName) + candidate.EnableMeasurementEvents(instrument); + }; + listener.SetMeasurementEventCallback((_, _, tags, _) => + { + foreach (var tag in tags) + observedKeys.Add(tag.Key); + }); + listener.SetMeasurementEventCallback((_, _, tags, _) => + { + foreach (var tag in tags) + observedKeys.Add(tag.Key); + }); + listener.Start(); + using var telemetry = new RuntimeTelemetry(); + + telemetry.RecordJobAttempt( + DurableJobType.DownloadCompletion, + DurableJobStage.MapFiles, + "retry", + TimeSpan.FromMilliseconds(20)); + + CollectionAssert.AreEquivalent( + new[] { "job.type", "job.stage", "outcome" }, + observedKeys.ToArray()); + Assert.IsFalse(observedKeys.Any(key => + key.Contains("path", StringComparison.OrdinalIgnoreCase) + || key.Contains("title", StringComparison.OrdinalIgnoreCase) + || key.Contains("argument", StringComparison.OrdinalIgnoreCase))); + } + + [TestMethod] + public void ScheduledTaskMetrics_NormalizeUnknownTaskIds() + { + var observed = new Dictionary(StringComparer.Ordinal); + using var listener = new MeterListener(); + listener.InstrumentPublished = (instrument, candidate) => + { + if (instrument.Meter.Name == RuntimeTelemetry.MeterName + && instrument.Name == "sdw.scheduled_task.runs") + candidate.EnableMeasurementEvents(instrument); + }; + listener.SetMeasurementEventCallback((_, _, tags, _) => + { + foreach (var tag in tags) + observed[tag.Key] = tag.Value?.ToString(); + }); + listener.Start(); + using var telemetry = new RuntimeTelemetry(); + + telemetry.RecordScheduledTask( + "private/title/or/tool-arguments", + "failed", + TimeSpan.FromMilliseconds(1)); + + Assert.AreEqual("other", observed["task.id"]); + Assert.AreEqual("failed", observed["outcome"]); + Assert.IsFalse(observed.Values.Contains("private/title/or/tool-arguments")); + } +} diff --git a/SecondDimensionWatcherReDive.Test/ScheduledTaskBaseTests.cs b/SecondDimensionWatcherReDive.Test/ScheduledTaskBaseTests.cs new file mode 100644 index 0000000..94b5105 --- /dev/null +++ b/SecondDimensionWatcherReDive.Test/ScheduledTaskBaseTests.cs @@ -0,0 +1,168 @@ +using SecondDimensionWatcherReDive.Framework.Tasks; +using SecondDimensionWatcherReDive.Services; + +namespace SecondDimensionWatcherReDive.Test; + +[TestClass] +public sealed class ScheduledTaskBaseTests +{ + [TestMethod] + public async Task RunNowAsync_ConcurrentRequestsAreCoalesced() + { + var task = new BlockingTask(); + var leaseManager = new FakeLeaseManager(); + using var cancellation = new CancellationTokenSource(); + var processor = task.ProcessQueueAsync(leaseManager, cancellation.Token); + + var first = task.RunNowAsync(CancellationToken.None); + await task.Started.Task.WaitAsync(TimeSpan.FromSeconds(2)); + var second = task.RunNowAsync(CancellationToken.None); + task.Release.TrySetResult(); + await Task.WhenAll(first, second).WaitAsync(TimeSpan.FromSeconds(2)); + + Assert.AreEqual(1, task.ExecutionCount); + Assert.AreEqual(1, leaseManager.AcquireCount); + Assert.AreEqual(1, leaseManager.Lease.CompletionCount); + + await cancellation.CancelAsync(); + await AssertCanceledAsync(processor); + } + + [TestMethod] + public async Task RunNowAsync_LeaseOwnedByAnotherInstance_SkipsExecution() + { + var task = new BlockingTask(); + var leaseManager = new FakeLeaseManager { Deny = true }; + using var cancellation = new CancellationTokenSource(); + var processor = task.ProcessQueueAsync(leaseManager, cancellation.Token); + + await task.RunNowAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(2)); + + Assert.AreEqual(0, task.ExecutionCount); + Assert.AreEqual(1, leaseManager.AcquireCount); + Assert.IsTrue(leaseManager.LastForce); + + await cancellation.CancelAsync(); + await AssertCanceledAsync(processor); + } + + [TestMethod] + public async Task RunScheduledAsync_ContentionReportsSkippedWithoutForcingCooldown() + { + var task = new BlockingTask(); + var leaseManager = new FakeLeaseManager { Deny = true }; + using var cancellation = new CancellationTokenSource(); + var processor = task.ProcessQueueAsync(leaseManager, cancellation.Token); + + var executed = await task.RunScheduledAsync(CancellationToken.None) + .WaitAsync(TimeSpan.FromSeconds(2)); + + Assert.IsFalse(executed); + Assert.IsFalse(leaseManager.LastForce); + await cancellation.CancelAsync(); + await AssertCanceledAsync(processor); + } + + [TestMethod] + public async Task RunScheduledAsync_TemporaryLeaseStoreFailureDoesNotStopQueue() + { + var task = new BlockingTask(); + var leaseManager = new FakeLeaseManager + { + AcquireException = new InvalidOperationException("database unavailable") + }; + using var cancellation = new CancellationTokenSource(); + var processor = task.ProcessQueueAsync(leaseManager, cancellation.Token); + + var exception = await Assert.ThrowsExactlyAsync( + () => task.RunScheduledAsync(CancellationToken.None)); + Assert.IsInstanceOfType(exception.InnerException); + + leaseManager.AcquireException = null; + leaseManager.Deny = true; + Assert.IsFalse(await task.RunScheduledAsync(CancellationToken.None)); + await cancellation.CancelAsync(); + await AssertCanceledAsync(processor); + } + + [TestMethod] + public void MediaLibraryQueue_RejectsItemsBeyondCapacity() + { + var queue = new MediaLibraryScanQueue(); + var accepted = Enumerable.Range(0, MediaLibraryScanQueue.Capacity) + .Select(_ => queue.Enqueue(Guid.NewGuid())) + .ToList(); + + Assert.IsTrue(accepted.All(value => value)); + Assert.IsFalse(queue.Enqueue(Guid.NewGuid())); + } + + private sealed class BlockingTask : ScheduledTaskBase + { + public TaskCompletionSource Started { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + public int ExecutionCount { get; private set; } + public override string Id => "test"; + public override TimeSpan Interval => TimeSpan.FromMinutes(1); + + protected override async Task ExecuteTaskAsync(CancellationToken cancellationToken) + { + ExecutionCount++; + Started.TrySetResult(); + await Release.Task.WaitAsync(cancellationToken); + } + } + + private static async Task AssertCanceledAsync(Task task) + { + try + { + await task; + Assert.Fail("Expected the queue processor to be cancelled."); + } + catch (OperationCanceledException) + { + } + } + + private sealed class FakeLeaseManager : IScheduledTaskLeaseManager + { + public FakeLease Lease { get; } = new(); + public bool Deny { get; set; } + public Exception? AcquireException { get; set; } + public int AcquireCount { get; private set; } + public bool LastForce { get; private set; } + + public Task TryAcquireAsync( + string taskId, + TimeSpan interval, + bool force, + CancellationToken cancellationToken) + { + AcquireCount++; + LastForce = force; + if (AcquireException is not null) + return Task.FromException(AcquireException); + return Task.FromResult(Deny ? null : Lease); + } + } + + private sealed class FakeLease : IScheduledTaskExecutionLease + { + public int CompletionCount { get; private set; } + public CancellationToken LeaseLostToken => CancellationToken.None; + + public Task CompleteAsync( + bool succeeded, + string? error, + CancellationToken cancellationToken) + { + CompletionCount++; + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } +} diff --git a/SecondDimensionWatcherReDive/Controllers/DurableJobsController.cs b/SecondDimensionWatcherReDive/Controllers/DurableJobsController.cs new file mode 100644 index 0000000..05f7e1f --- /dev/null +++ b/SecondDimensionWatcherReDive/Controllers/DurableJobsController.cs @@ -0,0 +1,120 @@ +using Microsoft.AspNetCore.Authentication.JwtBearer; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Controllers; + +[ApiController] +[Route("api/jobs")] +[Authorize(AuthenticationSchemes = JwtBearerDefaults.AuthenticationScheme)] +internal sealed class DurableJobsController(IDurableJobRepository repository) : ControllerBase +{ + [HttpGet] + public async Task GetAsync( + [FromQuery] string? status, + [FromQuery] int skip = 0, + [FromQuery] int take = 50, + CancellationToken cancellationToken = default) + { + if (skip < 0 || take is < 1 or > 200) + return BadRequest(new + { + message = "skip must be non-negative and take must be between 1 and 200." + }); + if (!TryParseStatus(status, out var parsedStatus)) + return BadRequest(new { message = $"Unknown job status '{status}'." }); + + var page = await repository.GetPageAsync( + parsedStatus, + skip, + take, + cancellationToken); + return Ok(new External.DurableJobListResponse( + page.Items.Select(ToExternal).ToList(), + page.TotalCount)); + } + + [HttpPost("retry")] + public async Task RetryAsync( + [FromBody] External.DurableJobMutationRequest request, + CancellationToken cancellationToken) + { + if (request.Ids is null || request.Ids.Count is < 1 or > 200) + return BadRequest(new { message = "ids must contain between 1 and 200 jobs." }); + + var affected = await repository.RetryAsync( + request.Ids.Distinct().ToList(), + DateTimeOffset.UtcNow, + cancellationToken); + return Ok(new External.DurableJobMutationResponse(affected)); + } + + [HttpPost("resolve")] + public async Task ResolveAsync( + [FromBody] External.DurableJobMutationRequest request, + CancellationToken cancellationToken) + { + if (request.Ids is null || request.Ids.Count is < 1 or > 200) + return BadRequest(new { message = "ids must contain between 1 and 200 jobs." }); + + var affected = await repository.ResolveAsync( + request.Ids.Distinct().ToList(), + DateTimeOffset.UtcNow, + cancellationToken); + return Ok(new External.DurableJobMutationResponse(affected)); + } + + private static External.DurableJobItem ToExternal(DurableJob job) => new( + job.Id, + ToApiValue(job.Type), + ToApiValue(job.Status), + ToApiValue(job.Stage), + job.AttemptCount, + job.CreatedAt, + job.UpdatedAt, + job.NextAttemptAt, + job.LastAttemptAt, + job.CompletedAt, + job.LastError); + + private static bool TryParseStatus(string? value, out DurableJobStatus? status) + { + status = value?.Trim().ToLowerInvariant() switch + { + null or "" => null, + "pending" => DurableJobStatus.Pending, + "processing" => DurableJobStatus.Processing, + "completed" => DurableJobStatus.Completed, + "deadletter" or "dead-letter" => DurableJobStatus.DeadLetter, + "resolved" => DurableJobStatus.Resolved, + _ => (DurableJobStatus?)(-1) + }; + return status != (DurableJobStatus?)(-1); + } + + private static string ToApiValue(DurableJobType type) => type switch + { + DurableJobType.DownloadCompletion => "downloadCompletion", + _ => throw new ArgumentOutOfRangeException(nameof(type), type, null) + }; + + private static string ToApiValue(DurableJobStatus status) => status switch + { + DurableJobStatus.Pending => "pending", + DurableJobStatus.Processing => "processing", + DurableJobStatus.Completed => "completed", + DurableJobStatus.DeadLetter => "deadLetter", + DurableJobStatus.Resolved => "resolved", + _ => throw new ArgumentOutOfRangeException(nameof(status), status, null) + }; + + private static string ToApiValue(DurableJobStage stage) => stage switch + { + DurableJobStage.MapFiles => "mapFiles", + DurableJobStage.Notify => "notify", + DurableJobStage.InvokePlugins => "invokePlugins", + DurableJobStage.Done => "done", + _ => throw new ArgumentOutOfRangeException(nameof(stage), stage, null) + }; +} diff --git a/SecondDimensionWatcherReDive/Controllers/External/AppJsonSerializerContext.cs b/SecondDimensionWatcherReDive/Controllers/External/AppJsonSerializerContext.cs index 0f20309..f11d162 100644 --- a/SecondDimensionWatcherReDive/Controllers/External/AppJsonSerializerContext.cs +++ b/SecondDimensionWatcherReDive/Controllers/External/AppJsonSerializerContext.cs @@ -58,6 +58,10 @@ namespace SecondDimensionWatcherReDive.Controllers.External; [JsonSerializable(typeof(IncidentListResponse))] [JsonSerializable(typeof(IncidentRetryError))] [JsonSerializable(typeof(IncidentRetryBatchResponse))] +[JsonSerializable(typeof(DurableJobItem))] +[JsonSerializable(typeof(DurableJobListResponse))] +[JsonSerializable(typeof(DurableJobMutationRequest))] +[JsonSerializable(typeof(DurableJobMutationResponse))] [JsonSerializable(typeof(MediaLibrarySourceResponse))] [JsonSerializable(typeof(List))] [JsonSerializable(typeof(CreateMediaLibrarySourceRequest))] diff --git a/SecondDimensionWatcherReDive/Controllers/External/DurableJob.cs b/SecondDimensionWatcherReDive/Controllers/External/DurableJob.cs new file mode 100644 index 0000000..32cfea3 --- /dev/null +++ b/SecondDimensionWatcherReDive/Controllers/External/DurableJob.cs @@ -0,0 +1,22 @@ +namespace SecondDimensionWatcherReDive.Controllers.External; + +public sealed record DurableJobItem( + Guid Id, + string Type, + string Status, + string Stage, + int AttemptCount, + DateTimeOffset CreatedAt, + DateTimeOffset UpdatedAt, + DateTimeOffset NextAttemptAt, + DateTimeOffset? LastAttemptAt, + DateTimeOffset? CompletedAt, + string? LastError); + +public sealed record DurableJobListResponse( + IReadOnlyList Items, + int TotalCount); + +public sealed record DurableJobMutationRequest(IReadOnlyList Ids); + +public sealed record DurableJobMutationResponse(int AffectedCount); diff --git a/SecondDimensionWatcherReDive/Migrations/20260829145227_AddDurableJobsAndTaskLeases.Designer.cs b/SecondDimensionWatcherReDive/Migrations/20260829145227_AddDurableJobsAndTaskLeases.Designer.cs new file mode 100644 index 0000000..5b9dcda --- /dev/null +++ b/SecondDimensionWatcherReDive/Migrations/20260829145227_AddDurableJobsAndTaskLeases.Designer.cs @@ -0,0 +1,1086 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using SecondDimensionWatcherReDive.Models; + +#nullable disable + +namespace SecondDimensionWatcherReDive.Migrations +{ + [DbContext(typeof(ApplicationContext))] + [Migration("20260829145227_AddDurableJobsAndTaskLeases")] + partial class AddDurableJobsAndTaskLeases + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.Animation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("OriginalName") + .IsRequired() + .HasColumnType("text"); + + b.Property("PosterPath") + .HasColumnType("text"); + + b.Property("TmdbId") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("TmdbId") + .IsUnique(); + + b.ToTable("Animations"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.AnimationGroup", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("AnimationGroups"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.AnimationInfo", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AdditionalDownloadInfo") + .IsRequired() + .HasColumnType("text"); + + b.Property("AiRetryCount") + .HasColumnType("integer"); + + b.Property("AnimationId") + .HasColumnType("uuid"); + + b.Property("AutomationDisposition") + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("AutomationExplanationJson") + .HasColumnType("text"); + + b.Property("CachedDownloadData") + .IsRequired() + .HasColumnType("bytea"); + + b.Property("CurrentMetadataReviewOperationId") + .HasColumnType("uuid"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text"); + + b.Property("DownloadAttemptId") + .HasColumnType("uuid"); + + b.Property("DownloadCancellationId") + .HasColumnType("uuid"); + + b.Property("DownloadEndTime") + .HasColumnType("timestamp with time zone"); + + b.Property("DownloadStartTime") + .HasColumnType("timestamp with time zone"); + + b.Property("DownloadType") + .IsRequired() + .HasColumnType("text"); + + b.Property("DownloadUrl") + .IsRequired() + .HasColumnType("text"); + + b.Property("Episode") + .HasColumnType("integer"); + + b.Property("FileStore") + .HasColumnType("text"); + + b.Property("GroupId") + .HasColumnType("uuid"); + + b.Property("IsAiProcessed") + .HasColumnType("boolean"); + + b.Property("IsDownloadFinished") + .HasColumnType("boolean"); + + b.Property("IsDownloadTracked") + .HasColumnType("boolean"); + + b.Property("MediaLibraryMissingSince") + .HasColumnType("timestamp with time zone"); + + b.Property("MediaLibrarySourceId") + .HasColumnType("uuid"); + + b.Property("MetadataConfidence") + .HasColumnType("double precision"); + + b.Property("MetadataLastError") + .HasMaxLength(1024) + .HasColumnType("character varying(1024)"); + + b.Property("MetadataReviewedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("MetadataStatus") + .HasColumnType("integer"); + + b.Property("PublishTime") + .HasColumnType("timestamp with time zone"); + + b.Property("ReleaseSizeBytes") + .HasColumnType("bigint"); + + b.Property("Season") + .HasColumnType("integer"); + + b.Property("SourceFeedId") + .HasColumnType("uuid"); + + b.Property("StateVersion") + .IsConcurrencyToken() + .HasColumnType("bigint"); + + b.Property("StorePath") + .HasColumnType("text"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("AnimationId"); + + b.HasIndex("CurrentMetadataReviewOperationId") + .IsUnique(); + + b.HasIndex("GroupId"); + + b.HasIndex("MediaLibrarySourceId"); + + b.HasIndex("SourceFeedId"); + + b.HasIndex("FileStore", "StorePath") + .IsUnique() + .HasFilter("\"DownloadType\" = 'http://schemas.hcgstudio.com/ws/2023/06/sdw/downloadtype/media-library-import'"); + + b.HasIndex("MetadataStatus", "PublishTime"); + + b.ToTable("AnimationInfo", t => + { + t.HasCheckConstraint("CK_AnimationInfo_MetadataConfidence_Range", "\"MetadataConfidence\" IS NULL OR (\"MetadataConfidence\" >= 0 AND \"MetadataConfidence\" <= 1)"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ApplicationSettings", b => + { + b.Property("Id") + .HasColumnType("integer"); + + b.Property("ProtectedSecrets") + .HasColumnType("text"); + + b.Property("Revision") + .IsConcurrencyToken() + .HasColumnType("bigint"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("ValuesJson") + .IsRequired() + .HasColumnType("jsonb"); + + b.HasKey("Id"); + + b.ToTable("ApplicationSettings", t => + { + t.HasCheckConstraint("CK_ApplicationSettings_Revision_Positive", "\"Revision\" > 0"); + + t.HasCheckConstraint("CK_ApplicationSettings_Singleton", "\"Id\" = 1"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.BangumiSubgroup", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("MikanSubgroupId") + .HasColumnType("integer"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("ScrapedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("SeasonBangumiId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("SeasonBangumiId", "MikanSubgroupId") + .IsUnique(); + + b.ToTable("BangumiSubgroups"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ChatConversation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Title") + .HasColumnType("text"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.ToTable("ChatConversations"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ChatMessage", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Content") + .HasColumnType("text"); + + b.Property("ConversationId") + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Order") + .HasColumnType("integer"); + + b.Property("Role") + .IsRequired() + .HasColumnType("text"); + + b.Property("ToolCallId") + .HasColumnType("text"); + + b.Property("ToolCallsJson") + .HasColumnType("text"); + + b.Property("ToolName") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("ConversationId"); + + b.ToTable("ChatMessages"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.DurableJob", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AttemptCount") + .HasColumnType("integer"); + + b.Property("CompletedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("DeduplicationKey") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("LastAttemptAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastError") + .HasMaxLength(512) + .HasColumnType("character varying(512)"); + + b.Property("LeaseExpiresAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LeaseOwner") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("NextAttemptAt") + .HasColumnType("timestamp with time zone"); + + b.Property("PayloadJson") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("Stage") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(24) + .HasColumnType("character varying(24)"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(48) + .HasColumnType("character varying(48)"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DeduplicationKey") + .IsUnique(); + + b.HasIndex("Status", "NextAttemptAt", "LeaseExpiresAt"); + + b.ToTable("DurableJobs"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.Feed", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Name") + .HasColumnType("text"); + + b.Property("Url") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("Feeds"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.FileMapping", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AnimationInfoId") + .HasColumnType("uuid"); + + b.Property("FileStore") + .IsRequired() + .HasColumnType("text"); + + b.Property("PhysicalPath") + .IsRequired() + .HasColumnType("text"); + + b.Property("VirtualPath") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("AnimationInfoId"); + + b.HasIndex("VirtualPath") + .IsUnique(); + + b.ToTable("FileMappings"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.FileNameRegexRule", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AnimationId") + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("Pattern") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("character varying(512)"); + + b.HasKey("Id"); + + b.HasIndex("AnimationId", "CreatedAt"); + + b.HasIndex("AnimationId", "Pattern") + .IsUnique(); + + b.ToTable("FileNameRegexRules"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.Incident", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Detail") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("DetectedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Fingerprint") + .IsRequired() + .HasMaxLength(96) + .HasColumnType("character varying(96)"); + + b.Property("LastRetryAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastRetryError") + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("ResolvedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("RetryCount") + .HasColumnType("integer"); + + b.Property("Severity") + .HasColumnType("integer"); + + b.Property("SourceId") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("Title") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("Type") + .HasColumnType("integer"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("Fingerprint") + .IsUnique(); + + b.HasIndex("ResolvedAt", "Type", "UpdatedAt"); + + b.ToTable("Incidents"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MediaLibrarySource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("IsMonitoring") + .HasColumnType("boolean"); + + b.Property("LastError") + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("LastImportedCount") + .HasColumnType("integer"); + + b.Property("LastRemovedCount") + .HasColumnType("integer"); + + b.Property("LastScanAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastSkippedCount") + .HasColumnType("integer"); + + b.Property("LastUpdatedCount") + .HasColumnType("integer"); + + b.Property("Path") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Path") + .IsUnique(); + + b.ToTable("MediaLibrarySources"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewMappingSnapshot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("FileStore") + .IsRequired() + .HasColumnType("text"); + + b.Property("Kind") + .HasColumnType("integer"); + + b.Property("OperationId") + .HasColumnType("uuid"); + + b.Property("PhysicalPath") + .IsRequired() + .HasColumnType("text"); + + b.Property("VirtualPath") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("OperationId", "Kind", "VirtualPath") + .IsUnique(); + + b.ToTable("MetadataReviewMappingSnapshots"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AnimationInfoId") + .HasColumnType("uuid"); + + b.Property("AppliedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("AppliedVersion") + .HasColumnType("bigint"); + + b.Property("BaseFileStore") + .HasColumnType("text"); + + b.Property("BaseIsDownloadFinished") + .HasColumnType("boolean"); + + b.Property("BaseStorePath") + .HasColumnType("text"); + + b.Property("BaseVersion") + .HasColumnType("bigint"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("ExpiresAt") + .HasColumnType("timestamp with time zone"); + + b.Property("PreviousAiRetryCount") + .HasColumnType("integer"); + + b.Property("PreviousAnimationId") + .HasColumnType("uuid"); + + b.Property("PreviousConfidence") + .HasColumnType("double precision"); + + b.Property("PreviousCurrentOperationId") + .HasColumnType("uuid"); + + b.Property("PreviousDescription") + .HasColumnType("text"); + + b.Property("PreviousEpisode") + .HasColumnType("integer"); + + b.Property("PreviousGroupId") + .HasColumnType("uuid"); + + b.Property("PreviousIsAiProcessed") + .HasColumnType("boolean"); + + b.Property("PreviousLastError") + .HasColumnType("text"); + + b.Property("PreviousMetadataStatus") + .HasColumnType("integer"); + + b.Property("PreviousReviewedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("PreviousSeason") + .HasColumnType("integer"); + + b.Property("ProposedAnimationName") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProposedAnimationOriginalName") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProposedAnimationPosterPath") + .HasColumnType("text"); + + b.Property("ProposedAnimationTmdbId") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProposedDescription") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProposedEpisode") + .HasColumnType("integer"); + + b.Property("ProposedGroupName") + .HasColumnType("text"); + + b.Property("ProposedSeason") + .HasColumnType("integer"); + + b.Property("State") + .HasColumnType("integer"); + + b.Property("UndoneAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("AnimationInfoId", "AppliedVersion") + .IsUnique(); + + b.HasIndex("AnimationInfoId", "State"); + + b.HasIndex("State", "ExpiresAt"); + + b.ToTable("MetadataReviewOperations", t => + { + t.HasCheckConstraint("CK_MetadataReviewOperations_Expiry", "\"ExpiresAt\" > \"CreatedAt\""); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MigrationMarker", b => + { + b.Property("Key") + .HasColumnType("text"); + + b.Property("AppliedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Key"); + + b.ToTable("MigrationMarkers"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.PlaybackPreference", b => + { + b.Property("UserId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AudioLanguage") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("AudioTrackLabel") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("AutoPlayNext") + .HasColumnType("boolean"); + + b.Property("SubtitleLanguage") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("SubtitleTrackLabel") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("UserId"); + + b.ToTable("PlaybackPreferences"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.PlaybackProgress", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AnimationInfoId") + .HasColumnType("uuid"); + + b.Property("DurationSeconds") + .HasColumnType("double precision"); + + b.Property("IsWatched") + .HasColumnType("boolean"); + + b.Property("PositionSeconds") + .HasColumnType("double precision"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.Property("VirtualPath") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("WatchedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("AnimationInfoId"); + + b.HasIndex("UserId", "AnimationInfoId", "VirtualPath") + .IsUnique(); + + b.HasIndex("UserId", "IsWatched", "UpdatedAt"); + + b.ToTable("PlaybackProgresses", t => + { + t.HasCheckConstraint("CK_PlaybackProgresses_Duration_NonNegative", "\"DurationSeconds\" >= 0"); + + t.HasCheckConstraint("CK_PlaybackProgresses_Position_NonNegative", "\"PositionSeconds\" >= 0"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ScheduledTaskState", b => + { + b.Property("TaskId") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("LastCompletedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastError") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("LastStartedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastSucceededAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LeaseExpiresAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LeaseOwner") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("RunCount") + .HasColumnType("bigint"); + + b.HasKey("TaskId"); + + b.ToTable("ScheduledTaskStates"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SeasonBangumi", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("DayOfWeek") + .HasColumnType("integer"); + + b.Property("ImageUrl") + .HasColumnType("text"); + + b.Property("MikanId") + .HasColumnType("integer"); + + b.Property("ScrapedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("MikanId") + .IsUnique(); + + b.ToTable("SeasonBangumis"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SubscriptionAutomationPolicy", b => + { + b.Property("FeedId") + .HasColumnType("uuid"); + + b.PrimitiveCollection("Codecs") + .IsRequired() + .HasColumnType("text[]"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.PrimitiveCollection("ExcludedKeywords") + .IsRequired() + .HasColumnType("text[]"); + + b.PrimitiveCollection("Languages") + .IsRequired() + .HasColumnType("text[]"); + + b.Property("MaxSizeBytes") + .HasColumnType("bigint"); + + b.Property("MinSizeBytes") + .HasColumnType("bigint"); + + b.Property("Mode") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.PrimitiveCollection("Resolutions") + .IsRequired() + .HasColumnType("text[]"); + + b.PrimitiveCollection("SubtitleGroups") + .IsRequired() + .HasColumnType("text[]"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("FeedId"); + + b.HasIndex("UpdatedAt"); + + b.ToTable("SubscriptionAutomationPolicies"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.WebDavToken", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("TokenHash") + .IsRequired() + .HasColumnType("text"); + + b.Property("Username") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Username") + .IsUnique(); + + b.ToTable("WebDavTokens"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.AnimationInfo", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.Animation", "Animation") + .WithMany() + .HasForeignKey("AnimationId"); + + b.HasOne("SecondDimensionWatcherReDive.Models.AnimationGroup", "Group") + .WithMany() + .HasForeignKey("GroupId"); + + b.HasOne("SecondDimensionWatcherReDive.Models.MediaLibrarySource", null) + .WithMany() + .HasForeignKey("MediaLibrarySourceId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("SecondDimensionWatcherReDive.Models.Feed", null) + .WithMany() + .HasForeignKey("SourceFeedId") + .OnDelete(DeleteBehavior.SetNull); + + b.Navigation("Animation"); + + b.Navigation("Group"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.BangumiSubgroup", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.SeasonBangumi", "SeasonBangumi") + .WithMany("Subgroups") + .HasForeignKey("SeasonBangumiId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("SeasonBangumi"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ChatMessage", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.ChatConversation", "Conversation") + .WithMany("Messages") + .HasForeignKey("ConversationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Conversation"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.FileNameRegexRule", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.Animation", null) + .WithMany() + .HasForeignKey("AnimationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewMappingSnapshot", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.MetadataReviewOperation", "Operation") + .WithMany("MappingSnapshots") + .HasForeignKey("OperationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Operation"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewOperation", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.AnimationInfo", "AnimationInfo") + .WithMany() + .HasForeignKey("AnimationInfoId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AnimationInfo"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.PlaybackProgress", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.AnimationInfo", "AnimationInfo") + .WithMany() + .HasForeignKey("AnimationInfoId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AnimationInfo"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SubscriptionAutomationPolicy", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.Feed", "Feed") + .WithOne() + .HasForeignKey("SecondDimensionWatcherReDive.Models.SubscriptionAutomationPolicy", "FeedId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Feed"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ChatConversation", b => + { + b.Navigation("Messages"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewOperation", b => + { + b.Navigation("MappingSnapshots"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SeasonBangumi", b => + { + b.Navigation("Subgroups"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/SecondDimensionWatcherReDive/Migrations/20260829145227_AddDurableJobsAndTaskLeases.cs b/SecondDimensionWatcherReDive/Migrations/20260829145227_AddDurableJobsAndTaskLeases.cs new file mode 100644 index 0000000..f23a1de --- /dev/null +++ b/SecondDimensionWatcherReDive/Migrations/20260829145227_AddDurableJobsAndTaskLeases.cs @@ -0,0 +1,79 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace SecondDimensionWatcherReDive.Migrations +{ + /// + public partial class AddDurableJobsAndTaskLeases : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "DurableJobs", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + DeduplicationKey = table.Column(type: "character varying(256)", maxLength: 256, nullable: false), + Type = table.Column(type: "character varying(48)", maxLength: 48, nullable: false), + Status = table.Column(type: "character varying(24)", maxLength: 24, nullable: false), + Stage = table.Column(type: "character varying(32)", maxLength: 32, nullable: false), + PayloadJson = table.Column(type: "jsonb", nullable: false), + AttemptCount = table.Column(type: "integer", nullable: false), + CreatedAt = table.Column(type: "timestamp with time zone", nullable: false), + UpdatedAt = table.Column(type: "timestamp with time zone", nullable: false), + NextAttemptAt = table.Column(type: "timestamp with time zone", nullable: false), + LastAttemptAt = table.Column(type: "timestamp with time zone", nullable: true), + CompletedAt = table.Column(type: "timestamp with time zone", nullable: true), + LeaseOwner = table.Column(type: "character varying(128)", maxLength: 128, nullable: true), + LeaseExpiresAt = table.Column(type: "timestamp with time zone", nullable: true), + LastError = table.Column(type: "character varying(512)", maxLength: 512, nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_DurableJobs", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "ScheduledTaskStates", + columns: table => new + { + TaskId = table.Column(type: "character varying(128)", maxLength: 128, nullable: false), + LeaseOwner = table.Column(type: "character varying(128)", maxLength: 128, nullable: true), + LeaseExpiresAt = table.Column(type: "timestamp with time zone", nullable: true), + LastStartedAt = table.Column(type: "timestamp with time zone", nullable: true), + LastCompletedAt = table.Column(type: "timestamp with time zone", nullable: true), + LastSucceededAt = table.Column(type: "timestamp with time zone", nullable: true), + RunCount = table.Column(type: "bigint", nullable: false), + LastError = table.Column(type: "character varying(256)", maxLength: 256, nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_ScheduledTaskStates", x => x.TaskId); + }); + + migrationBuilder.CreateIndex( + name: "IX_DurableJobs_DeduplicationKey", + table: "DurableJobs", + column: "DeduplicationKey", + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_DurableJobs_Status_NextAttemptAt_LeaseExpiresAt", + table: "DurableJobs", + columns: new[] { "Status", "NextAttemptAt", "LeaseExpiresAt" }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "DurableJobs"); + + migrationBuilder.DropTable( + name: "ScheduledTaskStates"); + } + } +} diff --git a/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs b/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs index 8126b9e..a3a2407 100644 --- a/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs +++ b/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs @@ -322,6 +322,75 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("ChatMessages"); }); + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.DurableJob", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AttemptCount") + .HasColumnType("integer"); + + b.Property("CompletedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("DeduplicationKey") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("LastAttemptAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastError") + .HasMaxLength(512) + .HasColumnType("character varying(512)"); + + b.Property("LeaseExpiresAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LeaseOwner") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("NextAttemptAt") + .HasColumnType("timestamp with time zone"); + + b.Property("PayloadJson") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("Stage") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(24) + .HasColumnType("character varying(24)"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(48) + .HasColumnType("character varying(48)"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DeduplicationKey") + .IsUnique(); + + b.HasIndex("Status", "NextAttemptAt", "LeaseExpiresAt"); + + b.ToTable("DurableJobs"); + }); + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.Feed", b => { b.Property("Id") @@ -753,6 +822,40 @@ protected override void BuildModel(ModelBuilder modelBuilder) }); }); + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ScheduledTaskState", b => + { + b.Property("TaskId") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("LastCompletedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastError") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("LastStartedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastSucceededAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LeaseExpiresAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LeaseOwner") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("RunCount") + .HasColumnType("bigint"); + + b.HasKey("TaskId"); + + b.ToTable("ScheduledTaskStates"); + }); + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SeasonBangumi", b => { b.Property("Id") diff --git a/SecondDimensionWatcherReDive/Models/ApplicationContext.cs b/SecondDimensionWatcherReDive/Models/ApplicationContext.cs index 59764ac..fadeb6c 100644 --- a/SecondDimensionWatcherReDive/Models/ApplicationContext.cs +++ b/SecondDimensionWatcherReDive/Models/ApplicationContext.cs @@ -33,9 +33,64 @@ public ApplicationContext(DbContextOptions options) public DbSet PlaybackPreferences { get; set; } public DbSet MediaLibrarySources { get; set; } public DbSet ApplicationSettings { get; set; } + public DbSet DurableJobs { get; set; } + public DbSet ScheduledTaskStates { get; set; } protected override void OnModelCreating(ModelBuilder modelBuilder) { + modelBuilder.Entity() + .HasIndex(job => job.DeduplicationKey) + .IsUnique(); + + modelBuilder.Entity() + .HasIndex(job => new { job.Status, job.NextAttemptAt, job.LeaseExpiresAt }); + + modelBuilder.Entity() + .Property(job => job.DeduplicationKey) + .HasMaxLength(256); + + modelBuilder.Entity() + .Property(job => job.Type) + .HasConversion() + .HasMaxLength(48); + + modelBuilder.Entity() + .Property(job => job.Status) + .HasConversion() + .HasMaxLength(24); + + modelBuilder.Entity() + .Property(job => job.Stage) + .HasConversion() + .HasMaxLength(32); + + modelBuilder.Entity() + .Property(job => job.PayloadJson) + .HasColumnType("jsonb"); + + modelBuilder.Entity() + .Property(job => job.LeaseOwner) + .HasMaxLength(128); + + modelBuilder.Entity() + .Property(job => job.LastError) + .HasMaxLength(512); + + modelBuilder.Entity() + .HasKey(state => state.TaskId); + + modelBuilder.Entity() + .Property(state => state.TaskId) + .HasMaxLength(128); + + modelBuilder.Entity() + .Property(state => state.LeaseOwner) + .HasMaxLength(128); + + modelBuilder.Entity() + .Property(state => state.LastError) + .HasMaxLength(256); + modelBuilder.Entity() .Property(settings => settings.Id) .ValueGeneratedNever(); diff --git a/SecondDimensionWatcherReDive/Models/DurableJob.cs b/SecondDimensionWatcherReDive/Models/DurableJob.cs new file mode 100644 index 0000000..b3fa530 --- /dev/null +++ b/SecondDimensionWatcherReDive/Models/DurableJob.cs @@ -0,0 +1,22 @@ +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Models; + +public sealed class DurableJob +{ + public Guid Id { get; set; } + public string DeduplicationKey { get; set; } = string.Empty; + public DurableJobType Type { get; set; } + public DurableJobStatus Status { get; set; } + public DurableJobStage Stage { get; set; } + public string PayloadJson { get; set; } = string.Empty; + public int AttemptCount { get; set; } + public DateTimeOffset CreatedAt { get; set; } + public DateTimeOffset UpdatedAt { get; set; } + public DateTimeOffset NextAttemptAt { get; set; } + public DateTimeOffset? LastAttemptAt { get; set; } + public DateTimeOffset? CompletedAt { get; set; } + public string? LeaseOwner { get; set; } + public DateTimeOffset? LeaseExpiresAt { get; set; } + public string? LastError { get; set; } +} diff --git a/SecondDimensionWatcherReDive/Models/ScheduledTaskState.cs b/SecondDimensionWatcherReDive/Models/ScheduledTaskState.cs new file mode 100644 index 0000000..257af3d --- /dev/null +++ b/SecondDimensionWatcherReDive/Models/ScheduledTaskState.cs @@ -0,0 +1,13 @@ +namespace SecondDimensionWatcherReDive.Models; + +public sealed class ScheduledTaskState +{ + public string TaskId { get; set; } = string.Empty; + public string? LeaseOwner { get; set; } + public DateTimeOffset? LeaseExpiresAt { get; set; } + public DateTimeOffset? LastStartedAt { get; set; } + public DateTimeOffset? LastCompletedAt { get; set; } + public DateTimeOffset? LastSucceededAt { get; set; } + public long RunCount { get; set; } + public string? LastError { get; set; } +} diff --git a/SecondDimensionWatcherReDive/Observability/DurableJobMetricsBackgroundService.cs b/SecondDimensionWatcherReDive/Observability/DurableJobMetricsBackgroundService.cs new file mode 100644 index 0000000..db781ea --- /dev/null +++ b/SecondDimensionWatcherReDive/Observability/DurableJobMetricsBackgroundService.cs @@ -0,0 +1,37 @@ +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Observability; + +internal sealed partial class DurableJobMetricsBackgroundService( + IServiceScopeFactory scopeFactory, + RuntimeTelemetry telemetry, + ILogger logger) : BackgroundService +{ + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + using var timer = new PeriodicTimer(TimeSpan.FromSeconds(15)); + do + { + try + { + await using var scope = scopeFactory.CreateAsyncScope(); + var repository = scope.ServiceProvider.GetRequiredService(); + telemetry.UpdateJobStatistics(await repository.GetStatisticsAsync( + DateTimeOffset.UtcNow, + stoppingToken)); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + catch (Exception exception) + { + LogCollectionFailed(logger, exception); + } + } while (await timer.WaitForNextTickAsync(stoppingToken)); + } + + [LoggerMessage(Level = LogLevel.Debug, + Message = "Durable job metric collection failed")] + private static partial void LogCollectionFailed(ILogger logger, Exception exception); +} diff --git a/SecondDimensionWatcherReDive/Observability/ReadinessHealthChecks.cs b/SecondDimensionWatcherReDive/Observability/ReadinessHealthChecks.cs new file mode 100644 index 0000000..34865e4 --- /dev/null +++ b/SecondDimensionWatcherReDive/Observability/ReadinessHealthChecks.cs @@ -0,0 +1,111 @@ +using System.Text.Json; +using Microsoft.Extensions.Caching.Distributed; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using SecondDimensionWatcherReDive.AI.Abstractions; +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Observability; + +internal static class HealthTags +{ + public const string Ready = "ready"; +} + +internal sealed class DatabaseReadinessHealthCheck(IServiceScopeFactory scopeFactory) : IHealthCheck +{ + public async Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + { + await using var scope = scopeFactory.CreateAsyncScope(); + var repository = scope.ServiceProvider.GetRequiredService(); + return await repository.CanConnectAsync(cancellationToken) + ? HealthCheckResult.Healthy() + : HealthCheckResult.Unhealthy(); + } +} + +internal sealed class DistributedCacheReadinessHealthCheck(IDistributedCache cache) : IHealthCheck +{ + public async Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + { + await cache.GetAsync("health:ready", cancellationToken); + return HealthCheckResult.Healthy(); + } +} + +internal sealed class QbittorrentReadinessHealthCheck(IHttpClientFactory httpClientFactory) : IHealthCheck +{ + public async Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + { + using var client = httpClientFactory.CreateClient("RemoteTorrentDownloadClient"); + using var response = await client.GetAsync( + "/api/v2/app/version", + HttpCompletionOption.ResponseHeadersRead, + cancellationToken); + return response.IsSuccessStatusCode + ? HealthCheckResult.Healthy() + : HealthCheckResult.Unhealthy(); + } +} + +internal sealed class LocalStorageReadinessHealthCheck(IConfiguration configuration) : IHealthCheck +{ + public Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + var path = Path.GetFullPath(configuration["FileStore:Local"] ?? "./download"); + if (!Directory.Exists(path)) + return Task.FromResult(HealthCheckResult.Unhealthy()); + + // Force a real filesystem operation without creating or deleting data. + using var enumerator = Directory.EnumerateFileSystemEntries(path).GetEnumerator(); + _ = enumerator.MoveNext(); + return Task.FromResult(HealthCheckResult.Healthy()); + } +} + +internal sealed class AiReadinessHealthCheck(IServiceScopeFactory scopeFactory) : IHealthCheck +{ + public async Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + { + await using var scope = scopeFactory.CreateAsyncScope(); + var engine = scope.ServiceProvider.GetRequiredService(); + await engine.GetAvailableModelsAsync(cancellationToken); + return HealthCheckResult.Healthy(); + } +} + +internal static class HealthResponseWriter +{ + public static async Task WriteAsync( + HttpContext context, + HealthReport report) + { + context.Response.ContentType = "application/json; charset=utf-8"; + await using var writer = new Utf8JsonWriter(context.Response.Body); + writer.WriteStartObject(); + writer.WriteString("status", report.Status.ToString().ToLowerInvariant()); + writer.WriteStartObject("checks"); + foreach (var entry in report.Entries.OrderBy(pair => pair.Key, StringComparer.Ordinal)) + { + writer.WriteStartObject(entry.Key); + writer.WriteString("status", entry.Value.Status.ToString().ToLowerInvariant()); + writer.WriteNumber("durationMs", entry.Value.Duration.TotalMilliseconds); + if (entry.Value.Exception is not null) + writer.WriteString("errorType", entry.Value.Exception.GetType().Name); + writer.WriteEndObject(); + } + writer.WriteEndObject(); + writer.WriteEndObject(); + await writer.FlushAsync(context.RequestAborted); + } +} diff --git a/SecondDimensionWatcherReDive/Observability/RuntimeTelemetry.cs b/SecondDimensionWatcherReDive/Observability/RuntimeTelemetry.cs new file mode 100644 index 0000000..fb35928 --- /dev/null +++ b/SecondDimensionWatcherReDive/Observability/RuntimeTelemetry.cs @@ -0,0 +1,143 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Observability; + +public sealed class RuntimeTelemetry : IDisposable +{ + public const string MeterName = "SecondDimensionWatcherReDive.Runtime"; + public const string ActivitySourceName = "SecondDimensionWatcherReDive.Runtime"; + + private readonly Meter _meter = new(MeterName); + private readonly Counter _jobAttempts; + private readonly Histogram _jobDuration; + private readonly Counter _scheduledTaskRuns; + private readonly Histogram _scheduledTaskDuration; + private int _pendingJobs; + private int _processingJobs; + private int _deadLetterJobs; + private double _oldestPendingAge; + + public RuntimeTelemetry() + { + _jobAttempts = _meter.CreateCounter( + "sdw.durable_job.attempts", + "{attempt}", + "Durable job execution attempts."); + _jobDuration = _meter.CreateHistogram( + "sdw.durable_job.duration", + "s", + "Durable job execution duration."); + _scheduledTaskRuns = _meter.CreateCounter( + "sdw.scheduled_task.runs", + "{run}", + "Scheduled task run outcomes."); + _scheduledTaskDuration = _meter.CreateHistogram( + "sdw.scheduled_task.duration", + "s", + "Scheduled task request duration."); + _meter.CreateObservableGauge( + "sdw.durable_jobs", + ObserveJobCounts, + "{job}", + "Current durable job counts by status."); + _meter.CreateObservableGauge( + "sdw.durable_job.oldest_pending_age", + () => Volatile.Read(ref _oldestPendingAge), + "s", + "Age of the oldest pending durable job."); + } + + public static Activity? StartDurableJob(DurableJob job) + { + var activity = TelemetryActivitySource.Instance.StartActivity( + "durable_job.process", + ActivityKind.Consumer); + activity?.SetTag("job.type", ToTag(job.Type)); + activity?.SetTag("job.stage", ToTag(job.Stage)); + return activity; + } + + public void RecordJobAttempt( + DurableJobType type, + DurableJobStage stage, + string outcome, + TimeSpan duration) + { + var tags = new TagList + { + { "job.type", ToTag(type) }, + { "job.stage", ToTag(stage) }, + { "outcome", outcome } + }; + _jobAttempts.Add(1, tags); + _jobDuration.Record(duration.TotalSeconds, tags); + } + + public void UpdateJobStatistics(DurableJobStatistics statistics) + { + Volatile.Write(ref _pendingJobs, statistics.PendingCount); + Volatile.Write(ref _processingJobs, statistics.ProcessingCount); + Volatile.Write(ref _deadLetterJobs, statistics.DeadLetterCount); + Volatile.Write(ref _oldestPendingAge, statistics.OldestPendingAgeSeconds); + } + + public void RecordScheduledTask( + string taskId, + string outcome, + TimeSpan duration) + { + var tags = new TagList + { + { "task.id", NormalizeTaskId(taskId) }, + { "outcome", outcome } + }; + _scheduledTaskRuns.Add(1, tags); + _scheduledTaskDuration.Record(duration.TotalSeconds, tags); + } + + public void Dispose() => _meter.Dispose(); + + private IEnumerable> ObserveJobCounts() + { + yield return new Measurement( + Volatile.Read(ref _pendingJobs), + new KeyValuePair("status", "pending")); + yield return new Measurement( + Volatile.Read(ref _processingJobs), + new KeyValuePair("status", "processing")); + yield return new Measurement( + Volatile.Read(ref _deadLetterJobs), + new KeyValuePair("status", "dead_letter")); + } + + private static string ToTag(DurableJobType type) => type switch + { + DurableJobType.DownloadCompletion => "download_completion", + _ => "unknown" + }; + + private static string ToTag(DurableJobStage stage) => stage switch + { + DurableJobStage.MapFiles => "map_files", + DurableJobStage.Notify => "notify", + DurableJobStage.InvokePlugins => "invoke_plugins", + DurableJobStage.Done => "done", + _ => "unknown" + }; + + private static string NormalizeTaskId(string taskId) => taskId switch + { + "SyncFeed" => "sync_feed", + "ScrapeSeasonBangumi" => "scrape_season_bangumi", + "ScanMediaLibraries" => "scan_media_libraries", + "InferAnimationMetadata" => "infer_animation_metadata", + _ => "other" + }; + + private static class TelemetryActivitySource + { + internal static readonly ActivitySource Instance = new(ActivitySourceName); + } +} diff --git a/SecondDimensionWatcherReDive/Observability/SensitiveTagRedactionProcessor.cs b/SecondDimensionWatcherReDive/Observability/SensitiveTagRedactionProcessor.cs new file mode 100644 index 0000000..be5c5e3 --- /dev/null +++ b/SecondDimensionWatcherReDive/Observability/SensitiveTagRedactionProcessor.cs @@ -0,0 +1,49 @@ +using System.Diagnostics; +using OpenTelemetry; + +namespace SecondDimensionWatcherReDive.Observability; + +internal sealed class SensitiveTagRedactionProcessor : BaseProcessor +{ + private static readonly HashSet RemovedTags = new(StringComparer.Ordinal) + { + "db.statement", + "db.query.text", + "url.full", + "url.path", + "url.query", + "http.url", + "http.target", + "tool.arguments", + "tool.result" + }; + + public override void OnEnd(Activity activity) + { + foreach (var key in activity.TagObjects + .Select(pair => pair.Key) + .Where(ShouldRemove) + .ToArray()) + activity.SetTag(key, null); + + if (activity.Source.Name.Contains("EntityFrameworkCore", StringComparison.Ordinal)) + activity.DisplayName = "database.query"; + else if (activity.Source.Name.Contains("HttpClient", StringComparison.Ordinal)) + activity.DisplayName = $"HTTP {GetTag(activity, "http.request.method") ?? "request"}"; + else if (activity.Source.Name.Contains("AspNetCore", StringComparison.Ordinal)) + { + var method = GetTag(activity, "http.request.method") ?? "request"; + var route = GetTag(activity, "http.route"); + activity.DisplayName = route is null ? $"HTTP {method}" : $"{method} {route}"; + } + } + + private static string? GetTag(Activity activity, string key) => + activity.GetTagItem(key)?.ToString(); + + private static bool ShouldRemove(string key) => + RemovedTags.Contains(key) + || key.StartsWith("db.query.parameter.", StringComparison.Ordinal) + || key.StartsWith("tool.argument", StringComparison.Ordinal) + || key.StartsWith("tool.result", StringComparison.Ordinal); +} diff --git a/SecondDimensionWatcherReDive/Program.cs b/SecondDimensionWatcherReDive/Program.cs index 80f5f19..33db7e2 100644 --- a/SecondDimensionWatcherReDive/Program.cs +++ b/SecondDimensionWatcherReDive/Program.cs @@ -4,12 +4,16 @@ using System.Text; using System.Threading.Channels; using AspSpaService; +using Microsoft.AspNetCore.Diagnostics.HealthChecks; using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.DataProtection; using Microsoft.AspNetCore.StaticFiles; using Microsoft.EntityFrameworkCore; using Microsoft.IdentityModel.Tokens; using Microsoft.Net.Http.Headers; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; using SecondDimensionWatcherReDive; using SecondDimensionWatcherReDive.Auth; using SecondDimensionWatcherReDive.Configuration; @@ -19,10 +23,12 @@ using SecondDimensionWatcherReDive.Framework.FileDownload; using SecondDimensionWatcherReDive.Framework.FileStore; using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Framework.Notifications; using SecondDimensionWatcherReDive.Framework.Tasks; using SecondDimensionWatcherReDive.Inference.AI; using SecondDimensionWatcherReDive.Models; using SecondDimensionWatcherReDive.NFS; +using SecondDimensionWatcherReDive.Observability; using SecondDimensionWatcherReDive.Repositories; using SecondDimensionWatcherReDive.Chat; using SecondDimensionWatcherReDive.Plugin; @@ -109,6 +115,116 @@ }); }); +var healthChecks = builder.Services.AddHealthChecks() + .AddCheck( + "postgresql", + tags: [HealthTags.Ready], + timeout: TimeSpan.FromSeconds(10)); +if (!string.IsNullOrEmpty(builder.Configuration["Valkey:ConnectionString"]) + && builder.Configuration.GetValue("Health:ValkeyRequired", true)) + healthChecks.AddCheck( + "valkey", + tags: [HealthTags.Ready], + timeout: TimeSpan.FromSeconds(5)); +if (builder.Configuration.GetValue("Health:QbittorrentRequired", true)) + healthChecks.AddCheck( + "qbittorrent", + tags: [HealthTags.Ready], + timeout: TimeSpan.FromSeconds(5)); +if (builder.Configuration.GetValue("Health:StorageRequired", true)) + healthChecks.AddCheck( + "storage", + tags: [HealthTags.Ready], + timeout: TimeSpan.FromSeconds(5)); +if (builder.Configuration.GetValue("Health:AIRequired", false)) + healthChecks.AddCheck( + "ai", + tags: [HealthTags.Ready], + timeout: TimeSpan.FromSeconds(10)); + +builder.Services.AddSingleton(); +var otlpEndpoint = Uri.TryCreate( + builder.Configuration["OpenTelemetry:OtlpEndpoint"], + UriKind.Absolute, + out var configuredOtlpEndpoint) + ? configuredOtlpEndpoint + : null; +builder.Services.AddOpenTelemetry() + .ConfigureResource(resource => resource.AddService("SecondDimensionWatcherReDive")) + .WithTracing(tracing => + { + tracing + .AddSource(RuntimeTelemetry.ActivitySourceName) + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + // Query parameter capture is disabled by default. The processor below + // also removes statement/query-text tags before export. + .AddEntityFrameworkCoreInstrumentation() + .AddProcessor(new SensitiveTagRedactionProcessor()); + if (otlpEndpoint is not null) + tracing.AddOtlpExporter(options => options.Endpoint = otlpEndpoint); + }) + .WithMetrics(metrics => + { + metrics + .AddMeter(RuntimeTelemetry.MeterName) + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddRuntimeInstrumentation() + .AddView( + "http.server.request.duration", + new ExplicitBucketHistogramConfiguration + { + TagKeys = + [ + "http.request.method", + "http.response.status_code", + "http.route", + "network.protocol.version" + ] + }) + .AddView( + "http.client.request.duration", + new ExplicitBucketHistogramConfiguration + { + TagKeys = + [ + "http.request.method", + "http.response.status_code", + "server.address", + "server.port", + "network.protocol.version" + ] + }) + .AddView( + "sdw.durable_job.attempts", + new MetricStreamConfiguration + { + TagKeys = ["job.type", "job.stage", "outcome"] + }) + .AddView( + "sdw.durable_job.duration", + new ExplicitBucketHistogramConfiguration + { + TagKeys = ["job.type", "job.stage", "outcome"] + }) + .AddView( + "sdw.durable_jobs", + new MetricStreamConfiguration { TagKeys = ["status"] }) + .AddView( + "sdw.scheduled_task.runs", + new MetricStreamConfiguration { TagKeys = ["task.id", "outcome"] }) + .AddView( + "sdw.scheduled_task.duration", + new ExplicitBucketHistogramConfiguration + { + TagKeys = ["task.id", "outcome"] + }) + .AddPrometheusExporter(); + if (otlpEndpoint is not null) + metrics.AddOtlpExporter(options => options.Endpoint = otlpEndpoint); + }); + //Configure JWT var key = Encoding.ASCII.GetBytes(builder.Configuration["JwtSecret"] ?? throw new ApplicationException("JwtSecret must present in the config file.")); @@ -208,10 +324,29 @@ contentTypeProvider.Mappings.Add(".mkv", "video/x-matroska"); builder.Services.AddSingleton(contentTypeProvider); -//Add channels -builder.Services.AddSingleton(Channel.CreateUnbounded()); -builder.Services.AddSingleton(Channel.CreateUnbounded()); -builder.Services.AddSingleton(Channel.CreateUnbounded()); +// In-process channels are bounded. Download completion is persisted before its +// wake hint is emitted, while high-frequency progress may safely drop old samples. +builder.Services.AddSingleton(Channel.CreateBounded( + new BoundedChannelOptions(1024) + { + SingleReader = true, + SingleWriter = false, + FullMode = BoundedChannelFullMode.Wait + })); +builder.Services.AddSingleton(Channel.CreateBounded( + new BoundedChannelOptions(1024) + { + SingleReader = true, + SingleWriter = true, + FullMode = BoundedChannelFullMode.DropOldest + })); +builder.Services.AddSingleton(Channel.CreateBounded( + new BoundedChannelOptions(128) + { + SingleReader = true, + SingleWriter = true, + FullMode = BoundedChannelFullMode.DropOldest + })); // Persistent incident inbox and health probes. builder.Services.AddSingleton(); @@ -219,6 +354,7 @@ //Add hosting services builder.Services.AddHostedService(); +builder.Services.AddHostedService(); builder.Services.AddHostedService(); builder.Services.AddHostedService(); builder.Services.AddHostedService(); @@ -226,8 +362,10 @@ builder.Services.AddSingleton(sp => sp.GetRequiredService()); builder.Services.AddHostedService(); +builder.Services.AddSingleton(); //Add scheduled tasks +builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(sp => sp.GetRequiredService()); builder.Services.AddHostedService>(); @@ -277,6 +415,9 @@ builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); builder.Services.AddSingleton(); builder.Services.AddScoped(); builder.Services.AddScoped(); @@ -320,13 +461,27 @@ app.UseRouting(); +app.MapHealthChecks("/health/live", new HealthCheckOptions +{ + Predicate = _ => false, + ResponseWriter = HealthResponseWriter.WriteAsync +}).AllowAnonymous(); +app.MapHealthChecks("/health/ready", new HealthCheckOptions +{ + Predicate = registration => registration.Tags.Contains(HealthTags.Ready), + ResponseWriter = HealthResponseWriter.WriteAsync +}).AllowAnonymous(); +app.MapPrometheusScrapingEndpoint("/metrics").AllowAnonymous(); + app.MapControllers(); if (app.Environment.IsDevelopment()) { app.UseWhen( context => !context.Request.Path.StartsWithSegments("/api") && - !context.Request.Path.StartsWithSegments("/webdav"), + !context.Request.Path.StartsWithSegments("/webdav") && + !context.Request.Path.StartsWithSegments("/health") && + !context.Request.Path.StartsWithSegments("/metrics"), then => { then.UseSpa(config => diff --git a/SecondDimensionWatcherReDive/Repositories/AnimationInfoRepository.cs b/SecondDimensionWatcherReDive/Repositories/AnimationInfoRepository.cs index d90f6b6..65b0b44 100644 --- a/SecondDimensionWatcherReDive/Repositories/AnimationInfoRepository.cs +++ b/SecondDimensionWatcherReDive/Repositories/AnimationInfoRepository.cs @@ -1,4 +1,5 @@ using System.Runtime.CompilerServices; +using System.Text.Json; using Microsoft.EntityFrameworkCore; using SecondDimensionWatcherReDive.Framework.DataRepository; using SecondDimensionWatcherReDive.Framework.FileDownload; @@ -490,9 +491,37 @@ SubscriptionAutomationDisposition.AutoDownloadQueued or if (changed) { entity.StateVersion = checked(entity.StateVersion + 1); - await writeContext.SaveChangesAsync(cancellationToken); } + // Commit the durable side-effect workflow in the same transaction as + // the download completion state. A retry sees the same unique key and + // cannot create a second workflow. + var deduplicationKey = + $"download-completion:{id:N}:{downloadAttemptId?.ToString("N") ?? "legacy"}"; + if (!await writeContext.DurableJobs.AnyAsync( + job => job.DeduplicationKey == deduplicationKey, + cancellationToken)) + { + writeContext.DurableJobs.Add(new Models.DurableJob + { + Id = Guid.NewGuid(), + DeduplicationKey = deduplicationKey, + Type = DurableJobType.DownloadCompletion, + Status = DurableJobStatus.Pending, + Stage = DurableJobStage.MapFiles, + PayloadJson = JsonSerializer.Serialize(new DownloadCompletionJobPayload( + id, + storePath, + fileStore, + downloadAttemptId)), + CreatedAt = completedAt, + UpdatedAt = completedAt, + NextAttemptAt = completedAt + }); + } + + await writeContext.SaveChangesAsync(cancellationToken); + await writeContext.Entry(entity) .Reference(info => info.Animation) .LoadAsync(cancellationToken); diff --git a/SecondDimensionWatcherReDive/Repositories/DurableJobRepository.cs b/SecondDimensionWatcherReDive/Repositories/DurableJobRepository.cs new file mode 100644 index 0000000..853e54d --- /dev/null +++ b/SecondDimensionWatcherReDive/Repositories/DurableJobRepository.cs @@ -0,0 +1,218 @@ +using Microsoft.EntityFrameworkCore; +using SecondDimensionWatcherReDive.Framework.DataRepository; +using JobEntity = SecondDimensionWatcherReDive.Models.DurableJob; + +namespace SecondDimensionWatcherReDive.Repositories; + +public sealed class DurableJobRepository(Models.ApplicationContext context) + : IDurableJobRepository +{ + public async Task> ClaimDueAsync( + string workerId, + DateTimeOffset now, + DateTimeOffset leaseUntil, + int take, + CancellationToken cancellationToken) + { + var candidateIds = await context.DurableJobs + .AsNoTracking() + .Where(job => + job.NextAttemptAt <= now + && (job.Status == DurableJobStatus.Pending + || (job.Status == DurableJobStatus.Processing + && job.LeaseExpiresAt <= now))) + .OrderBy(job => job.NextAttemptAt) + .ThenBy(job => job.CreatedAt) + .Select(job => job.Id) + .Take(take) + .ToListAsync(cancellationToken); + + var claimedIds = new List(candidateIds.Count); + foreach (var id in candidateIds) + { + var affected = await context.DurableJobs + .Where(job => job.Id == id + && job.NextAttemptAt <= now + && (job.Status == DurableJobStatus.Pending + || (job.Status == DurableJobStatus.Processing + && job.LeaseExpiresAt <= now))) + .ExecuteUpdateAsync(setters => setters + .SetProperty(job => job.Status, DurableJobStatus.Processing) + .SetProperty(job => job.LeaseOwner, workerId) + .SetProperty(job => job.LeaseExpiresAt, leaseUntil) + .SetProperty(job => job.UpdatedAt, now), cancellationToken); + if (affected == 1) + claimedIds.Add(id); + } + + if (claimedIds.Count == 0) + return []; + + return (await context.DurableJobs + .AsNoTracking() + .Where(job => claimedIds.Contains(job.Id)) + .OrderBy(job => job.CreatedAt) + .ToListAsync(cancellationToken)) + .Select(ToRecord) + .ToList(); + } + + public async Task AdvanceStageAsync( + Guid id, + string workerId, + DurableJobStage expectedStage, + DurableJobStage nextStage, + DateTimeOffset now, + CancellationToken cancellationToken) + { + var completed = nextStage == DurableJobStage.Done; + var affected = await context.DurableJobs + .Where(job => job.Id == id + && job.Status == DurableJobStatus.Processing + && job.LeaseOwner == workerId + && job.LeaseExpiresAt > now + && job.Stage == expectedStage) + .ExecuteUpdateAsync(setters => setters + .SetProperty(job => job.Stage, nextStage) + .SetProperty(job => job.Status, + completed ? DurableJobStatus.Completed : DurableJobStatus.Processing) + .SetProperty(job => job.UpdatedAt, now) + .SetProperty(job => job.LastAttemptAt, now) + .SetProperty(job => job.CompletedAt, completed ? now : null) + .SetProperty(job => job.LeaseOwner, completed ? null : workerId) + .SetProperty(job => job.LeaseExpiresAt, + job => completed ? null : job.LeaseExpiresAt) + .SetProperty(job => job.LastError, (string?)null), cancellationToken); + return affected == 1; + } + + public async Task RenewLeaseAsync( + Guid id, + string workerId, + DateTimeOffset now, + DateTimeOffset leaseUntil, + CancellationToken cancellationToken) + { + var affected = await context.DurableJobs + .Where(job => job.Id == id + && job.Status == DurableJobStatus.Processing + && job.LeaseOwner == workerId + && job.LeaseExpiresAt > now) + .ExecuteUpdateAsync(setters => setters + .SetProperty(job => job.LeaseExpiresAt, leaseUntil) + .SetProperty(job => job.UpdatedAt, now), cancellationToken); + return affected == 1; + } + + public Task MarkFailedAsync( + Guid id, + string workerId, + int attemptCount, + DateTimeOffset attemptedAt, + DateTimeOffset? nextAttemptAt, + string error, + CancellationToken cancellationToken) => + context.DurableJobs + .Where(job => job.Id == id + && job.Status == DurableJobStatus.Processing + && job.LeaseOwner == workerId) + .ExecuteUpdateAsync(setters => setters + .SetProperty(job => job.Status, + nextAttemptAt.HasValue + ? DurableJobStatus.Pending + : DurableJobStatus.DeadLetter) + .SetProperty(job => job.AttemptCount, attemptCount) + .SetProperty(job => job.LastAttemptAt, attemptedAt) + .SetProperty(job => job.UpdatedAt, attemptedAt) + .SetProperty(job => job.NextAttemptAt, nextAttemptAt ?? attemptedAt) + .SetProperty(job => job.LeaseOwner, (string?)null) + .SetProperty(job => job.LeaseExpiresAt, (DateTimeOffset?)null) + .SetProperty(job => job.LastError, error), cancellationToken); + + public async Task GetPageAsync( + DurableJobStatus? status, + int skip, + int take, + CancellationToken cancellationToken) + { + var query = context.DurableJobs.AsNoTracking().AsQueryable(); + if (status.HasValue) + query = query.Where(job => job.Status == status.Value); + + var totalCount = await query.CountAsync(cancellationToken); + var jobs = await query + .OrderByDescending(job => job.UpdatedAt) + .ThenByDescending(job => job.CreatedAt) + .Skip(skip) + .Take(take) + .ToListAsync(cancellationToken); + return new DurableJobPage(jobs.Select(ToRecord).ToList(), totalCount); + } + + public Task RetryAsync( + IReadOnlyCollection ids, + DateTimeOffset now, + CancellationToken cancellationToken) => + context.DurableJobs + .Where(job => ids.Contains(job.Id) + && job.Status == DurableJobStatus.DeadLetter) + .ExecuteUpdateAsync(setters => setters + .SetProperty(job => job.Status, DurableJobStatus.Pending) + .SetProperty(job => job.AttemptCount, 0) + .SetProperty(job => job.NextAttemptAt, now) + .SetProperty(job => job.UpdatedAt, now) + .SetProperty(job => job.LastError, (string?)null) + .SetProperty(job => job.LeaseOwner, (string?)null) + .SetProperty(job => job.LeaseExpiresAt, (DateTimeOffset?)null), cancellationToken); + + public Task ResolveAsync( + IReadOnlyCollection ids, + DateTimeOffset now, + CancellationToken cancellationToken) => + context.DurableJobs + .Where(job => ids.Contains(job.Id) + && job.Status == DurableJobStatus.DeadLetter) + .ExecuteUpdateAsync(setters => setters + .SetProperty(job => job.Status, DurableJobStatus.Resolved) + .SetProperty(job => job.CompletedAt, now) + .SetProperty(job => job.UpdatedAt, now) + .SetProperty(job => job.LeaseOwner, (string?)null) + .SetProperty(job => job.LeaseExpiresAt, (DateTimeOffset?)null), cancellationToken); + + public async Task GetStatisticsAsync( + DateTimeOffset now, + CancellationToken cancellationToken) + { + var pendingCount = await context.DurableJobs.CountAsync( + job => job.Status == DurableJobStatus.Pending, cancellationToken); + var processingCount = await context.DurableJobs.CountAsync( + job => job.Status == DurableJobStatus.Processing, cancellationToken); + var deadLetterCount = await context.DurableJobs.CountAsync( + job => job.Status == DurableJobStatus.DeadLetter, cancellationToken); + var oldest = await context.DurableJobs + .Where(job => job.Status == DurableJobStatus.Pending) + .MinAsync(job => (DateTimeOffset?)job.CreatedAt, cancellationToken); + return new DurableJobStatistics( + pendingCount, + processingCount, + deadLetterCount, + oldest.HasValue ? Math.Max(0, (now - oldest.Value).TotalSeconds) : 0); + } + + private static DurableJob ToRecord(JobEntity job) => new( + job.Id, + job.DeduplicationKey, + job.Type, + job.Status, + job.Stage, + job.PayloadJson, + job.AttemptCount, + job.CreatedAt, + job.UpdatedAt, + job.NextAttemptAt, + job.LastAttemptAt, + job.CompletedAt, + job.LeaseOwner, + job.LeaseExpiresAt, + job.LastError); +} diff --git a/SecondDimensionWatcherReDive/Repositories/FileMappingRepositoryPostgreSqlTestFixture.cs b/SecondDimensionWatcherReDive/Repositories/FileMappingRepositoryPostgreSqlTestFixture.cs index 34b17f5..4068035 100644 --- a/SecondDimensionWatcherReDive/Repositories/FileMappingRepositoryPostgreSqlTestFixture.cs +++ b/SecondDimensionWatcherReDive/Repositories/FileMappingRepositoryPostgreSqlTestFixture.cs @@ -24,7 +24,7 @@ public async Task ResetAsync(CancellationToken cancellationToken) { await using var context = new Models.ApplicationContext(_contextOptions); await context.Database.ExecuteSqlRawAsync( - "TRUNCATE TABLE \"FileMappings\", \"AnimationInfo\" RESTART IDENTITY CASCADE", + "TRUNCATE TABLE \"DurableJobs\", \"ScheduledTaskStates\", \"FileMappings\", \"AnimationInfo\" RESTART IDENTITY CASCADE", cancellationToken); } @@ -83,4 +83,200 @@ public async Task GetAnimationInfoStateVersionsAsync(CancellationToken c .Select(info => info.StateVersion) .ToArrayAsync(cancellationToken); } + + public async Task<(Guid ItemId, Guid AttemptId)> SeedTrackedAnimationAsync( + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + var attemptId = Guid.NewGuid(); + var info = new Models.AnimationInfo + { + Id = Guid.NewGuid(), + Title = "tracked integration test", + IsDownloadTracked = true, + DownloadAttemptId = attemptId, + AutomationDisposition = SubscriptionAutomationDisposition.ManualDownloadQueued + }; + context.AnimationInfo.Add(info); + await context.SaveChangesAsync(cancellationToken); + return (info.Id, attemptId); + } + + public async Task CompleteTrackedAnimationAsync( + Guid itemId, + Guid attemptId, + string storePath, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + var repository = new AnimationInfoRepository(context, _contextOptions); + var result = await repository.TryCompleteDownloadAsync( + itemId, + attemptId, + "local", + storePath, + DateTimeOffset.UtcNow, + cancellationToken); + if (result is null) + throw new InvalidOperationException("The tracked download was not completed."); + } + + public async Task<(bool IsFinished, int JobCount, DownloadCompletionJobPayload Payload)> + GetCompletionStateAsync(Guid itemId, CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + var finished = await context.AnimationInfo + .Where(info => info.Id == itemId) + .Select(info => info.IsDownloadFinished) + .SingleAsync(cancellationToken); + var jobs = await context.DurableJobs + .Where(job => job.Type == DurableJobType.DownloadCompletion) + .ToListAsync(cancellationToken); + var payload = System.Text.Json.JsonSerializer + .Deserialize(jobs.Single().PayloadJson)!; + return (finished, jobs.Count, payload); + } + + public async Task SeedDurableJobAsync( + DurableJob job, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + context.DurableJobs.Add(new Models.DurableJob + { + Id = job.Id, + DeduplicationKey = job.DeduplicationKey, + Type = job.Type, + Status = job.Status, + Stage = job.Stage, + PayloadJson = job.PayloadJson, + AttemptCount = job.AttemptCount, + CreatedAt = job.CreatedAt, + UpdatedAt = job.UpdatedAt, + NextAttemptAt = job.NextAttemptAt, + LastAttemptAt = job.LastAttemptAt, + CompletedAt = job.CompletedAt, + LeaseOwner = job.LeaseOwner, + LeaseExpiresAt = job.LeaseExpiresAt, + LastError = job.LastError + }); + await context.SaveChangesAsync(cancellationToken); + } + + public async Task> ClaimDueJobsAsync( + string ownerId, + DateTimeOffset now, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + var repository = new DurableJobRepository(context); + return await repository.ClaimDueAsync( + ownerId, + now, + now.AddMinutes(1), + 10, + cancellationToken); + } + + public async Task RenewDurableJobLeaseAsync( + Guid id, + string ownerId, + DateTimeOffset now, + DateTimeOffset leaseUntil, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + return await new DurableJobRepository(context).RenewLeaseAsync( + id, + ownerId, + now, + leaseUntil, + cancellationToken); + } + + public async Task AdvanceDurableJobAsync( + Guid id, + string ownerId, + DurableJobStage expectedStage, + DurableJobStage nextStage, + DateTimeOffset now, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + return await new DurableJobRepository(context).AdvanceStageAsync( + id, + ownerId, + expectedStage, + nextStage, + now, + cancellationToken); + } + + public async Task TryAcquireTaskLeaseAsync( + string taskId, + string ownerId, + DateTimeOffset now, + DateTimeOffset leaseUntil, + bool force, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + var repository = new ScheduledTaskLeaseRepository(context); + return await repository.TryAcquireAsync( + taskId, + ownerId, + now, + leaseUntil, + force, + cancellationToken); + } + + public async Task CompleteTaskLeaseAsync( + string taskId, + string ownerId, + DateTimeOffset completedAt, + DateTimeOffset nextRunAt, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + await new ScheduledTaskLeaseRepository(context).CompleteAsync( + taskId, + ownerId, + completedAt, + nextRunAt, + true, + null, + cancellationToken); + } + + public async Task RetryJobsAsync( + IReadOnlyCollection ids, + DateTimeOffset now, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + return await new DurableJobRepository(context) + .RetryAsync(ids, now, cancellationToken); + } + + public async Task ResolveJobsAsync( + IReadOnlyCollection ids, + DateTimeOffset now, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + return await new DurableJobRepository(context) + .ResolveAsync(ids, now, cancellationToken); + } + + public async Task GetJobStatusAsync( + Guid id, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + return await context.DurableJobs + .Where(job => job.Id == id) + .Select(job => job.Status) + .SingleAsync(cancellationToken); + } } diff --git a/SecondDimensionWatcherReDive/Repositories/ReadinessRepository.cs b/SecondDimensionWatcherReDive/Repositories/ReadinessRepository.cs new file mode 100644 index 0000000..44c7ddd --- /dev/null +++ b/SecondDimensionWatcherReDive/Repositories/ReadinessRepository.cs @@ -0,0 +1,11 @@ +using Microsoft.EntityFrameworkCore; +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Repositories; + +public sealed class ReadinessRepository(Models.ApplicationContext context) + : IReadinessRepository +{ + public Task CanConnectAsync(CancellationToken cancellationToken) => + context.Database.CanConnectAsync(cancellationToken); +} diff --git a/SecondDimensionWatcherReDive/Repositories/ScheduledTaskLeaseRepository.cs b/SecondDimensionWatcherReDive/Repositories/ScheduledTaskLeaseRepository.cs new file mode 100644 index 0000000..7e1d49f --- /dev/null +++ b/SecondDimensionWatcherReDive/Repositories/ScheduledTaskLeaseRepository.cs @@ -0,0 +1,95 @@ +using Microsoft.EntityFrameworkCore; +using Npgsql; +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Repositories; + +public sealed class ScheduledTaskLeaseRepository(Models.ApplicationContext context) + : IScheduledTaskLeaseRepository +{ + public async Task TryAcquireAsync( + string taskId, + string ownerId, + DateTimeOffset now, + DateTimeOffset leaseUntil, + bool force, + CancellationToken cancellationToken) + { + var affected = await context.ScheduledTaskStates + .Where(state => state.TaskId == taskId + && (state.LeaseOwner == null + || state.LeaseExpiresAt <= now + || state.LeaseOwner == ownerId + || (force + && state.LastCompletedAt != null + && (state.LastStartedAt == null + || state.LastCompletedAt >= state.LastStartedAt)))) + .ExecuteUpdateAsync(setters => setters + .SetProperty(state => state.LeaseOwner, ownerId) + .SetProperty(state => state.LeaseExpiresAt, leaseUntil) + .SetProperty(state => state.LastStartedAt, now) + .SetProperty(state => state.RunCount, state => state.RunCount + 1), + cancellationToken); + if (affected == 1) + return true; + + var state = new Models.ScheduledTaskState + { + TaskId = taskId, + LeaseOwner = ownerId, + LeaseExpiresAt = leaseUntil, + LastStartedAt = now, + RunCount = 1 + }; + await context.ScheduledTaskStates.AddAsync(state, cancellationToken); + try + { + await context.SaveChangesAsync(cancellationToken); + return true; + } + catch (DbUpdateException exception) when ( + exception.InnerException is PostgresException { SqlState: PostgresErrorCodes.UniqueViolation }) + { + context.ChangeTracker.Clear(); + return false; + } + } + + public async Task RenewAsync( + string taskId, + string ownerId, + DateTimeOffset now, + DateTimeOffset leaseUntil, + CancellationToken cancellationToken) + { + var affected = await context.ScheduledTaskStates + .Where(state => state.TaskId == taskId + && state.LeaseOwner == ownerId) + .ExecuteUpdateAsync(setters => setters + .SetProperty(state => state.LeaseExpiresAt, leaseUntil), cancellationToken); + return affected == 1; + } + + public Task CompleteAsync( + string taskId, + string ownerId, + DateTimeOffset completedAt, + DateTimeOffset leaseUntil, + bool succeeded, + string? error, + CancellationToken cancellationToken) => + context.ScheduledTaskStates + .Where(state => state.TaskId == taskId + && state.LeaseOwner == ownerId) + .ExecuteUpdateAsync(setters => setters + // Keep a cooldown lease until the next periodic due time. Other + // instances poll this row and can take over promptly after a crash + // without immediately duplicating a normally completed run. + .SetProperty(state => state.LeaseOwner, ownerId) + .SetProperty(state => state.LeaseExpiresAt, leaseUntil) + .SetProperty(state => state.LastCompletedAt, completedAt) + .SetProperty(state => state.LastSucceededAt, + state => succeeded ? completedAt : state.LastSucceededAt) + .SetProperty(state => state.LastError, + succeeded ? null : error), cancellationToken); +} diff --git a/SecondDimensionWatcherReDive/SecondDimensionWatcherReDive.csproj b/SecondDimensionWatcherReDive/SecondDimensionWatcherReDive.csproj index caf806d..5840222 100644 --- a/SecondDimensionWatcherReDive/SecondDimensionWatcherReDive.csproj +++ b/SecondDimensionWatcherReDive/SecondDimensionWatcherReDive.csproj @@ -31,6 +31,13 @@ + + + + + + + diff --git a/SecondDimensionWatcherReDive/Services/CompleteDownloadBackgroundService.cs b/SecondDimensionWatcherReDive/Services/CompleteDownloadBackgroundService.cs index d6f2eb9..00bafdb 100644 --- a/SecondDimensionWatcherReDive/Services/CompleteDownloadBackgroundService.cs +++ b/SecondDimensionWatcherReDive/Services/CompleteDownloadBackgroundService.cs @@ -1,206 +1,342 @@ +using System.Diagnostics; +using System.Text.Json; using System.Threading.Channels; using SecondDimensionWatcherReDive.Data; -using SecondDimensionWatcherReDive.Framework.PluginParams; using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Framework.Notifications; +using SecondDimensionWatcherReDive.Framework.PluginParams; using SecondDimensionWatcherReDive.Plugin; +using SecondDimensionWatcherReDive.Observability; using SecondDimensionWatcherReDive.Utils.FileStore; using SecondDimensionWatcherReDive.Utils.Incidents; namespace SecondDimensionWatcherReDive.Services; +/// +/// Executes persisted download-completion effects. The channel is deliberately +/// only a wake-up hint: polling and expired leases make work recoverable after a +/// process crash or a lost hint. +/// public partial class CompleteDownloadBackgroundService( Channel downloadCompleteRequest, IServiceScopeFactory scopeFactory, ILogger logger, - IIncidentReporter? incidentReporter = null) + IIncidentReporter? incidentReporter = null, + RuntimeTelemetry? telemetry = null) : BackgroundService { + internal const int MaxAttempts = 8; + internal static readonly TimeSpan PollInterval = TimeSpan.FromSeconds(10); + internal static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(2); + internal static readonly TimeSpan LeaseRenewInterval = TimeSpan.FromSeconds(30); + + private readonly string _workerId = $"{Environment.MachineName}:{Environment.ProcessId}:{Guid.NewGuid():N}"; + protected override async Task ExecuteAsync(CancellationToken cancellationToken) { - var reader = downloadCompleteRequest.Reader; - while (await reader.WaitToReadAsync(cancellationToken)) + while (!cancellationToken.IsCancellationRequested) { - var request = await reader.ReadAsync(cancellationToken); - for (var attempt = 1; attempt <= 3; attempt++) + var processed = 0; + try { - try - { - await ProcessRequestAsync(request, cancellationToken); - break; - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - throw; - } - catch (Exception ex) when (attempt < 3) - { - LogProcessingRequestRetry(logger, ex, request.ItemId, attempt); - await Task.Delay(TimeSpan.FromMilliseconds(250 * attempt), cancellationToken); - } - catch (Exception ex) - { - LogProcessingRequestFailed(logger, ex, request.ItemId); - await ReportCompletionFailureAsync(request, ex, cancellationToken); - - // The torrent tracker removes finished torrents after handing - // them to this queue. Requeue here so a transient database - // outage cannot permanently lose the completion transition. - await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken); - await downloadCompleteRequest.Writer.WriteAsync(request, cancellationToken); - LogProcessingRequestRequeued(logger, request.ItemId); - } + processed = await ProcessDueJobsAsync(cancellationToken); } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + // A temporary database outage must not terminate the hosted service. + LogPollFailed(logger, exception); + } + + if (processed > 0) + continue; + + await WaitForWakeOrPollAsync(cancellationToken); } } - internal async Task ProcessRequestAsync( - DownloadCompleteRequest request, - CancellationToken cancellationToken) + internal async Task ProcessDueJobsAsync(CancellationToken cancellationToken) { - LogProcessingRequest(logger, request.ItemId, request.StorePath, request.FileStore); - await using var scope = scopeFactory.CreateAsyncScope(); - var animationInfoRepository = scope.ServiceProvider.GetRequiredService(); - - var info = await animationInfoRepository.TryCompleteDownloadAsync( - request.ItemId, - request.DownloadAttemptId, - request.FileStore, - request.StorePath, - DateTimeOffset.Now, + var repository = scope.ServiceProvider.GetRequiredService(); + var now = DateTimeOffset.UtcNow; + var jobs = await repository.ClaimDueAsync( + _workerId, + now, + now + LeaseDuration, + 1, cancellationToken); - if (info is null) - { - LogCompletionIgnored(logger, request.ItemId); - return; - } - LogDownloadMarkedFinished(logger, request.ItemId, info.Title); + foreach (var job in jobs) + await ProcessClaimedJobAsync(scope.ServiceProvider, repository, job, cancellationToken); - if (incidentReporter is not null) - { - await incidentReporter.ResolveAsync( - IncidentType.DownloadStalled, - request.ItemId.ToString(), - cancellationToken); - } + return jobs.Count; + } - // Build virtual-fs mappings for the downloaded files. + internal async Task ProcessClaimedJobAsync( + IServiceProvider serviceProvider, + IDurableJobRepository repository, + DurableJob job, + CancellationToken cancellationToken) + { + var startedAt = Stopwatch.GetTimestamp(); + var currentStage = job.Stage; + using var activity = RuntimeTelemetry.StartDurableJob(job); + using var renewalCancellation = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken); + using var leaseLost = new CancellationTokenSource(); + using var effectCancellation = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + leaseLost.Token); + var renewalTask = RenewJobLeaseAsync( + job.Id, + leaseLost, + renewalCancellation.Token); + Guid? itemId = null; try { - var fileMapper = scope.ServiceProvider.GetRequiredService(); - if (!await fileMapper.MapDownloadAsync(request.ItemId, cancellationToken)) - throw new InvalidOperationException("No file mapping could be produced."); + if (job.Type != DurableJobType.DownloadCompletion) + throw new NotSupportedException($"Unsupported durable job type: {job.Type}"); + + var payload = JsonSerializer.Deserialize(job.PayloadJson) + ?? throw new JsonException("The durable job payload is empty."); + itemId = payload.ItemId; + var stage = job.Stage; + currentStage = stage; - if (incidentReporter is not null) + if (stage == DurableJobStage.MapFiles) { - await incidentReporter.ResolveAsync( - IncidentType.FileMappingFailure, - request.ItemId.ToString(), - cancellationToken); + var mapper = serviceProvider.GetRequiredService(); + if (!await mapper.MapDownloadAsync(payload.ItemId, effectCancellation.Token)) + throw new InvalidOperationException("No file mapping could be produced."); + + if (incidentReporter is not null) + await incidentReporter.ResolveAsync( + IncidentType.FileMappingFailure, + payload.ItemId.ToString(), + effectCancellation.Token); + + await AdvanceAsync( + repository, job.Id, stage, DurableJobStage.Notify, effectCancellation.Token); + stage = DurableJobStage.Notify; + currentStage = stage; + } + + if (stage == DurableJobStage.Notify) + { + var notifier = serviceProvider.GetRequiredService(); + await notifier.NotifyAsync(job.Id, payload, effectCancellation.Token); + await AdvanceAsync( + repository, job.Id, stage, DurableJobStage.InvokePlugins, effectCancellation.Token); + stage = DurableJobStage.InvokePlugins; + currentStage = stage; } + + if (stage == DurableJobStage.InvokePlugins) + { + var eventTrigger = serviceProvider + .GetRequiredService>(); + await eventTrigger.InvokeAsync( + new FileDownloadCompleteParam( + payload.ItemId, + payload.StorePath, + payload.FileStore, + job.Id), + effectCancellation.Token); + await AdvanceAsync( + repository, job.Id, stage, DurableJobStage.Done, effectCancellation.Token); + } + + LogJobCompleted(logger, job.Id, payload.ItemId); + activity?.SetStatus(ActivityStatusCode.Ok); + telemetry?.RecordJobAttempt( + job.Type, + currentStage, + "completed", + Stopwatch.GetElapsedTime(startedAt)); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (OperationCanceledException) when (leaseLost.IsCancellationRequested) + { + // Another worker may already own the expired lease. Do not mutate the + // job with stale ownership; its persisted stage remains resumable. + LogJobLeaseLost(logger, job.Id); } - catch (Exception ex) + catch (Exception exception) { - LogFileMappingFailed(logger, ex, request.ItemId); - if (incidentReporter is not null) + var attemptCount = job.AttemptCount + 1; + var attemptedAt = DateTimeOffset.UtcNow; + var retryAt = attemptCount >= MaxAttempts + ? (DateTimeOffset?)null + : attemptedAt + RetryDelay(attemptCount); + var error = LimitError(exception); + + await repository.MarkFailedAsync( + job.Id, + _workerId, + attemptCount, + attemptedAt, + retryAt, + error, + cancellationToken); + + if (currentStage == DurableJobStage.MapFiles && incidentReporter is not null) { await incidentReporter.ReportAsync(new IncidentReport( IncidentType.FileMappingFailure, IncidentSeverity.Error, "Downloaded files could not be mapped", - ex.Message, - request.ItemId.ToString()), + error, + (itemId ?? job.Id).ToString()), cancellationToken); } + + if (retryAt.HasValue) + LogJobRetry(logger, exception, job.Id, attemptCount, retryAt.Value); + else + LogJobDeadLettered(logger, exception, job.Id, attemptCount); + activity?.SetStatus(ActivityStatusCode.Error, exception.GetType().Name); + telemetry?.RecordJobAttempt( + job.Type, + currentStage, + retryAt.HasValue ? "retry" : "dead_letter", + Stopwatch.GetElapsedTime(startedAt)); + } + finally + { + await renewalCancellation.CancelAsync(); + await renewalTask; } + } - // Fire plugin event + private async Task RenewJobLeaseAsync( + Guid jobId, + CancellationTokenSource leaseLost, + CancellationToken cancellationToken) + { try { - LogFiringPluginEvent(logger, request.ItemId); - var eventTrigger = scope.ServiceProvider - .GetRequiredService>(); - await eventTrigger.InvokeAsync(new FileDownloadCompleteParam( - request.ItemId, request.StorePath, request.FileStore), cancellationToken); - LogPluginEventCompleted(logger, request.ItemId); + using var timer = new PeriodicTimer(LeaseRenewInterval); + while (await timer.WaitForNextTickAsync(cancellationToken)) + { + var now = DateTimeOffset.UtcNow; + await using var scope = scopeFactory.CreateAsyncScope(); + var repository = scope.ServiceProvider.GetRequiredService(); + if (await repository.RenewLeaseAsync( + jobId, + _workerId, + now, + now + LeaseDuration, + cancellationToken)) + continue; + + leaseLost.Cancel(); + return; + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { } - catch (Exception ex) + catch (Exception exception) { - LogDownloadCompletedEventFailed(logger, ex, request.ItemId); + LogJobLeaseRenewalFailed(logger, exception, jobId); + leaseLost.Cancel(); } } - private async Task ReportCompletionFailureAsync( - DownloadCompleteRequest request, - Exception exception, + private async Task AdvanceAsync( + IDurableJobRepository repository, + Guid jobId, + DurableJobStage expectedStage, + DurableJobStage nextStage, CancellationToken cancellationToken) { - if (incidentReporter is null) return; + var advanced = await repository.AdvanceStageAsync( + jobId, + _workerId, + expectedStage, + nextStage, + DateTimeOffset.UtcNow, + cancellationToken); + if (!advanced) + throw new InvalidOperationException("The durable job lease was lost."); + } + private async Task WaitForWakeOrPollAsync(CancellationToken cancellationToken) + { + var reader = downloadCompleteRequest.Reader; + using var waitCancellation = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken); + waitCancellation.CancelAfter(PollInterval); try { - await incidentReporter.ReportAsync(new IncidentReport( - IncidentType.DownloadStalled, - IncidentSeverity.Error, - "Download completion could not be persisted", - exception.Message, - request.ItemId.ToString()), - cancellationToken); + await reader.WaitToReadAsync(waitCancellation.Token); } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + catch (OperationCanceledException) when ( + !cancellationToken.IsCancellationRequested + && waitCancellation.IsCancellationRequested) { - throw; + // The bounded timeout is the polling signal. Cancelling the channel + // wait prevents an abandoned waiter from accumulating every cycle. } - catch (Exception reportException) + + // Coalesce all hints. Their payload is intentionally not processed here; + // every authoritative request is already persisted transactionally. + while (reader.TryRead(out _)) { - // Persistence may be unavailable for both the completion and its - // incident. The queued retry remains the source of eventual recovery. - LogCompletionIncidentFailed(logger, reportException, request.ItemId); } } - [LoggerMessage(Level = LogLevel.Information, Message = "Processing download complete request for {ItemId}, storePath: {StorePath}, fileStore: {FileStore}")] - private static partial void LogProcessingRequest(ILogger logger, Guid itemId, string storePath, string fileStore); + internal static TimeSpan RetryDelay(int attemptCount) + { + var seconds = Math.Min(900, 5 * Math.Pow(2, Math.Max(0, attemptCount - 1))); + return TimeSpan.FromSeconds(seconds); + } - [LoggerMessage(Level = LogLevel.Information, - Message = "Ignoring completion for {ItemId} because it was cancelled, already completed, or removed")] - private static partial void LogCompletionIgnored(ILogger logger, Guid itemId); + private static string LimitError(Exception exception) + { + var value = $"{exception.GetType().Name}: {exception.Message}"; + return value.Length <= 512 ? value : value[..512]; + } - [LoggerMessage(Level = LogLevel.Error, Message = "Failed to process completion for {ItemId}")] - private static partial void LogProcessingRequestFailed(ILogger logger, Exception exception, Guid itemId); + [LoggerMessage(Level = LogLevel.Warning, Message = "Durable completion polling failed; it will retry")] + private static partial void LogPollFailed(ILogger logger, Exception exception); + + [LoggerMessage(Level = LogLevel.Information, + Message = "Durable completion job {JobId} completed for {ItemId}")] + private static partial void LogJobCompleted(ILogger logger, Guid jobId, Guid itemId); [LoggerMessage(Level = LogLevel.Warning, - Message = "Retrying completion for {ItemId} after attempt {Attempt}")] - private static partial void LogProcessingRequestRetry( + Message = "Durable completion job {JobId} failed on attempt {Attempt}; retrying at {RetryAt}")] + private static partial void LogJobRetry( + ILogger logger, + Exception exception, + Guid jobId, + int attempt, + DateTimeOffset retryAt); + + [LoggerMessage(Level = LogLevel.Error, + Message = "Durable completion job {JobId} entered dead-letter after {Attempt} attempts")] + private static partial void LogJobDeadLettered( ILogger logger, Exception exception, - Guid itemId, + Guid jobId, int attempt); [LoggerMessage(Level = LogLevel.Warning, - Message = "Requeued failed completion for {ItemId}")] - private static partial void LogProcessingRequestRequeued(ILogger logger, Guid itemId); + Message = "Durable completion job {JobId} lost its lease; another worker will resume it")] + private static partial void LogJobLeaseLost(ILogger logger, Guid jobId); [LoggerMessage(Level = LogLevel.Warning, - Message = "Failed to record completion incident for {ItemId}")] - private static partial void LogCompletionIncidentFailed( + Message = "Durable completion job {JobId} lease renewal failed")] + private static partial void LogJobLeaseRenewalFailed( ILogger logger, Exception exception, - Guid itemId); - - [LoggerMessage(Level = LogLevel.Information, Message = "Download marked finished for {ItemId}: {Title}")] - private static partial void LogDownloadMarkedFinished(ILogger logger, Guid itemId, string title); - - [LoggerMessage(Level = LogLevel.Debug, Message = "Firing OnFileDownloadCompleted plugin event for {ItemId}")] - private static partial void LogFiringPluginEvent(ILogger logger, Guid itemId); - - [LoggerMessage(Level = LogLevel.Debug, Message = "Plugin event completed for {ItemId}")] - private static partial void LogPluginEventCompleted(ILogger logger, Guid itemId); - - [LoggerMessage(Level = LogLevel.Warning, Message = "OnFileDownloadCompleted event failed for {ItemId}")] - private static partial void LogDownloadCompletedEventFailed(ILogger logger, Exception ex, Guid itemId); - - [LoggerMessage(Level = LogLevel.Warning, Message = "File mapping failed for {ItemId}")] - private static partial void LogFileMappingFailed(ILogger logger, Exception ex, Guid itemId); + Guid jobId); } diff --git a/SecondDimensionWatcherReDive/Services/FetchRemoteTorrentBackgroundService.cs b/SecondDimensionWatcherReDive/Services/FetchRemoteTorrentBackgroundService.cs index d587b9a..cef65f0 100644 --- a/SecondDimensionWatcherReDive/Services/FetchRemoteTorrentBackgroundService.cs +++ b/SecondDimensionWatcherReDive/Services/FetchRemoteTorrentBackgroundService.cs @@ -63,13 +63,31 @@ protected override async Task ExecuteAsync(CancellationToken cancellationToken) var reader = remoteTorrentTrackRequest.Reader; var tracked = new ConcurrentDictionary(); var observations = new ConcurrentDictionary(); - - // Add unfinished to track - await foreach (var request in FetchUnfinishedTaskFromDb(cancellationToken)) - tracked[request.Hash] = request; + var nextDatabaseRefreshAt = DateTimeOffset.MinValue; while (!cancellationToken.IsCancellationRequested) { + if (DateTimeOffset.UtcNow >= nextDatabaseRefreshAt) + { + try + { + // Periodic refresh recovers requests whose initial channel + // binding happened during a temporary database outage. + await foreach (var request in FetchUnfinishedTaskFromDb(cancellationToken)) + tracked[request.Hash] = request; + nextDatabaseRefreshAt = DateTimeOffset.UtcNow.AddSeconds(30); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + nextDatabaseRefreshAt = DateTimeOffset.UtcNow.AddSeconds(5); + LogRefreshTrackedDownloadsFailed(logger, exception); + } + } + // Drain channel messages in the supervised service loop so channel // failures/cancellation cannot disappear in an unobserved Task. while (reader.TryRead(out var request)) @@ -82,9 +100,23 @@ protected override async Task ExecuteAsync(CancellationToken cancellationToken) } else { - var boundRequest = await BindCurrentAttemptAsync(request, cancellationToken); - if (boundRequest is { } currentRequest) - tracked[request.Hash] = currentRequest; + try + { + var boundRequest = await BindCurrentAttemptAsync(request, cancellationToken); + if (boundRequest is { } currentRequest) + tracked[request.Hash] = currentRequest; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + // The periodic database refresh above will recover this + // unfinished attempt without relying on an unbounded queue. + nextDatabaseRefreshAt = DateTimeOffset.MinValue; + LogBindTrackedDownloadFailed(logger, exception, request.ItemId); + } } } await Task.Delay(500, cancellationToken); @@ -145,14 +177,28 @@ await ObserveHealthAsync( if (state != FileDownloadState.Finished) continue; - //Write complete request and stop tracking. - await downloadCompleteRequest.Writer.WriteAsync( - new DownloadCompleteRequest( - request.ItemId, - torrentInfo.SavePath, - FileStores.LocalDiskStore, - request.DownloadAttemptId), - cancellationToken); + var completion = new DownloadCompleteRequest( + request.ItemId, + torrentInfo.SavePath, + FileStores.LocalDiskStore, + request.DownloadAttemptId); + try + { + // The completion transition and its durable workflow are committed + // together. The channel is only a best-effort wake-up signal; the + // durable worker also polls after restart. + await PersistCompletionAsync(completion, cancellationToken); + downloadCompleteRequest.Writer.TryWrite(completion); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + LogPersistCompletionFailed(logger, exception, request.ItemId); + continue; + } tracked.TryRemove(torrentInfo.Hash, out _); observations.TryRemove(torrentInfo.Hash, out _); } @@ -165,6 +211,21 @@ await ReportMissingAfterThresholdAsync( } } + private async Task PersistCompletionAsync( + DownloadCompleteRequest request, + CancellationToken cancellationToken) + { + await using var scope = scopeFactory.CreateAsyncScope(); + var repository = scope.ServiceProvider.GetRequiredService(); + await repository.TryCompleteDownloadAsync( + request.ItemId, + request.DownloadAttemptId, + request.FileStore, + request.StorePath, + DateTimeOffset.UtcNow, + cancellationToken); + } + private async Task ObserveHealthAsync( RemoteTorrentTrackRequest request, RemoteTorrentInfo torrentInfo, @@ -323,4 +384,24 @@ private bool ShouldResolve(DownloadObservation observation, DateTimeOffset now) [LoggerMessage(Level = LogLevel.Warning, Message = "Failed to fetch torrent status from remote client")] private static partial void LogFetchTorrentStatusFailed(ILogger logger, Exception ex); + + [LoggerMessage(Level = LogLevel.Error, + Message = "Could not persist durable download completion for {ItemId}; tracking will retry")] + private static partial void LogPersistCompletionFailed( + ILogger logger, + Exception exception, + Guid itemId); + + [LoggerMessage(Level = LogLevel.Warning, + Message = "Could not refresh unfinished downloads; retrying")] + private static partial void LogRefreshTrackedDownloadsFailed( + ILogger logger, + Exception exception); + + [LoggerMessage(Level = LogLevel.Warning, + Message = "Could not bind download attempt {ItemId}; the database refresh will retry")] + private static partial void LogBindTrackedDownloadFailed( + ILogger logger, + Exception exception, + Guid itemId); } diff --git a/SecondDimensionWatcherReDive/Services/MediaLibraryScanQueue.cs b/SecondDimensionWatcherReDive/Services/MediaLibraryScanQueue.cs index 4901b70..fc5e1f2 100644 --- a/SecondDimensionWatcherReDive/Services/MediaLibraryScanQueue.cs +++ b/SecondDimensionWatcherReDive/Services/MediaLibraryScanQueue.cs @@ -14,8 +14,14 @@ public interface IMediaLibraryScanQueue public sealed class MediaLibraryScanQueue : IMediaLibraryScanQueue { - private readonly Channel _channel = Channel.CreateUnbounded( - new UnboundedChannelOptions { SingleReader = true, SingleWriter = false }); + internal const int Capacity = 256; + private readonly Channel _channel = Channel.CreateBounded( + new BoundedChannelOptions(Capacity) + { + SingleReader = true, + SingleWriter = false, + FullMode = BoundedChannelFullMode.Wait + }); private readonly ConcurrentDictionary _pending = new(); public bool Enqueue(Guid sourceId) diff --git a/SecondDimensionWatcherReDive/Services/NullDownloadCompletionNotifier.cs b/SecondDimensionWatcherReDive/Services/NullDownloadCompletionNotifier.cs new file mode 100644 index 0000000..73c3c65 --- /dev/null +++ b/SecondDimensionWatcherReDive/Services/NullDownloadCompletionNotifier.cs @@ -0,0 +1,12 @@ +using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Framework.Notifications; + +namespace SecondDimensionWatcherReDive.Services; + +internal sealed class NullDownloadCompletionNotifier : IDownloadCompletionNotifier +{ + public Task NotifyAsync( + Guid eventId, + DownloadCompletionJobPayload payload, + CancellationToken cancellationToken) => Task.CompletedTask; +} diff --git a/SecondDimensionWatcherReDive/Services/PostgresScheduledTaskLeaseManager.cs b/SecondDimensionWatcherReDive/Services/PostgresScheduledTaskLeaseManager.cs new file mode 100644 index 0000000..c16108d --- /dev/null +++ b/SecondDimensionWatcherReDive/Services/PostgresScheduledTaskLeaseManager.cs @@ -0,0 +1,146 @@ +using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Framework.Tasks; + +namespace SecondDimensionWatcherReDive.Services; + +public sealed partial class PostgresScheduledTaskLeaseManager( + IServiceScopeFactory scopeFactory, + ILogger logger) : IScheduledTaskLeaseManager +{ + internal static readonly TimeSpan LeaseDuration = TimeSpan.FromSeconds(30); + internal static readonly TimeSpan RenewInterval = TimeSpan.FromSeconds(10); + + private readonly string _ownerId = $"{Environment.MachineName}:{Environment.ProcessId}:{Guid.NewGuid():N}"; + + public async Task TryAcquireAsync( + string taskId, + TimeSpan interval, + bool force, + CancellationToken cancellationToken) + { + var now = DateTimeOffset.UtcNow; + await using var scope = scopeFactory.CreateAsyncScope(); + var repository = scope.ServiceProvider.GetRequiredService(); + var acquired = await repository.TryAcquireAsync( + taskId, + _ownerId, + now, + now + LeaseDuration, + force, + cancellationToken); + return acquired + ? new ExecutionLease(taskId, _ownerId, interval, scopeFactory, logger) + : null; + } + + private sealed class ExecutionLease : IScheduledTaskExecutionLease + { + private readonly string _taskId; + private readonly string _ownerId; + private readonly TimeSpan _interval; + private readonly IServiceScopeFactory _scopeFactory; + private readonly ILogger _logger; + private readonly CancellationTokenSource _renewalCancellation = new(); + private readonly CancellationTokenSource _leaseLost = new(); + private readonly Task _renewalTask; + private bool _completed; + + public ExecutionLease( + string taskId, + string ownerId, + TimeSpan interval, + IServiceScopeFactory scopeFactory, + ILogger logger) + { + _taskId = taskId; + _ownerId = ownerId; + _interval = interval; + _scopeFactory = scopeFactory; + _logger = logger; + _renewalTask = RenewLoopAsync(); + } + + public CancellationToken LeaseLostToken => _leaseLost.Token; + + public async Task CompleteAsync( + bool succeeded, + string? error, + CancellationToken cancellationToken) + { + if (_completed) return; + _completed = true; + await StopRenewalAsync(); + + await using var scope = _scopeFactory.CreateAsyncScope(); + var repository = scope.ServiceProvider.GetRequiredService(); + var completedAt = DateTimeOffset.UtcNow; + await repository.CompleteAsync( + _taskId, + _ownerId, + completedAt, + completedAt + _interval, + succeeded, + error, + cancellationToken); + } + + public async ValueTask DisposeAsync() + { + await StopRenewalAsync(); + _renewalCancellation.Dispose(); + _leaseLost.Dispose(); + } + + private async Task RenewLoopAsync() + { + try + { + using var timer = new PeriodicTimer(RenewInterval); + while (await timer.WaitForNextTickAsync(_renewalCancellation.Token)) + { + var now = DateTimeOffset.UtcNow; + await using var scope = _scopeFactory.CreateAsyncScope(); + var repository = scope.ServiceProvider + .GetRequiredService(); + if (await repository.RenewAsync( + _taskId, + _ownerId, + now, + now + LeaseDuration, + _renewalCancellation.Token)) + continue; + + LogLeaseLost(_logger, _taskId); + _leaseLost.Cancel(); + return; + } + } + catch (OperationCanceledException) when (_renewalCancellation.IsCancellationRequested) + { + } + catch (Exception exception) + { + LogLeaseRenewalFailed(_logger, exception, _taskId); + _leaseLost.Cancel(); + } + } + + private async Task StopRenewalAsync() + { + if (!_renewalCancellation.IsCancellationRequested) + await _renewalCancellation.CancelAsync(); + await _renewalTask; + } + } + + [LoggerMessage(Level = LogLevel.Warning, + Message = "Scheduled task {TaskId} lost its execution lease")] + private static partial void LogLeaseLost(ILogger logger, string taskId); + + [LoggerMessage(Level = LogLevel.Warning, + Message = "Scheduled task {TaskId} lease renewal failed")] + private static partial void LogLeaseRenewalFailed( + ILogger logger, + Exception exception, + string taskId); +} diff --git a/SecondDimensionWatcherReDive/Services/ScheduledTaskBackgroundService.cs b/SecondDimensionWatcherReDive/Services/ScheduledTaskBackgroundService.cs index 595c420..4bcfb25 100644 --- a/SecondDimensionWatcherReDive/Services/ScheduledTaskBackgroundService.cs +++ b/SecondDimensionWatcherReDive/Services/ScheduledTaskBackgroundService.cs @@ -1,18 +1,24 @@ +using System.Diagnostics; using SecondDimensionWatcherReDive.Framework.Tasks; +using SecondDimensionWatcherReDive.Observability; namespace SecondDimensionWatcherReDive.Services; public partial class ScheduledTaskBackgroundService( TTask task, + IScheduledTaskLeaseManager leaseManager, + RuntimeTelemetry telemetry, ILogger> logger) : BackgroundService where TTask : ScheduledTaskBase { + private static readonly TimeSpan ContendedLeasePollInterval = TimeSpan.FromSeconds(10); + protected override async Task ExecuteAsync(CancellationToken stoppingToken) { LogStartingScheduledTask(logger, task.Id); await Task.WhenAll( - task.ProcessQueueAsync(stoppingToken), + task.ProcessQueueAsync(leaseManager, stoppingToken), RunTimerLoopAsync(stoppingToken)); } @@ -20,19 +26,49 @@ private async Task RunTimerLoopAsync(CancellationToken stoppingToken) { while (!stoppingToken.IsCancellationRequested) { + var delay = task.Interval; if (task.IsEnabled) { + var startedAt = Stopwatch.GetTimestamp(); try { - await task.RunNowAsync(stoppingToken); + var executed = await task.RunScheduledAsync(stoppingToken); + if (!executed) + delay = ContendedLeasePollInterval; + telemetry.RecordScheduledTask( + task.Id, + executed ? "completed" : "contended", + Stopwatch.GetElapsedTime(startedAt)); + } + catch (ScheduledTaskLeaseUnavailableException ex) + { + delay = ContendedLeasePollInterval; + LogScheduledTaskLeaseUnavailable(logger, ex, task.Id); + telemetry.RecordScheduledTask( + task.Id, + "lease_unavailable", + Stopwatch.GetElapsedTime(startedAt)); + } + catch (OperationCanceledException ex) when (!stoppingToken.IsCancellationRequested) + { + delay = ContendedLeasePollInterval; + LogScheduledTaskLeaseLost(logger, ex, task.Id); + telemetry.RecordScheduledTask( + task.Id, + "lease_lost", + Stopwatch.GetElapsedTime(startedAt)); } catch (Exception ex) when (ex is not OperationCanceledException) { LogScheduledTaskFailed(logger, ex, task.Id); + telemetry.RecordScheduledTask( + task.Id, + "failed", + Stopwatch.GetElapsedTime(startedAt)); } } - await Task.Delay(task.Interval, stoppingToken); + await Task.Delay(delay, stoppingToken); } } @@ -41,4 +77,18 @@ private async Task RunTimerLoopAsync(CancellationToken stoppingToken) [LoggerMessage(Level = LogLevel.Error, Message = "Scheduled task {TaskId} failed during timer execution")] private static partial void LogScheduledTaskFailed(ILogger logger, Exception ex, string taskId); + + [LoggerMessage(Level = LogLevel.Warning, + Message = "Scheduled task {TaskId} lease is unavailable; retrying soon")] + private static partial void LogScheduledTaskLeaseUnavailable( + ILogger logger, + Exception exception, + string taskId); + + [LoggerMessage(Level = LogLevel.Warning, + Message = "Scheduled task {TaskId} execution was cancelled or lost its lease; retrying soon")] + private static partial void LogScheduledTaskLeaseLost( + ILogger logger, + Exception exception, + string taskId); } diff --git a/SecondDimensionWatcherReDive/Utils/FileDownload/RemoteTorrentDownloadClient.cs b/SecondDimensionWatcherReDive/Utils/FileDownload/RemoteTorrentDownloadClient.cs index 9bb0f72..e7746a0 100644 --- a/SecondDimensionWatcherReDive/Utils/FileDownload/RemoteTorrentDownloadClient.cs +++ b/SecondDimensionWatcherReDive/Utils/FileDownload/RemoteTorrentDownloadClient.cs @@ -38,7 +38,8 @@ public override async Task SubmitDownloadTaskAsync( if (response.IsSuccessStatusCode) await remoteTorrentTrackRequest.Writer.WriteAsync( - new(itemId, additionalDownloadInfo)); + new(itemId, additionalDownloadInfo), + cancellationToken); return response.IsSuccessStatusCode; } @@ -51,7 +52,8 @@ public override async Task SubmitQueryDownloadProgressAsync( CancellationToken cancellationToken) { await remoteTorrentTrackRequest.Writer.WriteAsync( - new(itemId, additionalDownloadInfo)); + new(itemId, additionalDownloadInfo), + cancellationToken); } public override async Task PauseDownloadTaskAsync( diff --git a/SecondDimensionWatcherReDive/appsettings.example.json b/SecondDimensionWatcherReDive/appsettings.example.json index da9d5ab..a266ec7 100644 --- a/SecondDimensionWatcherReDive/appsettings.example.json +++ b/SecondDimensionWatcherReDive/appsettings.example.json @@ -92,5 +92,20 @@ "Valkey": { "ConnectionString": "", "InstanceName": "sdw-redive:" + }, + + // /health/live never probes dependencies. /health/ready always probes PostgreSQL; + // these switches control the other deployment-specific readiness dependencies. + "Health": { + "ValkeyRequired": true, + "QbittorrentRequired": true, + "StorageRequired": true, + "AIRequired": false + }, + + // Prometheus metrics are always exposed at /metrics. Set an OTLP endpoint to + // additionally export traces and metrics to an OpenTelemetry collector. + "OpenTelemetry": { + "OtlpEndpoint": "" } } diff --git a/docs/runtime-reliability.md b/docs/runtime-reliability.md new file mode 100644 index 0000000..0ac218f --- /dev/null +++ b/docs/runtime-reliability.md @@ -0,0 +1,66 @@ +# 运行可靠性与可观测性 + +## 持久下载完成流程 + +下载器确认完成后，会在同一个 PostgreSQL 事务中更新下载状态并创建唯一的 +`DownloadCompletion` 持久任务。进程内的完成 Channel 仅用于立即唤醒消费者；即使 +信号丢失或进程在任意阶段退出，消费者也会轮询数据库并在租约到期后继续。 +消费者一次只领取一个任务，2 分钟任务租约每 30 秒续期；慢速映射不会被其他实例并发 +领取，而失联消费者的任务会在租约到期后恢复。 + +每个任务按以下阶段推进，阶段成功后才持久化下一个阶段： + +1. `MapFiles`：以替换方式重建该下载的虚拟文件映射，重复执行不会新增重复映射。 +2. `Notify`：调用可选的通知扩展点。 +3. `InvokePlugins`：触发下载完成插件事件。 +4. `Done`：标记任务完成。 + +通知实现和插件处理器会收到稳定的 `EventId`（持久任务 ID），必须在执行外部副作用前 +把它作为幂等键保存。这样，即使进程恰好在外部调用成功、阶段提交前退出，重新投递也 +不会重复发送通知或执行插件副作用。 + +失败任务采用指数退避（5 秒起步，最长 15 分钟），第 8 次失败后进入死信。登录后打开 +「后台任务」可以查看失败阶段、尝试次数和错误，并选择「重试」或「标记已处理」。对应 +API 为： + +- `GET /api/jobs?status=deadLetter` +- `POST /api/jobs/retry`，请求体 `{"ids":["..."]}` +- `POST /api/jobs/resolve`，请求体 `{"ids":["..."]}` + +API 不返回任务 payload、去重键或租约所有者，避免暴露存储路径和内部拓扑。 + +## 多实例定时任务 + +每个定时任务执行前都会获取 PostgreSQL 租约，并每 10 秒续租一次；默认租约为 30 秒。 +同一时刻只有一个实例执行同名任务。未持有租约的实例每 10 秒检查一次，实例退出或失联后 +可在租约到期时接管；正常完成会把租约冷却期保留到下次应运行时间，避免其他实例立即 +重复运行。手动触发可以越过“已完成”的冷却期，但不能越过仍在执行的租约。同一实例内 +的手动与周期触发会合并为一个容量为 1 的信号，避免积压重复执行。 + +所有生产 Channel 都有明确容量与溢出策略：下载跟踪使用背压；高频进度和已持久化的 +完成唤醒信号丢弃旧值；媒体库扫描在满载时拒绝新请求；聊天 SSE 使用背压。 +Codex app-server 的请求内更新缓冲最多保留 1024 项，超限会终止该轮对话并返回明确错误， +不会继续无界占用内存。 + +## 健康探针 + +- `GET /health/live`：只说明进程可响应，不访问任何外部依赖。 +- `GET /health/ready`：始终检查 PostgreSQL，并按配置检查 Valkey、qBittorrent、下载 + 存储和可选 AI Provider。任何必需依赖失败时返回 `503`。 + +`Health:QbittorrentRequired`、`Health:StorageRequired` 和 +`Health:ValkeyRequired` 默认启用（Valkey 仅在配置连接时注册）； +`Health:AIRequired` 默认关闭。探针响应只包含检查名、状态、耗时和异常类型，不包含地址、 +凭据或异常正文。 + +## 指标与链路 + +`GET /metrics` 提供 Prometheus 格式的 ASP.NET Core、HttpClient、运行时和持久任务指标。 +设置 `OpenTelemetry:OtlpEndpoint` 后，应用也会通过 OTLP 导出指标及 ASP.NET Core、 +HttpClient、EF Core 和持久任务链路。 + +HTTP 指标使用显式标签白名单：服务端只保留方法、状态码、路由模板和协议版本；客户端 +只保留方法、状态码、服务地址/端口和协议版本。自定义任务指标只使用固定枚举值 +`job.type`、`job.stage`、`outcome` 和 `status`。原始路径、查询串、动画标题、工具参数、 +SQL 文本和查询参数不会作为指标标签或导出的链路标签。定时任务指标的 `task.id` 也只 +允许四个内置任务值，未知扩展统一归为 `other`。 diff --git a/packaging/appsettings.yml b/packaging/appsettings.yml index 4cf660f..3a1bf21 100644 --- a/packaging/appsettings.yml +++ b/packaging/appsettings.yml @@ -92,3 +92,14 @@ Nfs: # Valkey: # ConnectionString: "localhost:6379" # InstanceName: "sdw-redive:" + +# 健康探针：/health/live 不访问外部依赖；/health/ready 始终检查 PostgreSQL +Health: + ValkeyRequired: true + QbittorrentRequired: true + StorageRequired: true + AIRequired: false + +# /metrics 始终提供 Prometheus 指标；配置地址后同时向 OTLP Collector 导出 +OpenTelemetry: + OtlpEndpoint: "" From 72709678651cb2d67c5abc3eef35a1aa888bcbd0 Mon Sep 17 00:00:00 2001 From: mahoshojoHCG Date: Sun, 30 Aug 2026 00:53:31 +0800 Subject: [PATCH 05/45] feat: add household profiles and scoped access --- .../ChatController.cs | 86 +- .../ChatServiceExtensions.cs | 1 + .../ConversationTitleGenerator.cs | 8 +- .../Tools/ManageDownloadsTool.cs | 15 +- README.md | 13 +- .../package.json | 3 +- .../src/Main.tsx | 12 + .../src/accounts/api.ts | 58 + .../src/accounts/hooks.ts | 23 + .../src/accounts/types.ts | 25 + .../src/auth/IAuthResult.ts | 21 + .../src/auth/hooks.ts | 62 +- .../src/auth/httpClient.test.ts | 291 ++++ .../src/auth/httpClient.ts | 513 ++++++- .../src/auth/sessionApi.ts | 15 + .../src/auth/utils.ts | 79 +- .../src/chat/api.ts | 28 +- .../src/chat/useStreamingChat.ts | 140 +- .../src/components/AnimationInfo.tsx | 43 +- .../src/components/AppHeader.tsx | 142 +- .../src/components/FileBrowser.tsx | 14 +- .../src/components/chat/ChatMessage.tsx | 8 +- .../src/components/chat/ChatMessageList.tsx | 38 +- .../settings/WebDavSettingsSection.tsx | 120 +- .../src/file/vfsHooks.ts | 51 +- .../src/i18n/locales/en/accounts.json | 37 + .../src/i18n/locales/en/animation.json | 2 + .../src/i18n/locales/en/auth.json | 2 + .../src/i18n/locales/en/common.json | 2 + .../src/i18n/locales/en/settings.json | 11 +- .../src/i18n/locales/ja/accounts.json | 37 + .../src/i18n/locales/ja/animation.json | 2 + .../src/i18n/locales/ja/auth.json | 2 + .../src/i18n/locales/ja/common.json | 2 + .../src/i18n/locales/ja/settings.json | 11 +- .../src/i18n/locales/zh-CN/accounts.json | 37 + .../src/i18n/locales/zh-CN/animation.json | 2 + .../src/i18n/locales/zh-CN/auth.json | 2 + .../src/i18n/locales/zh-CN/common.json | 2 + .../src/i18n/locales/zh-CN/settings.json | 11 +- .../src/i18n/resources.ts | 6 + .../src/incidents/hooks.ts | 11 +- .../src/pages/AccountPage.tsx | 454 +++++++ .../src/pages/ChatPage.tsx | 12 + .../src/pages/FeedsPage.tsx | 119 +- .../src/pages/LoginPage.tsx | 49 +- .../src/pages/PlayerPage.tsx | 169 ++- .../src/pages/SettingsPage.tsx | 13 +- .../src/season/SeasonDiscovery.tsx | 252 ++-- .../src/settings/IWebDavToken.ts | 9 + .../src/settings/utils.ts | 11 +- .../tsconfig.json | 1 + SecondDimensionWatcherReDive.Client/yarn.lock | 3 +- .../Authorization/AccessControl.cs | 42 + .../DataRepository/IChatRepository.cs | 47 +- .../DataRepository/IIdentityRepository.cs | 80 ++ .../DataRepository/IWebDavTokenRepository.cs | 5 +- .../DataRepository/Identity.cs | 71 + .../DataRepository/WebDavToken.cs | 7 +- .../Auth/RoleAuthorizationTests.cs | 118 ++ .../Helpers/FakeWebDavTokenRepository.cs | 40 +- .../HouseholdMigrationPostgreSqlTests.cs | 188 +++ ...HouseholdMigrationPostgreSqlTestFixture.cs | 786 +++++++++++ .../Vfs/ScopedDeviceTokenTests.cs | 137 ++ .../WebDavWebApplicationFactory.cs | 74 +- .../AccountsControllerTests.cs | 193 +++ .../AnimationInfoControllerTests.cs | 36 +- .../BasicAuthenticationHandlerTests.cs | 40 +- .../ConversationTitleGeneratorTests.cs | 35 +- .../DevicePathScopeTests.cs | 41 + .../ManageDownloadsToolAuthorizationTests.cs | 154 +++ .../PlaybackControllerTests.cs | 3 +- .../WebDavTokenControllerTests.cs | 50 +- .../Auth/BasicAuthenticationHandler.cs | 23 +- .../Auth/DevicePathScope.cs | 95 ++ .../Auth/SessionTokenIssuer.cs | 133 ++ .../Controllers/AccountsController.cs | 319 +++++ .../Controllers/AnimationInfoController.cs | 12 + .../Controllers/AuthController.cs | 342 +++-- .../Controllers/Converter.cs | 10 +- .../Controllers/External/Accounts.cs | 52 + .../External/AppJsonSerializerContext.cs | 13 +- .../Controllers/External/Auth.cs | 32 +- .../Controllers/External/File.cs | 8 +- .../Controllers/External/WebDavToken.cs | 20 +- .../Controllers/FeedController.cs | 3 + .../Controllers/FileController.cs | 59 +- .../Controllers/IncidentsController.cs | 2 + .../Controllers/MediaLibraryController.cs | 2 + .../Controllers/MetadataReviewController.cs | 2 + .../Controllers/PlaybackController.cs | 27 +- .../Controllers/SeasonController.cs | 3 + .../Controllers/SettingsController.cs | 3 + .../SubscriptionPoliciesController.cs | 3 + .../Controllers/TasksController.cs | 2 + .../Controllers/VfsController.cs | 98 +- .../Controllers/WebDavController.cs | 86 +- .../Controllers/WebDavTokenController.cs | 63 +- ...useholdIdentityAndAccessScopes.Designer.cs | 1205 +++++++++++++++++ ...550_AddHouseholdIdentityAndAccessScopes.cs | 345 +++++ .../ApplicationContextModelSnapshot.cs | 226 +++- .../Models/ApplicationContext.cs | 104 ++ .../Models/ChatConversation.cs | 2 + .../Models/LoginSession.cs | 17 + .../Models/PlaybackPreference.cs | 2 + .../Models/PlaybackProgress.cs | 2 + .../Models/UserAccount.cs | 16 + .../Models/UserProfile.cs | 14 + .../Models/WebDavToken.cs | 12 + SecondDimensionWatcherReDive/Program.cs | 73 +- .../Repositories/ChatRepository.cs | 79 +- .../Repositories/IdentityRepository.cs | 381 ++++++ .../Repositories/RepositoryConverter.cs | 14 +- .../Repositories/WebDavTokenRepository.cs | 8 +- 114 files changed, 8378 insertions(+), 761 deletions(-) create mode 100644 SecondDimensionWatcherReDive.Client/src/accounts/api.ts create mode 100644 SecondDimensionWatcherReDive.Client/src/accounts/hooks.ts create mode 100644 SecondDimensionWatcherReDive.Client/src/accounts/types.ts create mode 100644 SecondDimensionWatcherReDive.Client/src/auth/httpClient.test.ts create mode 100644 SecondDimensionWatcherReDive.Client/src/auth/sessionApi.ts create mode 100644 SecondDimensionWatcherReDive.Client/src/i18n/locales/en/accounts.json create mode 100644 SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/accounts.json create mode 100644 SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/accounts.json create mode 100644 SecondDimensionWatcherReDive.Client/src/pages/AccountPage.tsx create mode 100644 SecondDimensionWatcherReDive.Framework/Authorization/AccessControl.cs create mode 100644 SecondDimensionWatcherReDive.Framework/DataRepository/IIdentityRepository.cs create mode 100644 SecondDimensionWatcherReDive.Framework/DataRepository/Identity.cs create mode 100644 SecondDimensionWatcherReDive.IntegrationTest/Auth/RoleAuthorizationTests.cs create mode 100644 SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/HouseholdMigrationPostgreSqlTests.cs create mode 100644 SecondDimensionWatcherReDive.IntegrationTest/Repositories/HouseholdMigrationPostgreSqlTestFixture.cs create mode 100644 SecondDimensionWatcherReDive.IntegrationTest/Vfs/ScopedDeviceTokenTests.cs create mode 100644 SecondDimensionWatcherReDive.Test/AccountsControllerTests.cs create mode 100644 SecondDimensionWatcherReDive.Test/DevicePathScopeTests.cs create mode 100644 SecondDimensionWatcherReDive.Test/ManageDownloadsToolAuthorizationTests.cs create mode 100644 SecondDimensionWatcherReDive/Auth/DevicePathScope.cs create mode 100644 SecondDimensionWatcherReDive/Auth/SessionTokenIssuer.cs create mode 100644 SecondDimensionWatcherReDive/Controllers/AccountsController.cs create mode 100644 SecondDimensionWatcherReDive/Controllers/External/Accounts.cs create mode 100644 SecondDimensionWatcherReDive/Migrations/20260829155550_AddHouseholdIdentityAndAccessScopes.Designer.cs create mode 100644 SecondDimensionWatcherReDive/Migrations/20260829155550_AddHouseholdIdentityAndAccessScopes.cs create mode 100644 SecondDimensionWatcherReDive/Models/LoginSession.cs create mode 100644 SecondDimensionWatcherReDive/Models/UserAccount.cs create mode 100644 SecondDimensionWatcherReDive/Models/UserProfile.cs create mode 100644 SecondDimensionWatcherReDive/Repositories/IdentityRepository.cs diff --git a/Plugins/SecondDimensionWatcherReDive.Chat/ChatController.cs b/Plugins/SecondDimensionWatcherReDive.Chat/ChatController.cs index 203ffc1..701fca4 100644 --- a/Plugins/SecondDimensionWatcherReDive.Chat/ChatController.cs +++ b/Plugins/SecondDimensionWatcherReDive.Chat/ChatController.cs @@ -14,6 +14,7 @@ using SecondDimensionWatcherReDive.AI.Models; using SecondDimensionWatcherReDive.Chat.External; using SecondDimensionWatcherReDive.Chat.Tools; +using SecondDimensionWatcherReDive.Framework.Authorization; using SecondDimensionWatcherReDive.Framework.DataRepository; namespace SecondDimensionWatcherReDive.Chat; @@ -66,16 +67,19 @@ public async Task GetModels(CancellationToken cancellationToken) } [HttpGet("conversations")] - public async Task> GetConversations( + public async Task GetConversations( CancellationToken cancellationToken) { - return await chatRepository.GetConversationsAsync(cancellationToken); + if (!User.TryGetProfileId(out var profileId)) return Unauthorized(); + return Ok(await chatRepository.GetConversationsAsync(profileId, cancellationToken)); } [HttpGet("conversations/{id:guid}")] public async Task GetConversation(Guid id, CancellationToken cancellationToken) { - var detail = await chatRepository.GetConversationWithMessagesAsync(id, cancellationToken); + if (!User.TryGetProfileId(out var profileId)) return Unauthorized(); + var detail = await chatRepository.GetConversationWithMessagesAsync( + id, profileId, cancellationToken); if (detail is null) { LogConversationNotFound(id); @@ -85,19 +89,25 @@ public async Task GetConversation(Guid id, CancellationToken canc } [HttpPost("conversations")] - public async Task CreateConversation( + [Authorize(Policy = AccessPolicies.ChatWrite)] + public async Task CreateConversation( [FromBody] CreateConversationRequest? request, CancellationToken cancellationToken) { - var conv = await chatRepository.CreateConversationAsync(request?.Title, cancellationToken); + if (!User.TryGetProfileId(out var profileId)) return Unauthorized(); + var conv = await chatRepository.CreateConversationAsync( + profileId, request?.Title, cancellationToken); LogConversationCreated(conv.Id, request?.Title); - return conv; + return Ok(conv); } [HttpDelete("conversations/{id:guid}")] + [Authorize(Policy = AccessPolicies.ChatWrite)] public async Task DeleteConversation(Guid id, CancellationToken cancellationToken) { - var deleted = await chatRepository.DeleteConversationAsync(id, cancellationToken); + if (!User.TryGetProfileId(out var profileId)) return Unauthorized(); + var deleted = await chatRepository.DeleteConversationAsync( + id, profileId, cancellationToken); if (deleted) LogConversationDeleted(id); else @@ -106,27 +116,34 @@ public async Task DeleteConversation(Guid id, CancellationToken c } [HttpPatch("conversations/{id:guid}")] + [Authorize(Policy = AccessPolicies.ChatWrite)] public async Task UpdateConversationTitle( Guid id, [FromBody] UpdateConversationRequest request, CancellationToken cancellationToken) { - await chatRepository.UpdateConversationTitleAsync(id, request.Title, cancellationToken); + if (!User.TryGetProfileId(out var profileId)) return Unauthorized(); + await chatRepository.UpdateConversationTitleAsync( + id, profileId, request.Title, cancellationToken); return Ok(); } [HttpPost("conversations/{id:guid}/messages")] + [Authorize(Policy = AccessPolicies.ChatWrite)] public async Task SendMessage( Guid id, [FromBody] SendMessageRequest request, CancellationToken cancellationToken) { + if (!User.TryGetProfileId(out var profileId)) + return TypedResults.Unauthorized(); var aiEngine = serviceProvider.GetService(); var status = serviceProvider.GetService(); if (aiEngine is null || status is { IsConfigured: false }) return TypedResults.StatusCode(503); - var conversation = await chatRepository.GetConversationWithMessagesAsync(id, cancellationToken); + var conversation = await chatRepository.GetConversationWithMessagesAsync( + id, profileId, cancellationToken); if (conversation is null) { LogConversationNotFound(id); @@ -134,13 +151,15 @@ public async Task SendMessage( } // Get current message count for ordering - var messageOrder = await chatRepository.GetMessageCountAsync(id, cancellationToken); + var messageOrder = await chatRepository.GetMessageCountAsync( + id, profileId, cancellationToken); // Save user message var userMessage = new ChatMessageRecord( Guid.NewGuid(), "user", request.Content, null, null, null, messageOrder, DateTimeOffset.Now); - await chatRepository.AddMessageAsync(id, userMessage, cancellationToken); + await chatRepository.AddMessageAsync( + id, profileId, userMessage, cancellationToken); messageOrder++; LogUserMessageReceived(id, messageOrder - 1, request.Content.Length); @@ -154,15 +173,21 @@ public async Task SendMessage( var messages = BuildMessagesFromHistory(conversation.Messages, request.Content); LogHistoryBuilt(id, messages.Count); - var toolExecutor = new ToolExecutorBuilder(serviceProvider) + IToolExecutorBuilder toolBuilder = new ToolExecutorBuilder(serviceProvider) .AddTool() - .AddTool() .AddTool() - .AddTool() - .AddTool() - .AddTool() - .AddTool() - .Build(); + .AddTool(); + if (User.IsInRole(nameof(UserRole.Admin)) + || User.IsInRole(nameof(UserRole.Member))) + { + toolBuilder = toolBuilder + .AddTool() + .AddTool() + .AddTool(); + } + if (User.IsInRole(nameof(UserRole.Admin))) + toolBuilder = toolBuilder.AddTool(); + var toolExecutor = toolBuilder.Build(); var chatOptions = new ChatOptions { @@ -174,7 +199,7 @@ public async Task SendMessage( LogStreamingStarted(id, request.Model); return TypedResults.ServerSentEvents( - StreamChatEvents(aiEngine, messages, chatOptions, id, messageOrder, + StreamChatEvents(aiEngine, messages, chatOptions, id, profileId, messageOrder, request.Content, !hadPriorAssistant && titleEligible, request.Model, cancellationToken)); } @@ -184,6 +209,7 @@ private async IAsyncEnumerable> StreamChatEvents( List messages, ChatOptions chatOptions, Guid conversationId, + Guid profileId, int messageOrder, string firstUserMessage, bool autoTitleEligible, @@ -196,7 +222,7 @@ private async IAsyncEnumerable> StreamChatEvents( // Keep the task and await it during iterator disposal so a disconnected request cannot // release this controller's scoped repository before tool-call audit records are saved. var producer = ProduceChatEventsAsync( - aiEngine, messages, chatOptions, conversationId, messageOrder, + aiEngine, messages, chatOptions, conversationId, profileId, messageOrder, firstUserMessage, autoTitleEligible, model, channel.Writer, cancellationToken); @@ -219,6 +245,7 @@ private async Task ProduceChatEventsAsync( List messages, ChatOptions chatOptions, Guid conversationId, + Guid profileId, int messageOrder, string firstUserMessage, bool autoTitleEligible, @@ -344,7 +371,8 @@ await writer.WriteAsync( if (messagesToSave.Count > 0) { - await chatRepository.AddMessagesAsync(conversationId, messagesToSave, CancellationToken.None); + await chatRepository.AddMessagesAsync( + conversationId, profileId, messagesToSave, CancellationToken.None); LogMessagesSaved(conversationId, messagesToSave.Count); // Capture data needed for the post-stream auto-title task. @@ -372,7 +400,8 @@ await writer.WriteAsync( // stalled provider can never hang the conversation. if (firstAssistantContentForTitle is not null) { - _ = RunAutoTitleAsync(conversationId, firstUserMessage, firstAssistantContentForTitle, model); + _ = RunAutoTitleAsync( + conversationId, profileId, firstUserMessage, firstAssistantContentForTitle, model); } } @@ -394,7 +423,11 @@ private static async Task WriteToolAuditEventAsync( } private async Task RunAutoTitleAsync( - Guid conversationId, string firstUserMessage, string firstAssistantMessage, string? model) + Guid conversationId, + Guid profileId, + string firstUserMessage, + string firstAssistantMessage, + string? model) { try { @@ -402,7 +435,12 @@ private async Task RunAutoTitleAsync( var generator = scope.ServiceProvider.GetRequiredService(); using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); await generator.TryAutoTitleAsync( - conversationId, firstUserMessage, firstAssistantMessage, model, cts.Token); + conversationId, + profileId, + firstUserMessage, + firstAssistantMessage, + model, + cts.Token); } catch (Exception ex) { diff --git a/Plugins/SecondDimensionWatcherReDive.Chat/ChatServiceExtensions.cs b/Plugins/SecondDimensionWatcherReDive.Chat/ChatServiceExtensions.cs index be8e6e1..ac2e9dc 100644 --- a/Plugins/SecondDimensionWatcherReDive.Chat/ChatServiceExtensions.cs +++ b/Plugins/SecondDimensionWatcherReDive.Chat/ChatServiceExtensions.cs @@ -7,6 +7,7 @@ public static class ChatServiceExtensions { public static IServiceCollection AddChat(this IServiceCollection services) { + services.AddHttpContextAccessor(); services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/Plugins/SecondDimensionWatcherReDive.Chat/ConversationTitleGenerator.cs b/Plugins/SecondDimensionWatcherReDive.Chat/ConversationTitleGenerator.cs index 140a2e1..a10bdbd 100644 --- a/Plugins/SecondDimensionWatcherReDive.Chat/ConversationTitleGenerator.cs +++ b/Plugins/SecondDimensionWatcherReDive.Chat/ConversationTitleGenerator.cs @@ -17,6 +17,7 @@ internal interface IConversationTitleGenerator Task TryAutoTitleAsync( Guid conversationId, + Guid profileId, string userMessage, string assistantMessage, string? model, @@ -93,6 +94,7 @@ internal sealed partial class ConversationTitleGenerator( public async Task TryAutoTitleAsync( Guid conversationId, + Guid profileId, string userMessage, string assistantMessage, string? model, @@ -108,7 +110,8 @@ public async Task TryAutoTitleAsync( } // Race-safety: only persist if title is still unset on the latest snapshot. - var current = await chatRepository.GetConversationWithMessagesAsync(conversationId, cancellationToken); + var current = await chatRepository.GetConversationWithMessagesAsync( + conversationId, profileId, cancellationToken); if (current is null) return; if (!IsAutoTitleEligible(current.Title)) @@ -117,7 +120,8 @@ public async Task TryAutoTitleAsync( return; } - await chatRepository.UpdateConversationTitleAsync(conversationId, title, cancellationToken); + await chatRepository.UpdateConversationTitleAsync( + conversationId, profileId, title, cancellationToken); LogTitleSaved(conversationId, title); } catch (OperationCanceledException) diff --git a/Plugins/SecondDimensionWatcherReDive.Chat/Tools/ManageDownloadsTool.cs b/Plugins/SecondDimensionWatcherReDive.Chat/Tools/ManageDownloadsTool.cs index 831ec4b..c6085a4 100644 --- a/Plugins/SecondDimensionWatcherReDive.Chat/Tools/ManageDownloadsTool.cs +++ b/Plugins/SecondDimensionWatcherReDive.Chat/Tools/ManageDownloadsTool.cs @@ -1,6 +1,9 @@ +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http; using SecondDimensionWatcherReDive.AI.Models; using SecondDimensionWatcherReDive.Framework.AI; using SecondDimensionWatcherReDive.Framework.Attributes; +using SecondDimensionWatcherReDive.Framework.Authorization; using SecondDimensionWatcherReDive.Framework.DataRepository; using SecondDimensionWatcherReDive.Framework.FileDownload; @@ -12,7 +15,9 @@ namespace SecondDimensionWatcherReDive.Chat.Tools; internal sealed partial class ManageDownloadsTool( IAnimationInfoRepository animationInfoRepository, IFileMappingRepository fileMappingRepository, - IFileDownloadClientProvider fileDownloadClientProvider) : ITool + IFileDownloadClientProvider fileDownloadClientProvider, + IHttpContextAccessor httpContextAccessor, + IAuthorizationService authorizationService) : ITool { private async Task ExecuteCoreAsync( ManageDownloadsParams param, CancellationToken cancellationToken) @@ -111,6 +116,14 @@ private async Task ResumeDownloadAsync( private async Task CancelDownloadAsync( AnimationInfo info, IFileDownloadClient client, bool removeFile, CancellationToken cancellationToken) { + if (removeFile) + { + var principal = httpContextAccessor.HttpContext?.User; + if (principal is null || !(await authorizationService.AuthorizeAsync( + principal, resource: null, AccessPolicies.RecentAdministrator)).Succeeded) + return new ToolFailureResult("Deleting downloaded files requires recent administrator authentication"); + } + var cancellationAttemptId = info.DownloadCancellationId ?? Guid.NewGuid(); cancellationToken.ThrowIfCancellationRequested(); using (var beginCancellation = CreateDownloadSagaTokenSource()) diff --git a/README.md b/README.md index a55a5f4..e4bad8c 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ - [x] HTTP 文件浏览和流媒体播放，支持外部播放器（VLC / PotPlayer / IINA / mpv / nPlayer）URL Scheme - [x] WebDAV 只读网关（RFC 4918，按设备签发的 Basic 访问令牌，独立于 JWT） - [x] JWT 认证 + 刷新令牌 +- [x] 家庭账户与独立档案（Admin / Member / Viewer、PIN、会话撤销、独立播放/聊天状态） - [x] AI 元数据推断（OpenAI / Anthropic）— 自动识别 TMDB ID、季度、集数、字幕组 - [x] 本地 Agent 执行模式（Codex app-server）— 同时用于元数据推断与对话助手 - [x] AI 对话助手：流式响应 + 7 个内置工具（动画 / 订阅 / 季度 / 下载 / 任务 / 文件查询） @@ -90,7 +91,7 @@ bash <(curl -fsSL https://raw.githubusercontent.com/HCGStudio/SecondDimensionWat |--------|------| | `ConnectionStrings:sdw` | PostgreSQL 连接字符串 | | `JwtSecret` | JWT 签名密钥 | -| `Password:Value` | 登录密码的 BCrypt 哈希；为空时允许首次注册写入 `password.json` | +| `Password:Value` | 旧版单站点密码的 BCrypt 哈希；存在时禁止公开注册，仅允许 `admin` 首次登录并迁移到数据库账户 | | `DataProtection:KeyRingPath` | 网页保存的 API key/密码所用加密密钥环；必须位于持久化目录 | | `Torrent:Remote:Url` | qBittorrent API 地址 | | `FileStore:Local` | 下载文件存储根目录 | @@ -118,6 +119,16 @@ bash <(curl -fsSL https://raw.githubusercontent.com/HCGStudio/SecondDimensionWat 设置页提交的 API key 和密码会经过浏览器与服务端之间的连接；除严格的本机访问外，必须为网页入口配置 HTTPS。配置带凭据的 AI 或 qBittorrent 端点时也应使用 TLS，或将明文 HTTP 严格限制在受信任的隔离网络内。 +### 家庭账户、档案与设备访问 + +首次安装由注册页创建管理员和默认档案；旧实例若仍配置 `Password:Value`，注册入口会保持关闭，使用用户名 `admin` 和原密码首次登录后才会安全迁移。右上角档案菜单可即时切换档案，「账户与档案」页可管理名称、头像、可选 PIN、家庭用户和登录会话。档案切换会轮换访问/刷新令牌，并清除浏览器中上一档案的播放、聊天等缓存；多个标签页通过 Web Locks 与浏览器消息同步轮换结果。 + +角色权限由服务端强制执行：Admin 可管理全局设置、用户、任务、元数据和设备凭据；Member 可管理订阅、下载任务和播放状态，但删除已下载文件仍需近期管理员验证；Viewer 仅可浏览和播放。敏感管理操作在超过近期验证窗口后会要求再次输入账户密码，无需退出登录。 + +管理员可在「设置 → 访问协议」为指定家庭用户签发 WebDAV/VFS 设备凭据。每个凭据固定为只读，可限制虚拟根路径并设置到期时间；撤销、到期和路径边界同时由 WebDAV 与 VFS 强制执行。路径 `/Anime` 不会授权 `/Anime2`，客户端看到的根目录和 WebDAV href 会重写到所授权的命名空间。 + +升级迁移会把旧播放进度、偏好和聊天记录归入默认 `Home` 档案。旧数据库结构无法表达多用户、档案归属、token 根路径、到期或撤销状态；因此一旦创建了新身份数据或受限设备凭据，向该迁移之前降级会在删除任何列之前明确失败并保持数据库原样，避免静默合并历史或扩大已撤销凭据权限。 + ### 使用本地 Codex app-server 需要 Codex app-server 0.144.5 或兼容版本提供实验性的 `permissionProfile/list` 与 `permissions` 协议。应用默认请求 `:read-only` 权限配置，并在每次创建 thread 后核验服务端实际返回 `readOnly` 且 agent network access 为 `false`；服务端不支持该协议、配置不可用或结果更宽松时会拒绝执行。当前 `:read-only` **不会把主机文件读取范围收窄到空目录**，而 agent sandbox 的网络开关也不限制 app-server 自身访问模型 API，所以仍必须把进程当作能够读取其操作系统账号可读文件的服务来隔离。 diff --git a/SecondDimensionWatcherReDive.Client/package.json b/SecondDimensionWatcherReDive.Client/package.json index 5f71a25..a9203a0 100644 --- a/SecondDimensionWatcherReDive.Client/package.json +++ b/SecondDimensionWatcherReDive.Client/package.json @@ -33,6 +33,7 @@ "@parcel/core": "^2.16.4", "@parcel/transformer-inline-string": "2.16.4", "@trivago/prettier-plugin-sort-imports": "^6.0.2", + "@types/node": "^26.4.0", "@types/react": "^19.2.18", "@types/react-dom": "^19.2.5", "@yarnpkg/sdks": "^3.3.1", @@ -52,7 +53,7 @@ "build": "rimraf dist && parcel build --no-source-maps", "mock": "node mock-server.mjs", "dev": "node mock-server.mjs & parcel --no-cache", - "test": "tsx --test src/playback/mkv/*.test.ts" + "test": "tsx --test src/auth/*.test.ts src/playback/mkv/*.test.ts" }, "source": "src/index.html", "@parcel/resolver-default": { diff --git a/SecondDimensionWatcherReDive.Client/src/Main.tsx b/SecondDimensionWatcherReDive.Client/src/Main.tsx index 47a5f24..5fe4209 100644 --- a/SecondDimensionWatcherReDive.Client/src/Main.tsx +++ b/SecondDimensionWatcherReDive.Client/src/Main.tsx @@ -3,7 +3,9 @@ import { useTranslation } from "react-i18next"; import { createBrowserRouter } from "react-router"; import { RouterProvider } from "react-router/dom"; +import { useAuthSynchronization } from "./auth/hooks"; import { ProtectedRoute } from "./components/ProtectedRoute"; +import { AccountPage } from "./pages/AccountPage"; import { ChatPage } from "./pages/ChatPage"; import { DownloadedPage } from "./pages/DownloadedPage"; import { DownloadingPage } from "./pages/DownloadingPage"; @@ -136,6 +138,15 @@ const router = createBrowserRouter([ ), errorElement: , }, + { + path: "/account", + element: ( + + + + ), + errorElement: , + }, { path: "/settings", element: ( @@ -153,6 +164,7 @@ const router = createBrowserRouter([ ]); export const Main: React.FC = () => { + useAuthSynchronization(); const { t } = useTranslation(); React.useEffect(() => { document.title = `${t("appName")} Re:Dive`; diff --git a/SecondDimensionWatcherReDive.Client/src/accounts/api.ts b/SecondDimensionWatcherReDive.Client/src/accounts/api.ts new file mode 100644 index 0000000..d949a4a --- /dev/null +++ b/SecondDimensionWatcherReDive.Client/src/accounts/api.ts @@ -0,0 +1,58 @@ +import { IAuthProfile, UserRole } from "../auth/IAuthResult"; +import fetcher from "../auth/httpClient"; +import { IUserAccount } from "./types"; + +export const createProfile = (value: { + name: string; + avatar?: string; + pin?: string; +}) => + fetcher("/api/accounts/profiles", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(value), + }); + +export const updateProfile = ( + id: string, + value: { + name: string; + avatar?: string; + pin?: string; + currentPin?: string; + replacePin: boolean; + }, +) => + fetcher(`/api/accounts/profiles/${id}`, { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(value), + }); + +export const revokeSession = (id: string, asAdministrator = false) => + fetcher(`/api/accounts/sessions/${id}${asAdministrator ? "/admin" : ""}`, { + method: "DELETE", + }); + +export const createUser = (value: { + username: string; + password: string; + role: UserRole; + profileName: string; +}) => + fetcher("/api/accounts/users", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(value), + }); + +export const updateUserAccess = ( + id: string, + role: UserRole, + isDisabled: boolean, +) => + fetcher(`/api/accounts/users/${id}`, { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ role, isDisabled }), + }); diff --git a/SecondDimensionWatcherReDive.Client/src/accounts/hooks.ts b/SecondDimensionWatcherReDive.Client/src/accounts/hooks.ts new file mode 100644 index 0000000..a117af3 --- /dev/null +++ b/SecondDimensionWatcherReDive.Client/src/accounts/hooks.ts @@ -0,0 +1,23 @@ +import useSWR from "swr"; + +import { IAuthProfile } from "../auth/IAuthResult"; +import fetcher from "../auth/httpClient"; +import { IAccountSession, IUserAccount } from "./types"; + +export const useProfiles = () => + useSWR("/api/accounts/profiles", fetcher); + +export const useSessions = () => + useSWR("/api/accounts/sessions", fetcher); + +export const useUsers = (isAdministrator: boolean) => + useSWR( + isAdministrator ? "/api/accounts/users" : null, + fetcher, + ); + +export const useAllSessions = (isAdministrator: boolean) => + useSWR( + isAdministrator ? "/api/accounts/sessions/all" : null, + fetcher, + ); diff --git a/SecondDimensionWatcherReDive.Client/src/accounts/types.ts b/SecondDimensionWatcherReDive.Client/src/accounts/types.ts new file mode 100644 index 0000000..2ecdea2 --- /dev/null +++ b/SecondDimensionWatcherReDive.Client/src/accounts/types.ts @@ -0,0 +1,25 @@ +import { IAuthProfile, UserRole } from "../auth/IAuthResult"; + +export interface IAccountSession { + id: string; + userId: string; + username: string; + profileId: string; + profileName: string; + deviceName?: string; + authenticatedAt: string; + createdAt: string; + lastSeenAt: string; + expiresAt: string; + revokedAt?: string; + isCurrent: boolean; +} + +export interface IUserAccount { + id: string; + username: string; + role: UserRole; + isDisabled: boolean; + createdAt: string; + profiles: IAuthProfile[]; +} diff --git a/SecondDimensionWatcherReDive.Client/src/auth/IAuthResult.ts b/SecondDimensionWatcherReDive.Client/src/auth/IAuthResult.ts index 6374424..5f5a15b 100644 --- a/SecondDimensionWatcherReDive.Client/src/auth/IAuthResult.ts +++ b/SecondDimensionWatcherReDive.Client/src/auth/IAuthResult.ts @@ -2,4 +2,25 @@ export interface IAuthResult { token: string; refreshToken: string; success: boolean; + sessionId?: string; + profileId?: string; +} + +export type UserRole = "Admin" | "Member" | "Viewer"; + +export interface IAuthProfile { + id: string; + name: string; + avatar?: string; + hasPin: boolean; + isDefault: boolean; +} + +export interface IAuthState { + userId: string; + username: string; + role: UserRole; + sessionId: string; + profileId: string; + profiles: IAuthProfile[]; } diff --git a/SecondDimensionWatcherReDive.Client/src/auth/hooks.ts b/SecondDimensionWatcherReDive.Client/src/auth/hooks.ts index 962f326..57e8102 100644 --- a/SecondDimensionWatcherReDive.Client/src/auth/hooks.ts +++ b/SecondDimensionWatcherReDive.Client/src/auth/hooks.ts @@ -1,5 +1,63 @@ -import useSwr from "swr"; +import React from "react"; +import useSwr, { mutate } from "swr"; + +import { IAuthState } from "./IAuthResult"; +import { AuthChangeDetail, subscribeToAuthChanges } from "./httpClient"; export const useAllowRegister = () => useSwr<{ allow: boolean }>("/api/auth/allowRegister"); -export const useLoginStatus = () => useSwr("/api/auth/verify"); +export const useLoginStatus = () => useSwr("/api/auth/verify"); +export const useAccess = () => { + const { data } = useLoginStatus(); + return { + isAdministrator: data?.role === "Admin", + canContentWrite: data?.role === "Admin" || data?.role === "Member", + canPlaybackWrite: data?.role === "Admin" || data?.role === "Member", + }; +}; + +type CacheMutator = ( + key: string | ((key: unknown) => boolean), + data?: unknown, + options?: { revalidate?: boolean }, +) => Promise; + +export const applyAuthChange = async ( + { auth, profileChanged }: AuthChangeDetail, + mutateCache: CacheMutator = mutate as CacheMutator, + redirectToLogin: () => void = () => window.location.assign("/login"), + reloadForProfileChange: () => void = () => window.location.reload(), +) => { + const apiKeys = (key: unknown) => { + const candidate = Array.isArray(key) ? key[0] : key; + return typeof candidate === "string" && candidate.startsWith("/api/"); + }; + if (!auth || profileChanged) { + // Remove every profile-scoped response before any component can render + // under the new identity. + await mutateCache(apiKeys, undefined, { revalidate: false }); + } + + if (!auth) { + if (window.location.pathname !== "/login") redirectToLogin(); + return; + } + + if (profileChanged) { + // A reload is a security boundary: it unmounts the player/chat and all + // profile-owned local state before the replacement identity can render. + reloadForProfileChange(); + } else { + await mutateCache("/api/auth/verify"); + } +}; + +export const useAuthSynchronization = () => { + React.useEffect( + () => + subscribeToAuthChanges((detail) => { + void applyAuthChange(detail); + }), + [], + ); +}; diff --git a/SecondDimensionWatcherReDive.Client/src/auth/httpClient.test.ts b/SecondDimensionWatcherReDive.Client/src/auth/httpClient.test.ts new file mode 100644 index 0000000..fd412fb --- /dev/null +++ b/SecondDimensionWatcherReDive.Client/src/auth/httpClient.test.ts @@ -0,0 +1,291 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { IAuthResult } from "./IAuthResult"; + +class MemoryStorage implements Storage { + private readonly values = new Map(); + get length() { + return this.values.size; + } + clear() { + this.values.clear(); + } + getItem(key: string) { + return this.values.get(key) ?? null; + } + key(index: number) { + return [...this.values.keys()][index] ?? null; + } + removeItem(key: string) { + this.values.delete(key); + } + setItem(key: string, value: string) { + this.values.set(key, value); + } +} + +class ExclusiveLocks { + private tail: Promise = Promise.resolve(); + + request( + _name: string, + _options: { mode: "exclusive" }, + callback: () => Promise, + ): Promise { + const result = this.tail.then(callback); + this.tail = result.catch(() => undefined); + return result; + } +} + +const stale: IAuthResult = { + token: "access-a", + refreshToken: "refresh-a", + sessionId: "session", + profileId: "profile-a", + success: true, +}; +const fresh: IAuthResult = { + token: "access-b", + refreshToken: "refresh-b", + sessionId: "session", + profileId: "profile-a", + success: true, +}; + +test("cross-tab refresh lock serializes rotation and reuses the winner", async () => { + const memoryStorage = new MemoryStorage(); + const windowTarget = new EventTarget() as EventTarget & { + location: { pathname: string; href: string; assign(path: string): void }; + }; + windowTarget.location = { + pathname: "/", + href: "/", + assign(path: string) { + this.href = path; + }, + }; + Object.defineProperty(globalThis, "localStorage", { + configurable: true, + value: memoryStorage, + }); + Object.defineProperty(globalThis, "window", { + configurable: true, + value: windowTarget, + }); + Object.defineProperty(globalThis, "navigator", { + configurable: true, + value: { locks: new ExclusiveLocks() }, + }); + Object.defineProperty(globalThis, "BroadcastChannel", { + configurable: true, + value: undefined, + }); + if (typeof CustomEvent === "undefined") { + class TestCustomEvent extends Event { + constructor( + type: string, + readonly init: CustomEventInit, + ) { + super(type); + } + get detail() { + return this.init.detail as T; + } + } + Object.defineProperty(globalThis, "CustomEvent", { + configurable: true, + value: TestCustomEvent, + }); + } + + let refreshCalls = 0; + Object.defineProperty(globalThis, "fetch", { + configurable: true, + value: async () => { + refreshCalls++; + await Promise.resolve(); + return new Response(JSON.stringify(fresh), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + }, + }); + + const auth = await import("./httpClient"); + auth.setAuthResult(stale); + + const [first, second] = await Promise.all([ + auth.refreshAuthSession(stale), + auth.refreshAuthSession(stale), + ]); + + assert.equal(refreshCalls, 1); + assert.deepEqual(first, fresh); + assert.deepEqual(second, fresh); + assert.deepEqual(JSON.parse(memoryStorage.getItem("auth")!), fresh); +}); + +test("profile changes clear array-keyed caches then reload and logout redirects", async () => { + const { applyAuthChange } = await import("./hooks"); + const calls: Array<{ + key: string | ((key: unknown) => boolean); + options?: { revalidate?: boolean }; + }> = []; + let reloaded = false; + const mutate = async ( + key: string | ((key: unknown) => boolean), + _data?: unknown, + options?: { revalidate?: boolean }, + ) => { + calls.push({ key, options }); + }; + + await applyAuthChange( + { auth: { ...fresh, profileId: "profile-b" }, profileChanged: true }, + mutate, + undefined, + () => { + reloaded = true; + }, + ); + + assert.equal(calls.length, 1); + assert.equal(typeof calls[0].key, "function"); + assert.equal(calls[0].options?.revalidate, false); + const apiKeyPredicate = calls[0].key as (key: unknown) => boolean; + assert.equal(apiKeyPredicate(["/api/chat/conversations", "profile-a"]), true); + assert.equal(apiKeyPredicate(["settings", "profile-a"]), false); + assert.equal(reloaded, true); + + let redirected = false; + calls.length = 0; + await applyAuthChange({ auth: null, profileChanged: true }, mutate, () => { + redirected = true; + }); + assert.equal(calls.length, 1); + assert.equal(calls[0].options?.revalidate, false); + assert.equal(redirected, true); +}); + +test("a late refresh response cannot overwrite a newer shared identity", async () => { + const auth = await import("./httpClient"); + auth.setAuthResult(fresh); + let finishRefresh: ((response: Response) => void) | undefined; + Object.defineProperty(globalThis, "fetch", { + configurable: true, + value: () => + new Promise((resolve) => { + finishRefresh = resolve; + }), + }); + + const refresh = auth.refreshAuthSession(fresh); + await Promise.resolve(); + const replacement: IAuthResult = { + ...fresh, + token: "access-new-session", + refreshToken: "refresh-new-session", + sessionId: "session-new", + profileId: "profile-new", + }; + // This is the shared-storage write made by another realm; intentionally do + // not dispatch storage yet, reproducing the narrow response/notification race. + localStorage.setItem("auth", JSON.stringify(replacement)); + finishRefresh?.( + new Response( + JSON.stringify({ + ...fresh, + token: "late-access-a", + refreshToken: "late-refresh-a", + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), + ); + + await assert.rejects(refresh, auth.AuthIdentityChangedError); + assert.deepEqual(JSON.parse(localStorage.getItem("auth")!), replacement); + + // Restore the original realm for the remaining state-machine tests. + localStorage.setItem("auth", JSON.stringify(fresh)); +}); + +test("Viewer playback is read-only while writable roles retain profile mutations", async () => { + const auth = await import("./httpClient"); + auth.setAuthResult(fresh); + const identity = auth.getAuthIdentityKey(); + + assert.equal(auth.canSendProfileMutation(identity, false), false); + assert.equal(auth.canSendProfileMutation(identity, true), true); +}); + +test("late logout cleanup preserves a replacement login session", async () => { + const auth = await import("./httpClient"); + auth.setAuthResult(fresh); + + assert.equal(auth.clearAuthForSession("an-older-session"), false); + assert.deepEqual(JSON.parse(localStorage.getItem("auth")!), fresh); +}); + +test("external-tab storage profile change aborts streams and forbids 401 replay", async () => { + const auth = await import("./httpClient"); + auth.setAuthResult(fresh); + const oldIdentity = auth.getAuthIdentityKey(); + const stream = auth.beginAuthBoundRequest(true); + let observedChange: import("./httpClient").AuthChangeDetail | undefined; + const unsubscribe = auth.subscribeToAuthChanges((detail) => { + observedChange = detail; + }); + + let finishFirstRequest: ((response: Response) => void) | undefined; + let fetchCalls = 0; + Object.defineProperty(globalThis, "fetch", { + configurable: true, + value: () => { + fetchCalls += 1; + return new Promise((resolve) => { + finishFirstRequest = resolve; + }); + }, + }); + + const oldMutation = auth.default("/api/playback/progress", { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: "{}", + }); + await Promise.resolve(); + assert.equal(fetchCalls, 1); + + const remoteProfile: IAuthResult = { + ...fresh, + token: "access-profile-b", + refreshToken: "refresh-profile-b", + profileId: "profile-b", + }; + localStorage.setItem("auth", JSON.stringify(remoteProfile)); + const storageEvent = new Event("storage"); + Object.defineProperty(storageEvent, "key", { value: "auth" }); + window.dispatchEvent(storageEvent); + + assert.equal(observedChange?.profileChanged, true); + assert.equal(observedChange?.auth?.profileId, "profile-b"); + assert.equal( + stream.signal.aborted, + true, + "the old chat/SSE signal is aborted", + ); + assert.equal(auth.canSendProfileMutation(oldIdentity, true), false); + assert.throws( + () => auth.beginAuthBoundRequest(true), + auth.AuthIdentityChangedError, + ); + + finishFirstRequest?.(new Response(null, { status: 401 })); + await assert.rejects(oldMutation, auth.AuthIdentityChangedError); + assert.equal(fetchCalls, 1, "401 was not refreshed or replayed as profile B"); + + unsubscribe(); + stream.dispose(); +}); diff --git a/SecondDimensionWatcherReDive.Client/src/auth/httpClient.ts b/SecondDimensionWatcherReDive.Client/src/auth/httpClient.ts index 1681a53..7fef28a 100644 --- a/SecondDimensionWatcherReDive.Client/src/auth/httpClient.ts +++ b/SecondDimensionWatcherReDive.Client/src/auth/httpClient.ts @@ -1,89 +1,492 @@ import { IAuthResult } from "./IAuthResult"; -import { refreshJwtToken } from "./utils"; +import { refreshJwtToken } from "./sessionApi"; -let authResult: IAuthResult | null = null; +const AUTH_STORAGE_KEY = "auth"; +const AUTH_CHANNEL_NAME = "sdw-auth"; +const AUTH_REFRESH_LOCK = "sdw-auth-refresh"; +const AUTH_CHANGED_EVENT = "sdw-auth-changed"; + +const mutationMethods = new Set(["POST", "PUT", "PATCH", "DELETE"]); + +type AuthSyncMessage = + { type: "updated"; value: IAuthResult } | { type: "cleared" }; + +export interface AuthChangeDetail { + auth: IAuthResult | null; + profileChanged: boolean; +} + +export const getAuthIdentityKey = ( + value: IAuthResult | null = getAuthResult(), +): string | null => + value?.sessionId && value.profileId + ? `${value.sessionId}\u0000${value.profileId}` + : null; + +const hasSameIdentity = ( + left: IAuthResult | null, + right: IAuthResult | null, +): boolean => + Boolean( + left && right && getAuthIdentityKey(left) === getAuthIdentityKey(right), + ); + +let identityTransitionInProgress = false; +const identityRequests = new Map>(); + +const abortIdentityRequests = (identityKey: string | null) => { + if (!identityKey) return; + const controllers = identityRequests.get(identityKey); + if (!controllers) return; + for (const controller of controllers) controller.abort(); + identityRequests.delete(identityKey); +}; + +const notifyAuthChanged = ( + previous: IAuthResult | null, + current: IAuthResult | null, +) => { + if (typeof window === "undefined") return; + const profileChanged = Boolean( + current && + (identityTransitionInProgress || + (previous && !hasSameIdentity(previous, current))), + ); + if (!current || profileChanged) { + // This runs synchronously before React/SWR sees the new identity. It closes + // streams and requests which captured the old profile, and prevents + // beforeunload/cleanup mutations from being sent with the replacement JWT. + identityTransitionInProgress = true; + abortIdentityRequests(getAuthIdentityKey(previous)); + } else if (!previous) { + // A fresh login can resume mutations. Once a profile transition has + // started, duplicate storage/BroadcastChannel delivery must not reopen the + // old page before the synchronization hook reloads it. + identityTransitionInProgress = false; + } + window.dispatchEvent( + new CustomEvent(AUTH_CHANGED_EVENT, { + detail: { + auth: current, + profileChanged, + }, + }), + ); +}; + +export const subscribeToAuthChanges = ( + listener: (detail: AuthChangeDetail) => void, +): (() => void) => { + if (typeof window === "undefined") return () => undefined; + const handler = (event: Event) => + listener((event as CustomEvent).detail); + window.addEventListener(AUTH_CHANGED_EVENT, handler); + return () => window.removeEventListener(AUTH_CHANGED_EVENT, handler); +}; + +const storage = (): Storage | null => + typeof localStorage === "undefined" ? null : localStorage; + +const readStoredAuth = (): IAuthResult | null => { + const raw = storage()?.getItem(AUTH_STORAGE_KEY); + if (!raw) return null; + try { + const parsed = JSON.parse(raw) as IAuthResult; + return parsed?.success && + parsed.token && + parsed.refreshToken && + parsed.sessionId && + parsed.profileId + ? parsed + : null; + } catch { + storage()?.removeItem(AUTH_STORAGE_KEY); + return null; + } +}; + +let authResult: IAuthResult | null = readStoredAuth(); let refreshPromise: Promise | null = null; +let authChannel: BroadcastChannel | null = null; + +const receiveAuthMessage = (message: AuthSyncMessage) => { + const previous = authResult; + authResult = message.type === "updated" ? message.value : null; + notifyAuthChanged(previous, authResult); +}; -function clearAuth() { +if (typeof window !== "undefined") { + window.addEventListener("storage", (event) => { + if (event.key !== AUTH_STORAGE_KEY) return; + const previous = authResult; + authResult = readStoredAuth(); + notifyAuthChanged(previous, authResult); + }); + + if (typeof BroadcastChannel !== "undefined") { + authChannel = new BroadcastChannel(AUTH_CHANNEL_NAME); + authChannel.addEventListener( + "message", + (event: MessageEvent) => { + receiveAuthMessage(event.data); + }, + ); + } +} + +function clearAuth(expectedRefreshToken?: string): boolean { + const current = readStoredAuth(); + if ( + expectedRefreshToken && + current && + current.refreshToken !== expectedRefreshToken + ) { + authResult = current; + return false; + } + const previous = authResult ?? current; authResult = null; - localStorage.removeItem("auth"); + storage()?.removeItem(AUTH_STORAGE_KEY); + authChannel?.postMessage({ type: "cleared" } satisfies AuthSyncMessage); + notifyAuthChanged(previous, null); + return true; } export const setAuthResult = (result: IAuthResult) => { if (result && result.success) { + const previous = authResult ?? readStoredAuth(); authResult = result; - localStorage.setItem("auth", JSON.stringify(result)); + storage()?.setItem(AUTH_STORAGE_KEY, JSON.stringify(result)); + authChannel?.postMessage({ + type: "updated", + value: result, + } satisfies AuthSyncMessage); + notifyAuthChanged(previous, result); } }; +export const getAuthResult = (): IAuthResult | null => + readStoredAuth() ?? authResult; + export { clearAuth }; -async function parseJsonSafe(res: Response): Promise { - const text = await res.text(); - return text ? JSON.parse(text) : undefined; +export const clearAuthForSession = (sessionId?: string): boolean => { + const current = getAuthResult(); + return current && sessionId && current.sessionId !== sessionId + ? false + : clearAuth(); +}; + +export class AuthIdentityChangedError extends Error { + constructor() { + super("Authentication identity changed"); + this.name = "AuthIdentityChangedError"; + } } -export default async function fetcher( - input: RequestInfo, - init?: RequestInit, -): Promise { - if (authResult) { - const res = await fetch(input, { - ...init, - headers: { - ...init?.headers, - Authorization: `Bearer ${authResult.token}`, - }, +export interface AuthBoundRequest { + auth: IAuthResult; + identityKey: string; + signal: AbortSignal; + isCurrent(): boolean; + abort(): void; + dispose(): void; +} + +/** + * Capture the session/profile for a request. Profile changes synchronously + * abort every bound request, including streaming responses. Mutations are not + * allowed once an identity transition has started because component teardown + * must never flush old profile state under the new JWT. + */ +export const beginAuthBoundRequest = ( + mutation = false, + externalSignal?: AbortSignal | null, +): AuthBoundRequest => { + if (mutation && identityTransitionInProgress) { + throw new AuthIdentityChangedError(); + } + const auth = getAuthResult(); + const identityKey = getAuthIdentityKey(auth); + if (!auth || !identityKey) throw new Error("Unauthorized"); + + const controller = new AbortController(); + const controllers = identityRequests.get(identityKey) ?? new Set(); + controllers.add(controller); + identityRequests.set(identityKey, controllers); + + const abortFromExternalSignal = () => controller.abort(); + if (externalSignal?.aborted) controller.abort(); + else + externalSignal?.addEventListener("abort", abortFromExternalSignal, { + once: true, }); - if (res.status !== 401) { - if (!res.ok) { - throw new Error(`${res.status}`); - } - return await parseJsonSafe(res); - } + let disposed = false; + const dispose = () => { + if (disposed) return; + disposed = true; + externalSignal?.removeEventListener("abort", abortFromExternalSignal); + controllers.delete(controller); + if (controllers.size === 0) identityRequests.delete(identityKey); + }; - // Token expired — deduplicate concurrent refresh calls - if (!refreshPromise) { - refreshPromise = refreshJwtToken(authResult).finally(() => { - refreshPromise = null; - }); + return { + auth, + identityKey, + signal: controller.signal, + isCurrent: () => + !controller.signal.aborted && + !identityTransitionInProgress && + getAuthIdentityKey() === identityKey, + abort: () => controller.abort(), + dispose, + }; +}; + +export const canSendProfileMutation = ( + capturedIdentityKey: string | null, + hasWriteAccess = true, +): boolean => + Boolean( + hasWriteAccess && + capturedIdentityKey && + !identityTransitionInProgress && + getAuthIdentityKey() === capturedIdentityKey, + ); + +type LockManagerWithRequest = { + request( + name: string, + options: { mode: "exclusive" }, + callback: () => Promise, + ): Promise; +}; + +const withRefreshLock = async (callback: () => Promise): Promise => { + const locks = ( + typeof navigator !== "undefined" + ? (navigator as Navigator & { locks?: LockManagerWithRequest }).locks + : undefined + ) as LockManagerWithRequest | undefined; + return locks + ? locks.request(AUTH_REFRESH_LOCK, { mode: "exclusive" }, callback) + : callback(); +}; + +/** + * Rotating refresh tokens are shared through localStorage. The Web Lock makes + * the read/rotate/write sequence atomic across tabs; the second tab observes + * and reuses the token produced by the first instead of replaying its old one. + */ +export const refreshAuthSession = async ( + staleAuth: IAuthResult, +): Promise => + withRefreshLock(async () => { + const current = readStoredAuth(); + if (current && current.refreshToken !== staleAuth.refreshToken) { + if (!hasSameIdentity(current, staleAuth)) { + throw new AuthIdentityChangedError(); + } + authResult = current; + return current; } try { - const newAuth = await refreshPromise; - setAuthResult(newAuth); - } catch { - clearAuth(); - window.location.href = "/login"; - throw new Error("Unauthorized"); + const refreshInput = current ?? staleAuth; + const refreshed = await refreshJwtToken(refreshInput); + if (!refreshed.success || !refreshed.token || !refreshed.refreshToken) { + throw new Error("Unauthorized"); + } + if (!hasSameIdentity(refreshed, refreshInput)) { + throw new AuthIdentityChangedError(); + } + const beforeCommit = readStoredAuth(); + if (!beforeCommit || !hasSameIdentity(beforeCommit, refreshInput)) { + throw new AuthIdentityChangedError(); + } + if (beforeCommit.refreshToken !== refreshInput.refreshToken) { + // A lockless/concurrent same-identity refresh already won. Preserve its + // newer rotation rather than rolling shared storage backwards. + authResult = beforeCommit; + return beforeCommit; + } + setAuthResult(refreshed); + return refreshed; + } catch (error) { + // A browser without Web Locks can still receive a concurrent tab's + // BroadcastChannel/storage update before its failed request completes. + const latest = readStoredAuth(); + if (latest && latest.refreshToken !== staleAuth.refreshToken) { + if (!hasSameIdentity(latest, staleAuth)) { + throw new AuthIdentityChangedError(); + } + authResult = latest; + return latest; + } + clearAuth(staleAuth.refreshToken); + throw error; } + }); - // Retry with new token - const retryRes = await fetch(input, { - ...init, - headers: { - ...init?.headers, - Authorization: `Bearer ${authResult.token}`, - }, - }); +const isSuccessfulAuth = (value: IAuthResult): boolean => + value.success && Boolean(value.token) && Boolean(value.refreshToken); + +/** Serialize endpoints which themselves rotate the current refresh token. */ +export const rotateAuthenticatedSession = async ( + path: string, + body: (auth: IAuthResult) => unknown, +): Promise => + withRefreshLock(async () => { + let current = getAuthResult(); + if (!current) throw new Error("Unauthorized"); + const operationIdentity = getAuthIdentityKey(current); + + const send = (auth: IAuthResult) => + fetch(path, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${auth.token}`, + }, + body: JSON.stringify(body(auth)), + }); - if (retryRes.status === 401) { - clearAuth(); - window.location.href = "/login"; - throw new Error("Unauthorized"); + let response = await send(current); + const afterResponse = getAuthResult(); + if ( + getAuthIdentityKey(afterResponse) !== operationIdentity || + afterResponse?.refreshToken !== current.refreshToken + ) { + throw new AuthIdentityChangedError(); + } + if (response.status === 401) { + const latest = readStoredAuth(); + if (latest && latest.refreshToken !== current.refreshToken) { + if (!hasSameIdentity(latest, current)) { + throw new AuthIdentityChangedError(); + } + current = latest; + } else { + const refreshInput = current; + const refreshed = await refreshJwtToken(refreshInput); + if (!isSuccessfulAuth(refreshed)) throw new Error("Unauthorized"); + if (getAuthIdentityKey(refreshed) !== operationIdentity) { + throw new AuthIdentityChangedError(); + } + const shared = readStoredAuth(); + if (!shared || !hasSameIdentity(shared, refreshInput)) { + throw new AuthIdentityChangedError(); + } + if (shared.refreshToken !== refreshInput.refreshToken) { + current = shared; + } else { + current = refreshed; + setAuthResult(current); + } + } + response = await send(current); + if (getAuthIdentityKey() !== operationIdentity) { + throw new AuthIdentityChangedError(); + } } - if (!retryRes.ok) { - throw new Error(`${retryRes.status}`); + if (!response.ok) throw new Error(`${response.status}`); + const rotated = (await response.json()) as IAuthResult; + if (!isSuccessfulAuth(rotated)) throw new Error("Unauthorized"); + const commitAuth = getAuthResult(); + if ( + getAuthIdentityKey(commitAuth) !== operationIdentity || + commitAuth?.refreshToken !== current.refreshToken + ) { + throw new AuthIdentityChangedError(); } + setAuthResult(rotated); + return rotated; + }); - return await parseJsonSafe(retryRes); - } +async function parseJsonSafe(res: Response): Promise { + const text = await res.text(); + return text ? (JSON.parse(text) as T) : (undefined as T); +} - if (localStorage.getItem("auth")) { - authResult = JSON.parse(localStorage.getItem("auth")!); - return await fetcher(input, init); +export default async function fetcher( + input: RequestInfo, + init?: RequestInit, +): Promise { + const currentAuth = getAuthResult(); + if (currentAuth) { + const method = (init?.method ?? "GET").toUpperCase(); + const bound = beginAuthBoundRequest( + mutationMethods.has(method), + init?.signal, + ); + authResult = currentAuth; + const send = (auth: IAuthResult) => + fetch(input, { + ...init, + signal: bound.signal, + headers: { + ...init?.headers, + Authorization: `Bearer ${auth.token}`, + }, + }); + + try { + let authForRequest = bound.auth; + let res = await send(authForRequest); + if (!bound.isCurrent()) throw new AuthIdentityChangedError(); + + if (res.status === 401) { + // Another request/tab may already have refreshed this same identity. + // Reuse that token, but never replay a request across session/profile. + const latest = getAuthResult(); + if (!hasSameIdentity(latest, bound.auth)) { + throw new AuthIdentityChangedError(); + } + if (latest!.token !== authForRequest.token) { + authForRequest = latest!; + } else { + if (!refreshPromise) { + refreshPromise = refreshAuthSession(bound.auth).finally(() => { + refreshPromise = null; + }); + } + authForRequest = await refreshPromise; + if ( + !bound.isCurrent() || + !hasSameIdentity(authForRequest, bound.auth) + ) { + throw new AuthIdentityChangedError(); + } + } + + // Re-check after refresh and again after the retry response. A remote + // profile switch during either await cancels instead of replaying. + if (!bound.isCurrent()) throw new AuthIdentityChangedError(); + res = await send(authForRequest); + if (!bound.isCurrent()) throw new AuthIdentityChangedError(); + if (res.status === 401) { + clearAuth(authForRequest.refreshToken); + throw new Error("Unauthorized"); + } + } + + if (!res.ok) throw new Error(`${res.status}`); + const result = await parseJsonSafe(res); + if (!bound.isCurrent()) throw new AuthIdentityChangedError(); + return result; + } catch (error) { + if ( + identityTransitionInProgress || + getAuthIdentityKey() !== bound.identityKey + ) { + throw new AuthIdentityChangedError(); + } + throw error; + } finally { + bound.dispose(); + } } // No auth available diff --git a/SecondDimensionWatcherReDive.Client/src/auth/sessionApi.ts b/SecondDimensionWatcherReDive.Client/src/auth/sessionApi.ts new file mode 100644 index 0000000..416366c --- /dev/null +++ b/SecondDimensionWatcherReDive.Client/src/auth/sessionApi.ts @@ -0,0 +1,15 @@ +import { IAuthResult } from "./IAuthResult"; + +export const refreshJwtToken = async ( + oldToken: IAuthResult, +): Promise => { + const response = await fetch("/api/auth/refresh", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(oldToken), + }); + if (!response.ok) throw new Error(`${response.status}`); + return (await response.json()) as IAuthResult; +}; diff --git a/SecondDimensionWatcherReDive.Client/src/auth/utils.ts b/SecondDimensionWatcherReDive.Client/src/auth/utils.ts index 66a4b51..1bc2dc8 100644 --- a/SecondDimensionWatcherReDive.Client/src/auth/utils.ts +++ b/SecondDimensionWatcherReDive.Client/src/auth/utils.ts @@ -1,38 +1,87 @@ import { IAuthResult } from "./IAuthResult"; +import fetcher, { + clearAuthForSession, + getAuthResult, + rotateAuthenticatedSession, +} from "./httpClient"; -export const login = async (password: string): Promise => { - const response = await fetch("/api/auth/login", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ password }), - }); - return (await response.json()) as IAuthResult; -}; +export { refreshJwtToken } from "./sessionApi"; + +interface LoginOptions { + username?: string; + deviceName?: string; + profileName?: string; +} -export const refreshJwtToken = async ( - oldToken: IAuthResult, +export const login = async ( + password: string, + options: LoginOptions = {}, ): Promise => { - const response = await fetch("/api/auth/refresh", { + const response = await fetch("/api/auth/login", { method: "POST", headers: { "Content-Type": "application/json", }, - body: JSON.stringify(oldToken), + body: JSON.stringify({ password, ...options }), }); + if (!response.ok) throw new Error(`${response.status}`); return (await response.json()) as IAuthResult; }; export const register = async ( password: string, + options: LoginOptions = {}, ): Promise => { const response = await fetch("/api/auth/register", { method: "POST", headers: { "Content-Type": "application/json", }, - body: JSON.stringify({ password }), + body: JSON.stringify({ password, ...options }), }); + if (!response.ok) throw new Error(`${response.status}`); return (await response.json()) as IAuthResult; }; + +export const switchProfile = (profileId: string, pin?: string) => + rotateAuthenticatedSession("/api/accounts/profiles/switch", (auth) => ({ + profileId, + pin: pin || null, + refreshToken: auth.refreshToken, + })); + +export const reauthenticate = (password: string) => + rotateAuthenticatedSession("/api/auth/reauthenticate", (auth) => ({ + password, + refreshToken: auth.refreshToken, + })); + +export const retryAfterReauthentication = async ( + operation: () => Promise, + promptMessage: string, +): Promise => { + try { + return await operation(); + } catch (error) { + if (!(error instanceof Error) || error.message !== "403") throw error; + const password = window.prompt(promptMessage); + if (!password) throw error; + await reauthenticate(password); + return operation(); + } +}; + +export const logout = async (): Promise => { + const sessionId = getAuthResult()?.sessionId; + try { + await fetcher("/api/auth/logout", { method: "POST" }); + } catch { + // Local logout must remain available if the session is already invalid or + // the server is unreachable. The server revocation above is best-effort. + } finally { + // A late logout from an old tab must not erase a newer login session from + // shared storage. Profile changes within this same session are still + // cleared because the server revocation applies to the whole session. + clearAuthForSession(sessionId); + } +}; diff --git a/SecondDimensionWatcherReDive.Client/src/chat/api.ts b/SecondDimensionWatcherReDive.Client/src/chat/api.ts index b819ea0..45e0b7a 100644 --- a/SecondDimensionWatcherReDive.Client/src/chat/api.ts +++ b/SecondDimensionWatcherReDive.Client/src/chat/api.ts @@ -1,39 +1,25 @@ -const API_BASE = "/api/chat"; +import fetcher from "../auth/httpClient"; -function getAuthHeaders(): HeadersInit { - const authStr = localStorage.getItem("auth"); - if (!authStr) return {}; - try { - const auth = JSON.parse(authStr); - return { Authorization: `Bearer ${auth.token}` }; - } catch { - return {}; - } -} +const API_BASE = "/api/chat"; export async function createConversation(title?: string) { - const res = await fetch(`${API_BASE}/conversations`, { + return await fetcher(`${API_BASE}/conversations`, { method: "POST", - headers: { "Content-Type": "application/json", ...getAuthHeaders() }, + headers: { "Content-Type": "application/json" }, body: JSON.stringify({ title: title ?? null }), }); - if (!res.ok) throw new Error("Failed to create conversation"); - return res.json(); } export async function deleteConversation(id: string) { - const res = await fetch(`${API_BASE}/conversations/${id}`, { + await fetcher(`${API_BASE}/conversations/${id}`, { method: "DELETE", - headers: getAuthHeaders(), }); - if (!res.ok) throw new Error("Failed to delete conversation"); } export async function updateConversationTitle(id: string, title: string) { - const res = await fetch(`${API_BASE}/conversations/${id}`, { + await fetcher(`${API_BASE}/conversations/${id}`, { method: "PATCH", - headers: { "Content-Type": "application/json", ...getAuthHeaders() }, + headers: { "Content-Type": "application/json" }, body: JSON.stringify({ title }), }); - if (!res.ok) throw new Error("Failed to update title"); } diff --git a/SecondDimensionWatcherReDive.Client/src/chat/useStreamingChat.ts b/SecondDimensionWatcherReDive.Client/src/chat/useStreamingChat.ts index 42487de..66fdcf2 100644 --- a/SecondDimensionWatcherReDive.Client/src/chat/useStreamingChat.ts +++ b/SecondDimensionWatcherReDive.Client/src/chat/useStreamingChat.ts @@ -1,4 +1,10 @@ -import { useCallback, useReducer } from "react"; +import { useCallback, useEffect, useReducer, useRef } from "react"; + +import { + AuthBoundRequest, + AuthIdentityChangedError, + beginAuthBoundRequest, +} from "../auth/httpClient"; interface StreamingToolCall { id: string; @@ -81,8 +87,7 @@ function reducer( return { ...state, contentBlocks: state.contentBlocks.map((block) => - block.type === "tool_call" && - block.toolCall.id === action.toolCallId + block.type === "tool_call" && block.toolCall.id === action.toolCallId ? { ...block, toolCall: { ...block.toolCall, result: action.result }, @@ -113,44 +118,43 @@ const initialState: StreamingState = { export function useStreamingChat() { const [state, dispatch] = useReducer(reducer, initialState); + const activeRequestRef = useRef(null); + const requestGenerationRef = useRef(0); const sendMessage = useCallback( async (conversationId: string, content: string, model?: string) => { + activeRequestRef.current?.abort(); + activeRequestRef.current?.dispose(); + const generation = requestGenerationRef.current + 1; + requestGenerationRef.current = generation; dispatch({ type: "start" }); - const authStr = localStorage.getItem("auth"); - if (!authStr) { - dispatch({ type: "error", message: "Not authenticated" }); - return; - } - - let token: string; - try { - token = JSON.parse(authStr).token; - } catch { - dispatch({ type: "error", message: "Invalid auth token" }); - return; - } - + let request: AuthBoundRequest | null = null; try { + request = beginAuthBoundRequest(true); + activeRequestRef.current = request; const response = await fetch( `/api/chat/conversations/${conversationId}/messages`, { method: "POST", headers: { "Content-Type": "application/json", - Authorization: `Bearer ${token}`, + Authorization: `Bearer ${request.auth.token}`, }, body: JSON.stringify({ content, model: model ?? null }), + signal: request.signal, }, ); + if (!request.isCurrent()) throw new AuthIdentityChangedError(); if (!response.ok) { const text = await response.text(); - dispatch({ - type: "error", - message: text || `HTTP ${response.status}`, - }); + if (requestGenerationRef.current === generation) { + dispatch({ + type: "error", + message: text || `HTTP ${response.status}`, + }); + } return; } @@ -166,6 +170,7 @@ export function useStreamingChat() { while (true) { const { done, value } = await reader.read(); + if (!request.isCurrent()) throw new AuthIdentityChangedError(); if (done) break; buffer += decoder.decode(value, { stream: true }); @@ -181,36 +186,48 @@ export function useStreamingChat() { const data = JSON.parse(line.slice(6)); switch (currentEvent) { case "text_delta": - dispatch({ type: "text_delta", text: data.text }); + if (requestGenerationRef.current === generation) { + dispatch({ type: "text_delta", text: data.text }); + } break; case "tool_call_begin": - dispatch({ - type: "tool_call_begin", - id: data.id, - name: data.name, - }); + if (requestGenerationRef.current === generation) { + dispatch({ + type: "tool_call_begin", + id: data.id, + name: data.name, + }); + } break; case "tool_call_delta": - dispatch({ - type: "tool_call_delta", - id: data.id, - argumentsDelta: data.arguments_delta, - }); + if (requestGenerationRef.current === generation) { + dispatch({ + type: "tool_call_delta", + id: data.id, + argumentsDelta: data.arguments_delta, + }); + } break; case "tool_result": - dispatch({ - type: "tool_result", - toolCallId: data.tool_call_id, - name: data.name, - result: data.result, - }); + if (requestGenerationRef.current === generation) { + dispatch({ + type: "tool_result", + toolCallId: data.tool_call_id, + name: data.name, + result: data.result, + }); + } break; case "finished": receivedFinished = true; - dispatch({ type: "finished" }); + if (requestGenerationRef.current === generation) { + dispatch({ type: "finished" }); + } break; case "error": - dispatch({ type: "error", message: data.message }); + if (requestGenerationRef.current === generation) { + dispatch({ type: "error", message: data.message }); + } break; } } catch { @@ -221,20 +238,49 @@ export function useStreamingChat() { } } - if (!receivedFinished) { + if (!receivedFinished && requestGenerationRef.current === generation) { dispatch({ type: "finished" }); } } catch (err) { - dispatch({ - type: "error", - message: err instanceof Error ? err.message : "Unknown error", - }); + if (requestGenerationRef.current !== generation) return; + if ( + err instanceof AuthIdentityChangedError || + (err instanceof DOMException && err.name === "AbortError") + ) { + dispatch({ type: "reset" }); + } else { + dispatch({ + type: "error", + message: err instanceof Error ? err.message : "Unknown error", + }); + } + } finally { + request?.dispose(); + if (activeRequestRef.current === request) { + activeRequestRef.current = null; + } } }, [], ); - const reset = useCallback(() => dispatch({ type: "reset" }), []); + const reset = useCallback(() => { + requestGenerationRef.current += 1; + activeRequestRef.current?.abort(); + activeRequestRef.current?.dispose(); + activeRequestRef.current = null; + dispatch({ type: "reset" }); + }, []); + + useEffect( + () => () => { + requestGenerationRef.current += 1; + activeRequestRef.current?.abort(); + activeRequestRef.current?.dispose(); + activeRequestRef.current = null; + }, + [], + ); return { ...state, sendMessage, reset }; } diff --git a/SecondDimensionWatcherReDive.Client/src/components/AnimationInfo.tsx b/SecondDimensionWatcherReDive.Client/src/components/AnimationInfo.tsx index 82691e1..a0a750f 100644 --- a/SecondDimensionWatcherReDive.Client/src/components/AnimationInfo.tsx +++ b/SecondDimensionWatcherReDive.Client/src/components/AnimationInfo.tsx @@ -30,6 +30,7 @@ import { retryInference, submitDownload, } from "../animation/utils"; +import { useAccess } from "../auth/hooks"; import { setPlaybackWatched } from "../playback/api"; import { usePlaybackStates } from "../playback/hooks"; import { formatBytes, formatFileSize } from "../utils/formatBytes"; @@ -127,6 +128,7 @@ const AutomationDispositionBadge: React.FC<{ const ActionButtons: React.FC<{ value: IAnimationInfo }> = ({ value }) => { const { t } = useTranslation("animation"); + const { canContentWrite, isAdministrator } = useAccess(); const { data: status } = useAnimationDownloadStatus( value.isDownloadTracked && !value.isDownloadFinished ? value.id : null, ); @@ -142,8 +144,9 @@ const ActionButtons: React.FC<{ value: IAnimationInfo }> = ({ value }) => { playbackStates.length > 0 && playbackStates.every((state) => state.isWatched); - const showRetryItem = value.isAiProcessed; + const showRetryItem = isAdministrator && value.isAiProcessed; const showAiReidentifyItem = + isAdministrator && value.isDownloadFinished && value.animation != null && value.season != null && @@ -251,16 +254,20 @@ const ActionButtons: React.FC<{ value: IAnimationInfo }> = ({ value }) => { const hasOverflowItems = showRetryItem || showAiReidentifyItem || - (value.isDownloadTracked && !value.isDownloadFinished && status) || + (canContentWrite && + value.isDownloadTracked && + !value.isDownloadFinished && + status) || (value.isDownloadTracked && value.isDownloadFinished && - !value.isMediaLibraryImport); + !value.isMediaLibraryImport && + isAdministrator); return ( <>
{/* Primary action: icon-only button */} - {!value.isDownloadTracked ? ( + {!value.isDownloadTracked && canContentWrite ? ( ) : null} - {value.isDownloadTracked && !value.isDownloadFinished && status ? ( + {canContentWrite && + value.isDownloadTracked && + !value.isDownloadFinished && + status ? ( <> {status.state === "Downloading" ? ( +
+ {status.username} · {status.role} +
+ {status.profiles.map((profile) => ( + { + if (profile.id === status.profileId) return; + const pin = profile.hasPin + ? window.prompt(t("user.profilePin")) + : undefined; + if (profile.hasPin && pin === null) return; + void switchProfile(profile.id, pin || undefined).then(() => { + window.location.assign("/"); + }); + }} + > + + {profile.name} + + ))} + navigate("/account")}> + + {t("user.manageAccount")} + +
{t("user.language")}
@@ -209,7 +274,7 @@ const UserMenu: React.FC = () => { ))} - + void onLogout()}> {t("user.logout")} @@ -220,9 +285,12 @@ const UserMenu: React.FC = () => { export const AppHeader: React.FC = () => { const { t } = useTranslation(); const { data: status } = useLoginStatus(); - const { data: incidents } = useIncidents({ take: 1 }); + const { data: incidents } = useIncidents({ + take: 1, + enabled: status?.role === "Admin", + }); const navigate = useNavigate(); - const items = createNavItems(incidents?.openCount); + const items = createNavItems(status?.role, incidents?.openCount); return (
@@ -250,7 +318,7 @@ export const AppHeader: React.FC = () => {
{status ? ( - + ) : (
diff --git a/SecondDimensionWatcherReDive.Client/src/components/chat/ChatMessageList.tsx b/SecondDimensionWatcherReDive.Client/src/components/chat/ChatMessageList.tsx index ce6352e..a913123 100644 --- a/SecondDimensionWatcherReDive.Client/src/components/chat/ChatMessageList.tsx +++ b/SecondDimensionWatcherReDive.Client/src/components/chat/ChatMessageList.tsx @@ -3,7 +3,7 @@ import { useTranslation } from "react-i18next"; import { ChatMessageData } from "../../chat/types"; import { StreamingContentBlock } from "../../chat/useStreamingChat"; -import { AssistantGroup, UserBubble, StreamingMessage } from "./ChatMessage"; +import { AssistantGroup, StreamingMessage, UserBubble } from "./ChatMessage"; interface ChatMessageListProps { messages: ChatMessageData[]; @@ -13,13 +13,12 @@ interface ChatMessageListProps { } /** Group consecutive non-user messages into runs. Each user message is its own group. */ -function groupMessages( - messages: ChatMessageData[], -): { type: "user"; message: ChatMessageData }[] | { type: "assistant"; messages: ChatMessageData[] }[] { - const groups: ( - | { type: "user"; message: ChatMessageData } - | { type: "assistant"; messages: ChatMessageData[] } - )[] = []; +type MessageGroup = + | { type: "user"; message: ChatMessageData } + | { type: "assistant"; messages: ChatMessageData[] }; + +function groupMessages(messages: ChatMessageData[]): MessageGroup[] { + const groups: MessageGroup[] = []; for (const msg of messages) { if (msg.role === "system") continue; @@ -61,31 +60,32 @@ export const ChatMessageList: React.FC = ({

{t("emptyTitle")}

-

- {t("emptyHelp")} -

+

{t("emptyHelp")}

)} - {groups.map((group, i) => + {groups.map((group) => group.type === "user" ? ( ) : ( - + ), )} {pendingUserMessage && !messages.some( (m) => m.role === "user" && m.content === pendingUserMessage, ) && ( -
-
-
- {pendingUserMessage} +
+
+
+ {pendingUserMessage} +
-
- )} + )} {isStreaming && ( { const { t } = useTranslation(["settings", "errors"]); const { data, error, mutate } = useWebDavTokens(); + const { data: users } = useUsers(true); const { addToast } = useToast(); const [username, setUsername] = React.useState(""); const [description, setDescription] = React.useState(""); + const [virtualRoot, setVirtualRoot] = React.useState("/"); + const [expiresAt, setExpiresAt] = React.useState(""); + const [userId, setUserId] = React.useState(""); const [creating, setCreating] = React.useState(false); const [created, setCreated] = React.useState(null); @@ -38,9 +44,18 @@ export const WebDavSettingsSection: React.FC = () => { if (creating) return; setCreating(true); try { - const response = await createWebDavToken( - username.trim() || undefined, - description.trim() || undefined, + const response = await retryAfterReauthentication( + () => + createWebDavToken( + username.trim() || undefined, + description.trim() || undefined, + virtualRoot.trim() || "/", + expiresAt + ? new Date(`${expiresAt}T23:59:59`).toISOString() + : undefined, + userId || undefined, + ), + t("settings:system.reauthenticatePrompt"), ); setCreated(response); setUsername(""); @@ -58,7 +73,17 @@ export const WebDavSettingsSection: React.FC = () => { } finally { setCreating(false); } - }, [addToast, creating, description, mutate, t, username]); + }, [ + addToast, + creating, + description, + expiresAt, + mutate, + t, + username, + userId, + virtualRoot, + ]); const remove = React.useCallback( async (token: IWebDavToken) => { @@ -71,7 +96,10 @@ export const WebDavSettingsSection: React.FC = () => { ) return; try { - await deleteWebDavToken(token.id); + await retryAfterReauthentication( + () => deleteWebDavToken(token.id), + t("settings:system.reauthenticatePrompt"), + ); await mutate(); addToast({ title: t("settings:webdav.toast.deleted"), @@ -118,6 +146,29 @@ export const WebDavSettingsSection: React.FC = () => { name: t("settings:webdav.list.columns.description"), render: (value: string | undefined) => value || "-", }, + { + field: "userId", + name: t("settings:webdav.list.columns.user"), + render: (value: string) => + users?.find((user) => user.id === value)?.username ?? value, + }, + { + field: "virtualRoot", + name: t("settings:webdav.list.columns.virtualRoot"), + render: (value: string) => ( + {value} + ), + }, + { + field: "expiresAt", + name: t("settings:webdav.list.columns.expiresAt"), + render: (value: string | undefined, item) => + item.revokedAt + ? t("settings:webdav.list.revoked") + : value + ? new Date(value).toLocaleString() + : "-", + }, { field: "createdAt", name: t("settings:webdav.list.columns.createdAt"), @@ -125,19 +176,20 @@ export const WebDavSettingsSection: React.FC = () => { }, { name: t("settings:webdav.list.columns.actions"), - render: (_value, item) => ( - - ), + render: (_value, item) => + item.revokedAt ? null : ( + + ), width: "60px", }, ]; @@ -156,7 +208,23 @@ export const WebDavSettingsSection: React.FC = () => { icon={} title={t("settings:webdav.create.title")} > -
+
+ + + { onChange={(event) => setUsername(event.target.value)} /> + + setVirtualRoot(event.target.value)} + /> + + + setExpiresAt(event.target.value)} + /> + { @@ -10,32 +14,29 @@ export const useVfsList = (path: string) => { ); }; -function getAuthHeaders(): HeadersInit { - const authStr = localStorage.getItem("auth"); - if (!authStr) return {}; - try { - const auth = JSON.parse(authStr); - return { Authorization: `Bearer ${auth.token}` }; - } catch { - return {}; - } -} - export async function downloadVfsFile( path: string, fileName: string, ): Promise { - const res = await fetch(`/api/vfs/read?path=${encodeURIComponent(path)}`, { - headers: getAuthHeaders(), - }); - if (!res.ok) throw new Error(`${res.status}`); - const blob = await res.blob(); - const url = URL.createObjectURL(blob); - const a = document.createElement("a"); - a.href = url; - a.download = fileName; - document.body.appendChild(a); - a.click(); - a.remove(); - URL.revokeObjectURL(url); + const request = beginAuthBoundRequest(); + try { + const res = await fetch(`/api/vfs/read?path=${encodeURIComponent(path)}`, { + headers: { Authorization: `Bearer ${request.auth.token}` }, + signal: request.signal, + }); + if (!request.isCurrent()) throw new AuthIdentityChangedError(); + if (!res.ok) throw new Error(`${res.status}`); + const blob = await res.blob(); + if (!request.isCurrent()) throw new AuthIdentityChangedError(); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = fileName; + document.body.appendChild(a); + a.click(); + a.remove(); + URL.revokeObjectURL(url); + } finally { + request.dispose(); + } } diff --git a/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/accounts.json b/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/accounts.json new file mode 100644 index 0000000..c88b32f --- /dev/null +++ b/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/accounts.json @@ -0,0 +1,37 @@ +{ + "title": "Account and profiles", + "profiles": "Profiles", + "active": "Active", + "available": "Available", + "pinProtected": "PIN protected", + "noPin": "No PIN", + "editProfile": "Edit name, avatar or PIN", + "switchProfile": "Switch profile", + "createProfile": "Create profile", + "profileName": "Profile name", + "avatar": "Avatar URL", + "pinOptional": "PIN (optional, 4–8 digits)", + "create": "Create", + "pinPrompt": "Enter this profile's PIN", + "replacePinPrompt": "Do you also want to replace or clear this profile's PIN?", + "currentPin": "Enter the current PIN", + "newPin": "Enter a new 4–8 digit PIN, or leave empty to clear it", + "reauthPrompt": "Enter your account password to confirm this sensitive action", + "failed": "The operation failed", + "mySessions": "Your login sessions", + "allSessions": "All login sessions", + "unknownDevice": "Unknown device", + "current": "Current session", + "revoked": "Revoked", + "revoke": "Revoke", + "users": "Household users", + "createUser": "Create user", + "username": "Username", + "password": "Password", + "role": "Role", + "enable": "Enable", + "disable": "Disable", + "deviceTokens": "Device access tokens", + "deviceTokensHelp": "Issue path-scoped, expiring credentials for WebDAV and VFS clients.", + "manageDeviceTokens": "Manage device tokens" +} diff --git a/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/animation.json b/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/animation.json index 3816b50..a9754bb 100644 --- a/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/animation.json +++ b/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/animation.json @@ -56,6 +56,7 @@ "retryDownload": "Retry download", "pause": "Pause", "resume": "Resume", + "cancel": "Cancel task", "browse": "Browse files", "markAllWatched": "Mark every video in this release watched", "markAllUnwatched": "Mark every video in this release unwatched", @@ -70,6 +71,7 @@ "confirm": { "deleteFile": "Delete the downloaded files?", "cancelAndDelete": "Cancel the download and delete the files?", + "cancel": "Cancel the download task? Existing files will be retained.", "forceAiReidentifyFiles": "This will ignore regex rules, use AI to re-identify filenames, and replace the current virtual file mappings. Continue?" }, "toast": { diff --git a/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/auth.json b/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/auth.json index c739ce1..ce45c47 100644 --- a/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/auth.json +++ b/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/auth.json @@ -1,6 +1,8 @@ { "setupTitle": "Set a password", "setupHelp": "This is your first time using SDW Re:Dive. Please choose a password.", + "username": "Username", + "profileName": "Initial profile name", "password": "Password", "passwordPlaceholder": "Enter password", "repeatPassword": "Confirm password", diff --git a/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/common.json b/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/common.json index 541cb5e..83e996d 100644 --- a/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/common.json +++ b/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/common.json @@ -15,6 +15,8 @@ }, "user": { "account": "Account", + "manageAccount": "Manage account", + "profilePin": "Enter this profile's PIN", "language": "Language", "logout": "Sign out", "login": "Sign in" diff --git a/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/settings.json b/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/settings.json index 9c91e39..9d6ee88 100644 --- a/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/settings.json +++ b/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/settings.json @@ -2,6 +2,7 @@ "pageTitle": "System settings", "system": { "pageDescription": "Configure AI execution, external services, media processing, health monitoring, and access protocols. Secret values are never displayed, and leaving a new value blank preserves the current setting.", + "reauthenticatePrompt": "Enter your account password to confirm this sensitive action", "loadFailed": "System settings could not be loaded. Check the service and try again.", "retry": "Reload", "navigation": { @@ -242,11 +243,15 @@ "intro": "Issue per-device WebDAV username and access-token pairs. Tokens are stored as BCrypt hashes in the database and the plaintext is displayed exactly once at creation time. Revoke any pair at any time.", "create": { "title": "Create a new credential", + "userLabel": "Credential owner", + "currentUser": "Current administrator", "usernameLabel": "Username (optional)", "usernamePlaceholder": "Leave blank to auto-generate", "usernameHelp": "Usernames may contain letters, digits, '.', '_' or '-' and must be 3-32 characters long.", "descriptionLabel": "Note (optional)", "descriptionPlaceholder": "e.g. living-room Mac mini", + "virtualRootLabel": "Visible virtual root", + "expiresAtLabel": "Expires on (optional)", "submit": "Generate" }, "created": { @@ -263,12 +268,16 @@ }, "columns": { "username": "Username", + "user": "Owner", "description": "Note", + "virtualRoot": "Visible root", + "expiresAt": "Expiry / status", "createdAt": "Created", "actions": "Actions" }, "deleteConfirm": "Revoke credential \"{{username}}\"? This cannot be undone.", - "deleteAria": "Revoke {{username}}" + "deleteAria": "Revoke {{username}}", + "revoked": "Revoked" }, "toast": { "created": "Credential generated", diff --git a/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/accounts.json b/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/accounts.json new file mode 100644 index 0000000..4c885f0 --- /dev/null +++ b/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/accounts.json @@ -0,0 +1,37 @@ +{ + "title": "アカウントとプロフィール", + "profiles": "プロフィール", + "active": "使用中", + "available": "切り替え可能", + "pinProtected": "PIN 保護あり", + "noPin": "PIN なし", + "editProfile": "名前・アバター・PIN を編集", + "switchProfile": "プロフィールを切り替え", + "createProfile": "プロフィールを作成", + "profileName": "プロフィール名", + "avatar": "アバター URL", + "pinOptional": "PIN（任意、4～8 桁）", + "create": "作成", + "pinPrompt": "このプロフィールの PIN を入力してください", + "replacePinPrompt": "このプロフィールの PIN も変更または解除しますか？", + "currentPin": "現在の PIN を入力してください", + "newPin": "新しい 4～8 桁の PIN（空欄で解除）", + "reauthPrompt": "この重要な操作を確認するため、アカウントのパスワードを入力してください", + "failed": "操作に失敗しました", + "mySessions": "ログインセッション", + "allSessions": "すべてのログインセッション", + "unknownDevice": "不明なデバイス", + "current": "現在のセッション", + "revoked": "失効済み", + "revoke": "失効", + "users": "世帯ユーザー", + "createUser": "ユーザーを作成", + "username": "ユーザー名", + "password": "パスワード", + "role": "ロール", + "enable": "有効化", + "disable": "無効化", + "deviceTokens": "デバイスアクセストークン", + "deviceTokensHelp": "WebDAV / VFS クライアント向けにパスと期限を限定した資格情報を発行します。", + "manageDeviceTokens": "デバイストークンを管理" +} diff --git a/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/animation.json b/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/animation.json index 2184585..4ef7b29 100644 --- a/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/animation.json +++ b/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/animation.json @@ -53,6 +53,7 @@ "retryDownload": "ダウンロードを再試行", "pause": "一時停止", "resume": "再開", + "cancel": "タスクを中止", "browse": "ファイルを開く", "markAllWatched": "このリリースの全動画を視聴済みにする", "markAllUnwatched": "このリリースの全動画を未視聴に戻す", @@ -67,6 +68,7 @@ "confirm": { "deleteFile": "ダウンロード済みのファイルを削除しますか？", "cancelAndDelete": "ダウンロードを中止してファイルを削除しますか？", + "cancel": "ダウンロードタスクを中止しますか？既存ファイルは保持されます。", "forceAiReidentifyFiles": "正規表現ルールを無視して AI でファイル名を再識別し、現在の仮想ファイルマッピングを置き換えます。続行しますか？" }, "toast": { diff --git a/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/auth.json b/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/auth.json index a5c92f7..7394712 100644 --- a/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/auth.json +++ b/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/auth.json @@ -1,6 +1,8 @@ { "setupTitle": "パスワードを設定してください", "setupHelp": "SDW Re:Dive をはじめてご利用になります。パスワードを設定してください。", + "username": "ユーザー名", + "profileName": "最初のプロフィール名", "password": "パスワード", "passwordPlaceholder": "パスワードを入力", "repeatPassword": "パスワードを再入力", diff --git a/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/common.json b/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/common.json index cd05358..f78a876 100644 --- a/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/common.json +++ b/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/common.json @@ -15,6 +15,8 @@ }, "user": { "account": "アカウント", + "manageAccount": "アカウント管理", + "profilePin": "このプロフィールの PIN を入力してください", "language": "言語", "logout": "ログアウト", "login": "ログイン" diff --git a/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/settings.json b/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/settings.json index 40eebfe..3e2eec7 100644 --- a/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/settings.json +++ b/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/settings.json @@ -2,6 +2,7 @@ "pageTitle": "システム設定", "system": { "pageDescription": "AI 実行エンジン、外部サービス、メディア処理、ヘルス監視、アクセスプロトコルを設定します。シークレット値は表示されず、新しい値を空欄にすると現在の設定が保持されます。", + "reauthenticatePrompt": "この重要な操作を確認するため、アカウントのパスワードを入力してください", "loadFailed": "システム設定を読み込めませんでした。サービスを確認して再試行してください。", "retry": "再読み込み", "navigation": { @@ -242,11 +243,15 @@ "intro": "デバイスごとに個別の WebDAV ユーザー名とアクセストークンを発行できます。トークンはデータベースに BCrypt ハッシュで保存され、平文は作成時に一度だけ表示されます。いつでも失効可能です。", "create": { "title": "新しい認証情報を作成", + "userLabel": "認証情報の所有者", + "currentUser": "現在の管理者", "usernameLabel": "ユーザー名（任意）", "usernamePlaceholder": "空欄で自動生成", "usernameHelp": "ユーザー名は英数字、ドット、アンダースコア、ハイフンのみ、3-32 文字。", "descriptionLabel": "メモ（任意）", "descriptionPlaceholder": "例：リビングの Mac mini", + "virtualRootLabel": "表示する仮想ルート", + "expiresAtLabel": "有効期限（任意）", "submit": "生成" }, "created": { @@ -263,12 +268,16 @@ }, "columns": { "username": "ユーザー名", + "user": "所有者", "description": "メモ", + "virtualRoot": "表示ルート", + "expiresAt": "期限 / 状態", "createdAt": "作成日時", "actions": "操作" }, "deleteConfirm": "認証情報「{{username}}」を失効させますか？この操作は取り消せません。", - "deleteAria": "{{username}} を失効" + "deleteAria": "{{username}} を失効", + "revoked": "失効済み" }, "toast": { "created": "認証情報を作成しました", diff --git a/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/accounts.json b/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/accounts.json new file mode 100644 index 0000000..ebcf0aa --- /dev/null +++ b/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/accounts.json @@ -0,0 +1,37 @@ +{ + "title": "账户与档案", + "profiles": "档案", + "active": "当前使用", + "available": "可切换", + "pinProtected": "受 PIN 保护", + "noPin": "无 PIN", + "editProfile": "编辑名称、头像或 PIN", + "switchProfile": "切换档案", + "createProfile": "新建档案", + "profileName": "档案名称", + "avatar": "头像 URL", + "pinOptional": "PIN（可选，4–8 位数字）", + "create": "创建", + "pinPrompt": "请输入此档案的 PIN", + "replacePinPrompt": "是否同时重设或清除此档案的 PIN？", + "currentPin": "请输入当前 PIN", + "newPin": "输入新的 4–8 位 PIN，留空则清除", + "reauthPrompt": "请输入账户密码以确认此敏感操作", + "failed": "操作失败", + "mySessions": "你的登录会话", + "allSessions": "所有登录会话", + "unknownDevice": "未知设备", + "current": "当前会话", + "revoked": "已撤销", + "revoke": "撤销", + "users": "家庭用户", + "createUser": "创建用户", + "username": "用户名", + "password": "密码", + "role": "角色", + "enable": "启用", + "disable": "禁用", + "deviceTokens": "设备访问令牌", + "deviceTokensHelp": "为 WebDAV 和 VFS 客户端签发限定路径且会过期的凭据。", + "manageDeviceTokens": "管理设备令牌" +} diff --git a/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/animation.json b/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/animation.json index 1858136..ad00d5b 100644 --- a/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/animation.json +++ b/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/animation.json @@ -53,6 +53,7 @@ "retryDownload": "重试下载", "pause": "暂停", "resume": "恢复", + "cancel": "取消任务", "browse": "浏览文件", "markAllWatched": "将此版本的全部视频标记为已看", "markAllUnwatched": "将此版本的全部视频标记为未看", @@ -67,6 +68,7 @@ "confirm": { "deleteFile": "确定要删除已下载的文件吗？", "cancelAndDelete": "确定要取消下载并删除文件吗？", + "cancel": "确定要取消下载任务吗？已有文件会保留。", "forceAiReidentifyFiles": "将忽略正则规则并使用 AI 重新识别文件名，现有虚拟文件映射将被替换。确定继续吗？" }, "toast": { diff --git a/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/auth.json b/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/auth.json index f214d68..7bf5678 100644 --- a/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/auth.json +++ b/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/auth.json @@ -1,6 +1,8 @@ { "setupTitle": "请设置密码", "setupHelp": "您是第一次使用二次元观测器，请设置密码。", + "username": "用户名", + "profileName": "初始档案名称", "password": "密码", "passwordPlaceholder": "请输入密码", "repeatPassword": "重复密码", diff --git a/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/common.json b/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/common.json index 5ffff49..8ba2336 100644 --- a/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/common.json +++ b/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/common.json @@ -15,6 +15,8 @@ }, "user": { "account": "账户", + "manageAccount": "管理账户", + "profilePin": "请输入此档案的 PIN", "language": "语言", "logout": "注销", "login": "登录" diff --git a/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/settings.json b/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/settings.json index 1d101be..f26e5b3 100644 --- a/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/settings.json +++ b/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/settings.json @@ -2,6 +2,7 @@ "pageTitle": "系统设置", "system": { "pageDescription": "配置 AI 执行引擎、外部服务、媒体处理、健康监控和访问协议。敏感值不会回显，未填写新值时会保留当前配置。", + "reauthenticatePrompt": "请输入账户密码以确认此敏感操作", "loadFailed": "无法加载系统设置，请检查服务状态后重试。", "retry": "重新加载", "navigation": { @@ -242,11 +243,15 @@ "intro": "在这里为每台设备生成独立的 WebDAV 用户名和访问令牌，可随时撤销。访问令牌在数据库中以 BCrypt 哈希保存，明文仅在创建时显示一次。", "create": { "title": "创建新凭据", + "userLabel": "凭据归属用户", + "currentUser": "当前管理员", "usernameLabel": "用户名（可选）", "usernamePlaceholder": "留空则自动生成", "usernameHelp": "用户名仅允许字母、数字、点、下划线、连字符，长度 3-32。", "descriptionLabel": "备注（可选）", "descriptionPlaceholder": "例如：客厅 Mac mini", + "virtualRootLabel": "可见虚拟根目录", + "expiresAtLabel": "到期日期（可选）", "submit": "生成" }, "created": { @@ -263,12 +268,16 @@ }, "columns": { "username": "用户名", + "user": "归属", "description": "备注", + "virtualRoot": "可见根目录", + "expiresAt": "到期 / 状态", "createdAt": "创建时间", "actions": "操作" }, "deleteConfirm": "确认撤销凭据「{{username}}」吗？此操作不可恢复。", - "deleteAria": "撤销 {{username}}" + "deleteAria": "撤销 {{username}}", + "revoked": "已撤销" }, "toast": { "created": "凭据已生成", diff --git a/SecondDimensionWatcherReDive.Client/src/i18n/resources.ts b/SecondDimensionWatcherReDive.Client/src/i18n/resources.ts index 0533282..1bb8bc0 100644 --- a/SecondDimensionWatcherReDive.Client/src/i18n/resources.ts +++ b/SecondDimensionWatcherReDive.Client/src/i18n/resources.ts @@ -1,3 +1,4 @@ +import enAccounts from "./locales/en/accounts.json"; import enAnimation from "./locales/en/animation.json"; import enAuth from "./locales/en/auth.json"; import enChat from "./locales/en/chat.json"; @@ -11,6 +12,7 @@ import enPlayer from "./locales/en/player.json"; import enSeason from "./locales/en/season.json"; import enSettings from "./locales/en/settings.json"; import enTasks from "./locales/en/tasks.json"; +import jaAccounts from "./locales/ja/accounts.json"; import jaAnimation from "./locales/ja/animation.json"; import jaAuth from "./locales/ja/auth.json"; import jaChat from "./locales/ja/chat.json"; @@ -24,6 +26,7 @@ import jaPlayer from "./locales/ja/player.json"; import jaSeason from "./locales/ja/season.json"; import jaSettings from "./locales/ja/settings.json"; import jaTasks from "./locales/ja/tasks.json"; +import zhCnAccounts from "./locales/zh-CN/accounts.json"; import zhCnAnimation from "./locales/zh-CN/animation.json"; import zhCnAuth from "./locales/zh-CN/auth.json"; import zhCnChat from "./locales/zh-CN/chat.json"; @@ -41,6 +44,7 @@ import zhCnTasks from "./locales/zh-CN/tasks.json"; export const resources = { "zh-cn": { common: zhCnCommon, + accounts: zhCnAccounts, auth: zhCnAuth, errors: zhCnErrors, animation: zhCnAnimation, @@ -56,6 +60,7 @@ export const resources = { }, en: { common: enCommon, + accounts: enAccounts, auth: enAuth, errors: enErrors, animation: enAnimation, @@ -71,6 +76,7 @@ export const resources = { }, ja: { common: jaCommon, + accounts: jaAccounts, auth: jaAuth, errors: jaErrors, animation: jaAnimation, diff --git a/SecondDimensionWatcherReDive.Client/src/incidents/hooks.ts b/SecondDimensionWatcherReDive.Client/src/incidents/hooks.ts index 06d3a02..3228504 100644 --- a/SecondDimensionWatcherReDive.Client/src/incidents/hooks.ts +++ b/SecondDimensionWatcherReDive.Client/src/incidents/hooks.ts @@ -8,6 +8,7 @@ export interface IncidentQuery { skip?: number; take?: number; includeResolved?: boolean; + enabled?: boolean; } export const incidentListKey = ({ @@ -26,6 +27,10 @@ export const incidentListKey = ({ }; export const useIncidents = (query: IncidentQuery = {}) => - useSWR(incidentListKey(query), fetcher, { - refreshInterval: 15_000, - }); + useSWR( + query.enabled === false ? null : incidentListKey(query), + fetcher, + { + refreshInterval: 15_000, + }, + ); diff --git a/SecondDimensionWatcherReDive.Client/src/pages/AccountPage.tsx b/SecondDimensionWatcherReDive.Client/src/pages/AccountPage.tsx new file mode 100644 index 0000000..9d99321 --- /dev/null +++ b/SecondDimensionWatcherReDive.Client/src/pages/AccountPage.tsx @@ -0,0 +1,454 @@ +import React from "react"; +import { useTranslation } from "react-i18next"; +import { useNavigate } from "react-router"; +import { mutate as mutateAll } from "swr"; + +import { KeyRound, Monitor, Plus, Shield, UserRound } from "lucide-react"; + +import { + createProfile, + createUser, + revokeSession, + updateProfile, + updateUserAccess, +} from "../accounts/api"; +import { + useAllSessions, + useProfiles, + useSessions, + useUsers, +} from "../accounts/hooks"; +import { IAccountSession } from "../accounts/types"; +import { IAuthProfile, UserRole } from "../auth/IAuthResult"; +import { useLoginStatus } from "../auth/hooks"; +import { clearAuthForSession } from "../auth/httpClient"; +import { reauthenticate, switchProfile } from "../auth/utils"; +import { Button } from "../components/ui/Button"; +import { Card } from "../components/ui/Card"; +import { FormRow } from "../components/ui/FormRow"; +import { Input } from "../components/ui/Input"; +import { PasswordInput } from "../components/ui/PasswordInput"; +import { PageTemplate } from "./PageTemplate"; + +const roles: UserRole[] = ["Admin", "Member", "Viewer"]; + +export const AccountPage: React.FC = () => { + const { t } = useTranslation("accounts"); + const navigate = useNavigate(); + const { data: status, mutate: mutateStatus } = useLoginStatus(); + const isAdmin = status?.role === "Admin"; + const canCreateProfile = + status?.role === "Admin" || status?.role === "Member"; + const { data: profiles, mutate: mutateProfiles } = useProfiles(); + const { data: sessions, mutate: mutateSessions } = useSessions(); + const { data: users, mutate: mutateUsers } = useUsers(isAdmin); + const { data: allSessions, mutate: mutateAllSessions } = + useAllSessions(isAdmin); + + const [profileName, setProfileName] = React.useState(""); + const [profileAvatar, setProfileAvatar] = React.useState(""); + const [profilePin, setProfilePin] = React.useState(""); + const [username, setUsername] = React.useState(""); + const [password, setPassword] = React.useState(""); + const [newUserProfile, setNewUserProfile] = React.useState("Home"); + const [newUserRole, setNewUserRole] = React.useState("Member"); + const [busy, setBusy] = React.useState(false); + const [error, setError] = React.useState(null); + + const run = React.useCallback( + async (operation: () => Promise) => { + if (busy) return; + setBusy(true); + setError(null); + try { + await operation(); + } catch (operationError) { + setError( + operationError instanceof Error + ? operationError.message + : t("failed"), + ); + } finally { + setBusy(false); + } + }, + [busy, t], + ); + + const stepUp = React.useCallback(async (): Promise => { + const value = window.prompt(t("reauthPrompt")); + if (!value) return false; + await reauthenticate(value); + await mutateStatus(); + return true; + }, [mutateStatus, t]); + + const activate = (profile: IAuthProfile) => + run(async () => { + const pin = profile.hasPin ? window.prompt(t("pinPrompt")) : undefined; + if (profile.hasPin && pin === null) return; + await switchProfile(profile.id, pin || undefined); + await mutateAll(() => true, undefined, { revalidate: false }); + window.location.assign("/"); + }); + + const saveCurrentProfile = (profile: IAuthProfile) => + run(async () => { + const name = window.prompt(t("profileName"), profile.name); + if (!name) return; + const avatar = window.prompt(t("avatar"), profile.avatar ?? ""); + if (avatar === null) return; + const replacePin = window.confirm(t("replacePinPrompt")); + let currentPin: string | undefined; + let pin: string | undefined; + if (replacePin) { + if (profile.hasPin) { + const value = window.prompt(t("currentPin")); + if (value === null) return; + currentPin = value; + } else if (!(await stepUp())) { + return; + } + const value = window.prompt(t("newPin")); + if (value === null) return; + pin = value; + } + await updateProfile(profile.id, { + name, + avatar: avatar || undefined, + currentPin, + pin, + replacePin, + }); + await Promise.all([mutateProfiles(), mutateStatus()]); + }); + + const addProfile = () => + run(async () => { + await createProfile({ + name: profileName, + avatar: profileAvatar || undefined, + pin: profilePin || undefined, + }); + setProfileName(""); + setProfileAvatar(""); + setProfilePin(""); + await Promise.all([mutateProfiles(), mutateStatus()]); + }); + + const removeSession = (session: IAccountSession, asAdmin = false) => + run(async () => { + if (asAdmin && !(await stepUp())) return; + await revokeSession(session.id, asAdmin); + if (session.isCurrent) { + if (clearAuthForSession(session.id)) { + navigate("/login", { replace: true }); + } + return; + } + await Promise.all([mutateSessions(), mutateAllSessions()]); + }); + + const addUser = () => + run(async () => { + if (!(await stepUp())) return; + await createUser({ + username, + password, + role: newUserRole, + profileName: newUserProfile, + }); + setUsername(""); + setPassword(""); + await mutateUsers(); + }); + + return ( + +
+

+ {t("title")} +

+

+ {status?.username} · {status?.role} +

+ {error ?

{error}

: null} +
+ +
+

+ {t("profiles")} +

+
+ {profiles?.map((profile) => { + const active = profile.id === status?.profileId; + return ( + + ) : ( + + ) + } + title={profile.name} + description={`${active ? t("active") : t("available")} · ${ + profile.hasPin ? t("pinProtected") : t("noPin") + }`} + footer={ + active ? ( + canCreateProfile ? ( + + ) : null + ) : ( + + ) + } + /> + ); + })} +
+ + {canCreateProfile ? ( + } + title={t("createProfile")} + > +
+ + setProfileName(event.target.value)} + /> + + + setProfileAvatar(event.target.value)} + /> + + + setProfilePin(event.target.value)} + /> + + + + +
+ + ) : null} +
+ + void removeSession(session)} + /> + + {isAdmin ? ( + <> +
+

+ {t("users")} +

+ } title={t("createUser")}> +
+ + setUsername(event.target.value)} + /> + + + setPassword(event.target.value)} + /> + + + setNewUserProfile(event.target.value)} + /> + + + + + + + +
+ +
+ {users?.map((user) => ( +
+
+
+ {user.username} +
+
+ {user.profiles.map((profile) => profile.name).join(", ")} +
+
+ + +
+ ))} +
+
+ + void removeSession(session, true)} + /> + + } + title={t("deviceTokens")} + > +

{t("deviceTokensHelp")}

+ + + + ) : null} + + ); +}; + +const SessionSection: React.FC<{ + title: string; + sessions?: IAccountSession[]; + busy: boolean; + onRevoke: (session: IAccountSession) => void; +}> = ({ title, sessions, busy, onRevoke }) => { + const { t } = useTranslation("accounts"); + return ( +
+

{title}

+
+ {sessions?.map((session) => ( +
+ +
+
+ {session.deviceName || t("unknownDevice")} + {session.isCurrent ? ` · ${t("current")}` : ""} +
+
+ {session.username} / {session.profileName} ·{" "} + {new Date(session.lastSeenAt).toLocaleString()} + {session.revokedAt ? ` · ${t("revoked")}` : ""} +
+
+ {!session.revokedAt ? ( + + ) : null} +
+ ))} +
+
+ ); +}; diff --git a/SecondDimensionWatcherReDive.Client/src/pages/ChatPage.tsx b/SecondDimensionWatcherReDive.Client/src/pages/ChatPage.tsx index 5fd1877..5829ca1 100644 --- a/SecondDimensionWatcherReDive.Client/src/pages/ChatPage.tsx +++ b/SecondDimensionWatcherReDive.Client/src/pages/ChatPage.tsx @@ -4,6 +4,7 @@ import { useNavigate, useParams } from "react-router"; import { MessageSquare } from "lucide-react"; +import { subscribeToAuthChanges } from "../auth/httpClient"; import { createConversation, deleteConversation } from "../chat/api"; import { useChatModels, @@ -48,6 +49,17 @@ export const ChatPage: React.FC = () => { reset: resetStreaming, } = useStreamingChat(); + useEffect( + () => + subscribeToAuthChanges(({ auth, profileChanged }) => { + if (!auth || profileChanged) { + setPendingUserMessage(null); + resetStreaming(); + } + }), + [resetStreaming], + ); + // Sync URL param with selected conversation useEffect(() => { const fromUrl = conversationId ?? null; diff --git a/SecondDimensionWatcherReDive.Client/src/pages/FeedsPage.tsx b/SecondDimensionWatcherReDive.Client/src/pages/FeedsPage.tsx index e689326..106f079 100644 --- a/SecondDimensionWatcherReDive.Client/src/pages/FeedsPage.tsx +++ b/SecondDimensionWatcherReDive.Client/src/pages/FeedsPage.tsx @@ -3,6 +3,7 @@ import { useTranslation } from "react-i18next"; import { AlertTriangle, Plus, SlidersHorizontal, Trash2 } from "lucide-react"; +import { useAccess } from "../auth/hooks"; import { SubscriptionPolicyModeBadge, SubscriptionPolicySheet, @@ -24,6 +25,7 @@ import { PageTemplate } from "./PageTemplate"; export const FeedsPage: React.FC = () => { const { t } = useTranslation(["feeds", "errors"]); + const { canContentWrite } = useAccess(); const { data: feeds, error, mutate } = useFeeds(); const { data: policies, @@ -115,30 +117,31 @@ export const FeedsPage: React.FC = () => { }, { name: t("feeds:columns.actions"), - render: (_value: any, item: IFeed) => ( -
- - -
- ), + render: (_value: any, item: IFeed) => + canContentWrite ? ( +
+ + +
+ ) : null, width: "190px", }, ]; @@ -150,28 +153,30 @@ export const FeedsPage: React.FC = () => {

{t("feeds:manualSubscribe")}

-
- - setUrl(e.target.value)} - /> - - - setName(e.target.value)} - /> - - - - -
+ {canContentWrite ? ( +
+ + setUrl(e.target.value)} + /> + + + setName(e.target.value)} + /> + + + + +
+ ) : null}
@@ -201,16 +206,18 @@ export const FeedsPage: React.FC = () => { /> ) : null}
- { - if (!open) setSelectedFeed(null); - }} - onPolicyChanged={() => mutatePolicies()} - /> + {canContentWrite ? ( + { + if (!open) setSelectedFeed(null); + }} + onPolicyChanged={() => mutatePolicies()} + /> + ) : null} ); }; diff --git a/SecondDimensionWatcherReDive.Client/src/pages/LoginPage.tsx b/SecondDimensionWatcherReDive.Client/src/pages/LoginPage.tsx index 87bed60..b9c8e7f 100644 --- a/SecondDimensionWatcherReDive.Client/src/pages/LoginPage.tsx +++ b/SecondDimensionWatcherReDive.Client/src/pages/LoginPage.tsx @@ -8,6 +8,7 @@ import { setAuthResult } from "../auth/httpClient"; import { login, register } from "../auth/utils"; import { Button } from "../components/ui/Button"; import { FormRow } from "../components/ui/FormRow"; +import { Input } from "../components/ui/Input"; import { PasswordInput } from "../components/ui/PasswordInput"; import { PageTemplate } from "./PageTemplate"; @@ -16,6 +17,8 @@ export const LoginPage: React.FC = () => { const { data: registerInfo } = useAllowRegister(); const { data: status } = useLoginStatus(); const [password, setPassword] = React.useState(""); + const [username, setUsername] = React.useState("admin"); + const [profileName, setProfileName] = React.useState("Home"); const [passwordConfirm, setPasswordConfirm] = React.useState(""); const [loginFailed, setLoginFailed] = React.useState(false); const [registerFailed, setRegisterFailed] = React.useState(false); @@ -39,10 +42,14 @@ export const LoginPage: React.FC = () => { setIsSubmitting(true); setRegisterFailed(false); try { - const r = await register(password); + const r = await register(password, { + username, + profileName, + deviceName: navigator.userAgent, + }); if (r?.success) { setAuthResult(r); - await mutate("/api/auth/verify", true, { revalidate: false }); + await mutate("/api/auth/verify"); navigate("/"); } else { setRegisterFailed(true); @@ -53,7 +60,7 @@ export const LoginPage: React.FC = () => { setIsSubmitting(false); } }, - [password, passwordConfirm, isSubmitting, navigate], + [password, passwordConfirm, isSubmitting, navigate, profileName, username], ); const onLogin = React.useCallback( @@ -63,10 +70,13 @@ export const LoginPage: React.FC = () => { setIsSubmitting(true); setLoginFailed(false); try { - const r = await login(password); + const r = await login(password, { + username, + deviceName: navigator.userAgent, + }); if (r?.success) { setAuthResult(r); - await mutate("/api/auth/verify", true, { revalidate: false }); + await mutate("/api/auth/verify"); navigate("/"); } else { setLoginFailed(true); @@ -77,7 +87,7 @@ export const LoginPage: React.FC = () => { setIsSubmitting(false); } }, - [password, isSubmitting, navigate], + [password, username, isSubmitting, navigate], ); React.useEffect(() => { @@ -96,6 +106,19 @@ export const LoginPage: React.FC = () => { {t("setupHelp")}

+ + setUsername(event.target.value)} + /> + + + setProfileName(event.target.value)} + /> + { 0) || + (password !== passwordConfirm && + passwordConfirm.length > 0) || registerFailed } - error={[ - registerFailed ? t("registerFailed") : t("mismatch"), - ]} + error={[registerFailed ? t("registerFailed") : t("mismatch")]} > { {t("welcomeBack")}
+ + setUsername(event.target.value)} + /> + [0]; mediaKey: string; + identityKey: string; } interface PendingPreferenceSave { preferences: PlaybackPreferences; version: number; + identityKey: string; } const preferenceAudioOptions: AudioTrackOption[] = [ @@ -224,6 +232,7 @@ export const PlayerPage: React.FC = () => { const [searchParams] = useSearchParams(); const navigate = useNavigate(); const { addToast } = useToast(); + const { canPlaybackWrite } = useAccess(); const file = searchParams.get("file") ?? undefined; const shouldAutoplay = searchParams.get("autoplay") === "1"; @@ -274,6 +283,9 @@ export const PlayerPage: React.FC = () => { const pendingPreferenceRef = React.useRef(null); const preferenceSaveRunningRef = React.useRef(false); const preferenceVersionRef = React.useRef(0); + const playerIdentityRef = React.useRef(getAuthIdentityKey()); + const canPlaybackWriteRef = React.useRef(canPlaybackWrite); + canPlaybackWriteRef.current = canPlaybackWrite; const activeMediaKey = `${animationId ?? ""}\u0000${file ?? ""}`; const activeMediaKeyRef = React.useRef(activeMediaKey); activeMediaKeyRef.current = activeMediaKey; @@ -295,6 +307,31 @@ export const PlayerPage: React.FC = () => { } }, [playbackContext]); + React.useEffect(() => { + if (canPlaybackWrite) return; + pendingProgressRef.current = null; + pendingPreferenceRef.current = null; + preferenceVersionRef.current += 1; + }, [canPlaybackWrite]); + + React.useEffect( + () => + subscribeToAuthChanges(({ auth, profileChanged }) => { + if (auth && !profileChanged) return; + // Keep the identity captured by this mounted player unchanged. Its + // teardown callbacks will therefore discard rather than persist the + // old profile's position/preferences with a replacement token. + pendingProgressRef.current = null; + pendingPreferenceRef.current = null; + preferenceVersionRef.current += 1; + contextRef.current = undefined; + preferencesRef.current = undefined; + lastSyncedTimeRef.current = -1; + artRef.current?.pause(); + }), + [], + ); + React.useEffect(() => { setExternalPlaybackUrl(null); setPlaybackUrl(null); @@ -527,9 +564,25 @@ export const PlayerPage: React.FC = () => { while (pendingProgressRef.current) { const pending = pendingProgressRef.current; pendingProgressRef.current = null; + if ( + !canSendProfileMutation( + pending.identityKey, + canPlaybackWriteRef.current, + ) + ) { + continue; + } try { const state = await savePlaybackProgress(pending.request); - if (activeMediaKeyRef.current !== pending.mediaKey) continue; + if ( + activeMediaKeyRef.current !== pending.mediaKey || + !canSendProfileMutation( + pending.identityKey, + canPlaybackWriteRef.current, + ) + ) { + continue; + } void mutateContext( (current) => (current ? { ...current, state } : current), false, @@ -541,7 +594,13 @@ export const PlayerPage: React.FC = () => { key.startsWith("/api/playback/states?")), ); } catch { - if (activeMediaKeyRef.current === pending.mediaKey) { + if ( + activeMediaKeyRef.current === pending.mediaKey && + canSendProfileMutation( + pending.identityKey, + canPlaybackWriteRef.current, + ) + ) { addToast({ title: i18n.t("player:progress.saveFailed"), color: "warning", @@ -556,6 +615,14 @@ export const PlayerPage: React.FC = () => { const persistCurrentProgress = React.useCallback( (force = false, keepalive = false) => { + const identityKey = playerIdentityRef.current; + if ( + !identityKey || + !canSendProfileMutation(identityKey, canPlaybackWriteRef.current) + ) { + pendingProgressRef.current = null; + return; + } const art = artRef.current; const context = contextRef.current; if (!art || !context) return; @@ -589,13 +656,15 @@ export const PlayerPage: React.FC = () => { // Teardown cannot wait behind an ordinary request. Drop any unsent // intermediate sample and dispatch the final position with keepalive. pendingProgressRef.current = null; - void savePlaybackProgress(request, true).catch(() => undefined); + if (canSendProfileMutation(identityKey, canPlaybackWriteRef.current)) { + void savePlaybackProgress(request, true).catch(() => undefined); + } return; } // Keep at most one unsent sample. Pause/seek events replace older timer // samples, while the single in-flight request preserves write order. - pendingProgressRef.current = { request, mediaKey }; + pendingProgressRef.current = { request, mediaKey, identityKey }; void flushProgressQueue(); }, [flushProgressQueue], @@ -837,9 +906,25 @@ export const PlayerPage: React.FC = () => { while (pendingPreferenceRef.current) { const pending = pendingPreferenceRef.current; pendingPreferenceRef.current = null; + if ( + !canSendProfileMutation( + pending.identityKey, + canPlaybackWriteRef.current, + ) + ) { + continue; + } try { const saved = await savePlaybackPreferences(pending.preferences); - if (preferenceVersionRef.current !== pending.version) continue; + if ( + preferenceVersionRef.current !== pending.version || + !canSendProfileMutation( + pending.identityKey, + canPlaybackWriteRef.current, + ) + ) { + continue; + } preferencesRef.current = saved; void mutateContext( (context) => @@ -847,7 +932,13 @@ export const PlayerPage: React.FC = () => { false, ); } catch { - if (preferenceVersionRef.current === pending.version) { + if ( + preferenceVersionRef.current === pending.version && + canSendProfileMutation( + pending.identityKey, + canPlaybackWriteRef.current, + ) + ) { addToast({ title: i18n.t("player:preferences.saveFailed"), color: "danger", @@ -864,13 +955,25 @@ export const PlayerPage: React.FC = () => { const updatePreferences = React.useCallback( (changes: Partial) => { + const identityKey = playerIdentityRef.current; + if ( + !identityKey || + !canSendProfileMutation(identityKey, canPlaybackWriteRef.current) + ) { + pendingPreferenceRef.current = null; + return; + } const current = preferencesRef.current; if (!current) return; const next: PlaybackPreferences = { ...current, ...changes }; preferencesRef.current = next; const version = preferenceVersionRef.current + 1; preferenceVersionRef.current = version; - pendingPreferenceRef.current = { preferences: next, version }; + pendingPreferenceRef.current = { + preferences: next, + version, + identityKey, + }; void mutateContext( (context) => (context ? { ...context, preferences: next } : context), false, @@ -921,7 +1024,13 @@ export const PlayerPage: React.FC = () => { ); const onToggleWatched = React.useCallback(async () => { - if (!playbackContext) return; + const identityKey = playerIdentityRef.current; + if ( + !playbackContext || + !canSendProfileMutation(identityKey, canPlaybackWriteRef.current) + ) { + return; + } const isWatched = !(playbackContext.state?.isWatched ?? false); setSavingWatched(true); try { @@ -930,6 +1039,9 @@ export const PlayerPage: React.FC = () => { path: playbackContext.media.path, isWatched, }); + if (!canSendProfileMutation(identityKey, canPlaybackWriteRef.current)) { + return; + } await mutateContext( (current) => (current ? { ...current, state } : current), false, @@ -945,7 +1057,9 @@ export const PlayerPage: React.FC = () => { color: "success", }); } catch { - addToast({ title: t("watched.failed"), color: "danger" }); + if (canSendProfileMutation(identityKey, canPlaybackWriteRef.current)) { + addToast({ title: t("watched.failed"), color: "danger" }); + } } finally { setSavingWatched(false); } @@ -1050,23 +1164,25 @@ export const PlayerPage: React.FC = () => {

- + {canPlaybackWrite ? ( + + ) : null} {playbackContext.next ? (
@@ -216,23 +236,32 @@ export const SeasonDiscovery: React.FC = () => {
{seasonData?.lastScrapedAt ? ( - {t("lastUpdated", { time: new Date(seasonData.lastScrapedAt).toLocaleString() })} + {t("lastUpdated", { + time: new Date(seasonData.lastScrapedAt).toLocaleString(), + })} ) : null} - + {isAdministrator ? ( + + ) : null}
{isLoading ? ( -
+
+ +
) : seasonData?.bangumis.length === 0 ? (

{t("empty")}

) : ( @@ -268,24 +297,26 @@ export const SeasonDiscovery: React.FC = () => { {bangumi.title}

- + {canContentWrite ? ( + + ) : null} + + {t("allSubgroups")} + + {canSubscribe ? ( + + ) : null}
); })()} @@ -410,24 +460,26 @@ const SubgroupList: React.FC<{ className="flex items-center justify-between rounded-md border border-border-light p-3" > {sg.name} - + {canSubscribe ? ( + + ) : null}
); })} diff --git a/SecondDimensionWatcherReDive.Client/src/settings/IWebDavToken.ts b/SecondDimensionWatcherReDive.Client/src/settings/IWebDavToken.ts index b42a2f9..96271b1 100644 --- a/SecondDimensionWatcherReDive.Client/src/settings/IWebDavToken.ts +++ b/SecondDimensionWatcherReDive.Client/src/settings/IWebDavToken.ts @@ -1,8 +1,13 @@ export interface IWebDavToken { id: string; + userId: string; username: string; description?: string; createdAt: string; + scope: string; + virtualRoot: string; + expiresAt?: string; + revokedAt?: string; } export interface ICreateWebDavTokenResponse { @@ -11,4 +16,8 @@ export interface ICreateWebDavTokenResponse { token: string; description?: string; createdAt: string; + userId: string; + scope: string; + virtualRoot: string; + expiresAt: string; } diff --git a/SecondDimensionWatcherReDive.Client/src/settings/utils.ts b/SecondDimensionWatcherReDive.Client/src/settings/utils.ts index 580a382..9b7aa5f 100644 --- a/SecondDimensionWatcherReDive.Client/src/settings/utils.ts +++ b/SecondDimensionWatcherReDive.Client/src/settings/utils.ts @@ -1,13 +1,22 @@ import fetcher from "../auth/httpClient"; import { ICreateWebDavTokenResponse } from "./IWebDavToken"; -export const createWebDavToken = (username?: string, description?: string) => +export const createWebDavToken = ( + username?: string, + description?: string, + virtualRoot = "/", + expiresAt?: string, + userId?: string, +) => fetcher("/api/webdav-tokens", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ username: username || null, description: description || null, + virtualRoot, + expiresAt: expiresAt || null, + userId: userId || null, }), }); diff --git a/SecondDimensionWatcherReDive.Client/tsconfig.json b/SecondDimensionWatcherReDive.Client/tsconfig.json index 82c0627..c81e7e9 100644 --- a/SecondDimensionWatcherReDive.Client/tsconfig.json +++ b/SecondDimensionWatcherReDive.Client/tsconfig.json @@ -3,6 +3,7 @@ "target": "esnext", "module": "esnext", "lib": ["esnext", "dom"], + "types": ["node"], "allowJs": false, "jsx": "react-jsx", "noEmit": false, diff --git a/SecondDimensionWatcherReDive.Client/yarn.lock b/SecondDimensionWatcherReDive.Client/yarn.lock index dbd57a3..caa74f5 100644 --- a/SecondDimensionWatcherReDive.Client/yarn.lock +++ b/SecondDimensionWatcherReDive.Client/yarn.lock @@ -415,6 +415,7 @@ __metadata: "@tailwindcss/postcss": "npm:^4.3.3" "@tailwindcss/typography": "npm:^0.5.20" "@trivago/prettier-plugin-sort-imports": "npm:^6.0.2" + "@types/node": "npm:^26.4.0" "@types/react": "npm:^19.2.18" "@types/react-dom": "npm:^19.2.5" "@yarnpkg/sdks": "npm:^3.3.1" @@ -2659,7 +2660,7 @@ __metadata: languageName: node linkType: hard -"@types/node@npm:*": +"@types/node@npm:*, @types/node@npm:^26.4.0": version: 26.4.0 resolution: "@types/node@npm:26.4.0" dependencies: diff --git a/SecondDimensionWatcherReDive.Framework/Authorization/AccessControl.cs b/SecondDimensionWatcherReDive.Framework/Authorization/AccessControl.cs new file mode 100644 index 0000000..0da0e60 --- /dev/null +++ b/SecondDimensionWatcherReDive.Framework/Authorization/AccessControl.cs @@ -0,0 +1,42 @@ +using System.Security.Claims; + +namespace SecondDimensionWatcherReDive.Framework.Authorization; + +public static class AccessPolicies +{ + public const string ContentWrite = nameof(ContentWrite); + public const string PlaybackWrite = nameof(PlaybackWrite); + public const string ChatWrite = nameof(ChatWrite); + public const string Administrator = nameof(Administrator); + public const string RecentAuthentication = nameof(RecentAuthentication); + public const string RecentAdministrator = nameof(RecentAdministrator); +} + +public static class IdentityClaimTypes +{ + public const string UserId = "userId"; + public const string ProfileId = "profileId"; + public const string SessionId = "sessionId"; + public const string AuthenticatedAt = "auth_time"; + public const string DeviceTokenId = "deviceTokenId"; + public const string DeviceScope = "deviceScope"; + public const string VirtualRoot = "virtualRoot"; +} + +public static class IdentityClaimsExtensions +{ + public static bool TryGetUserId(this ClaimsPrincipal principal, out Guid userId) => + TryGetGuid(principal, IdentityClaimTypes.UserId, out userId); + + public static bool TryGetProfileId(this ClaimsPrincipal principal, out Guid profileId) => + TryGetGuid(principal, IdentityClaimTypes.ProfileId, out profileId); + + public static bool TryGetSessionId(this ClaimsPrincipal principal, out Guid sessionId) => + TryGetGuid(principal, IdentityClaimTypes.SessionId, out sessionId); + + private static bool TryGetGuid( + ClaimsPrincipal principal, + string claimType, + out Guid value) => + Guid.TryParse(principal.FindFirst(claimType)?.Value, out value); +} diff --git a/SecondDimensionWatcherReDive.Framework/DataRepository/IChatRepository.cs b/SecondDimensionWatcherReDive.Framework/DataRepository/IChatRepository.cs index 8c94753..be7e26d 100644 --- a/SecondDimensionWatcherReDive.Framework/DataRepository/IChatRepository.cs +++ b/SecondDimensionWatcherReDive.Framework/DataRepository/IChatRepository.cs @@ -2,13 +2,42 @@ namespace SecondDimensionWatcherReDive.Framework.DataRepository; public interface IChatRepository { - Task> GetConversationsAsync(CancellationToken cancellationToken); - Task GetConversationWithMessagesAsync(Guid id, CancellationToken cancellationToken); - Task CreateConversationAsync(string? title, CancellationToken cancellationToken); - Task DeleteConversationAsync(Guid id, CancellationToken cancellationToken); - Task UpdateConversationTitleAsync(Guid id, string title, CancellationToken cancellationToken); - Task AddMessageAsync(Guid conversationId, ChatMessageRecord message, CancellationToken cancellationToken); - Task AddMessagesAsync(Guid conversationId, IEnumerable messages, CancellationToken cancellationToken); - Task> GetMessagesAsync(Guid conversationId, CancellationToken cancellationToken); - Task GetMessageCountAsync(Guid conversationId, CancellationToken cancellationToken); + Task> GetConversationsAsync( + Guid profileId, + CancellationToken cancellationToken); + Task GetConversationWithMessagesAsync( + Guid id, + Guid profileId, + CancellationToken cancellationToken); + Task CreateConversationAsync( + Guid profileId, + string? title, + CancellationToken cancellationToken); + Task DeleteConversationAsync( + Guid id, + Guid profileId, + CancellationToken cancellationToken); + Task UpdateConversationTitleAsync( + Guid id, + Guid profileId, + string title, + CancellationToken cancellationToken); + Task AddMessageAsync( + Guid conversationId, + Guid profileId, + ChatMessageRecord message, + CancellationToken cancellationToken); + Task AddMessagesAsync( + Guid conversationId, + Guid profileId, + IEnumerable messages, + CancellationToken cancellationToken); + Task> GetMessagesAsync( + Guid conversationId, + Guid profileId, + CancellationToken cancellationToken); + Task GetMessageCountAsync( + Guid conversationId, + Guid profileId, + CancellationToken cancellationToken); } diff --git a/SecondDimensionWatcherReDive.Framework/DataRepository/IIdentityRepository.cs b/SecondDimensionWatcherReDive.Framework/DataRepository/IIdentityRepository.cs new file mode 100644 index 0000000..2944e4e --- /dev/null +++ b/SecondDimensionWatcherReDive.Framework/DataRepository/IIdentityRepository.cs @@ -0,0 +1,80 @@ +namespace SecondDimensionWatcherReDive.Framework.DataRepository; + +public interface IIdentityRepository +{ + Task AnyUsersAsync(CancellationToken cancellationToken); + + Task FindUserByIdAsync(Guid id, CancellationToken cancellationToken); + + Task FindUserByUsernameAsync( + string username, + CancellationToken cancellationToken); + + Task FindProfileAsync(Guid id, CancellationToken cancellationToken); + + Task> GetProfilesAsync( + Guid userId, + CancellationToken cancellationToken); + + Task CreateUserWithProfileAsync( + UserAccount user, + UserProfile profile, + CancellationToken cancellationToken); + + Task SetPasswordHashAsync( + Guid userId, + string passwordHash, + DateTimeOffset now, + CancellationToken cancellationToken); + + Task AddProfileAsync( + UserProfile profile, + CancellationToken cancellationToken); + + Task UpdateProfileAsync( + Guid profileId, + Guid userId, + string name, + string? avatar, + string? pinHash, + bool replacePin, + DateTimeOffset now, + CancellationToken cancellationToken); + + Task> GetUsersAsync( + CancellationToken cancellationToken); + + Task UpdateUserAccessAsync( + Guid userId, + UserRole role, + bool isDisabled, + DateTimeOffset now, + CancellationToken cancellationToken); + + Task AddSessionAsync(UserSession session, CancellationToken cancellationToken); + + Task GetAuthenticatedSessionAsync( + Guid sessionId, + DateTimeOffset now, + CancellationToken cancellationToken); + + Task TryRotateSessionAsync( + Guid sessionId, + string expectedRefreshTokenHash, + string newRefreshTokenHash, + Guid activeProfileId, + DateTimeOffset? authenticatedAt, + DateTimeOffset now, + DateTimeOffset expiresAt, + CancellationToken cancellationToken); + + Task> GetSessionsAsync( + Guid? userId, + CancellationToken cancellationToken); + + Task RevokeSessionAsync( + Guid sessionId, + Guid? requiredUserId, + DateTimeOffset revokedAt, + CancellationToken cancellationToken); +} diff --git a/SecondDimensionWatcherReDive.Framework/DataRepository/IWebDavTokenRepository.cs b/SecondDimensionWatcherReDive.Framework/DataRepository/IWebDavTokenRepository.cs index 389f188..c303e20 100644 --- a/SecondDimensionWatcherReDive.Framework/DataRepository/IWebDavTokenRepository.cs +++ b/SecondDimensionWatcherReDive.Framework/DataRepository/IWebDavTokenRepository.cs @@ -10,5 +10,8 @@ public interface IWebDavTokenRepository Task AddAsync(WebDavToken token, CancellationToken cancellationToken); - Task RemoveByIdAsync(Guid id, CancellationToken cancellationToken); + Task RevokeByIdAsync( + Guid id, + DateTimeOffset revokedAt, + CancellationToken cancellationToken); } diff --git a/SecondDimensionWatcherReDive.Framework/DataRepository/Identity.cs b/SecondDimensionWatcherReDive.Framework/DataRepository/Identity.cs new file mode 100644 index 0000000..fb753b3 --- /dev/null +++ b/SecondDimensionWatcherReDive.Framework/DataRepository/Identity.cs @@ -0,0 +1,71 @@ +namespace SecondDimensionWatcherReDive.Framework.DataRepository; + +public enum UserRole +{ + Admin, + Member, + Viewer +} + +public enum UpdateUserAccessResult +{ + Updated, + NotFound, + LastAdministrator +} + +public sealed class IdentityConflictException(string message, Exception? innerException = null) + : Exception(message, innerException); + +public static class IdentityDefaults +{ + public static readonly Guid UserId = Guid.Parse("00000000-0000-0000-0000-000000000001"); + public static readonly Guid ProfileId = Guid.Empty; + public const string Username = "admin"; + public const string ProfileName = "Home"; +} + +public sealed record UserAccount( + Guid Id, + string Username, + string? PasswordHash, + UserRole Role, + bool IsDisabled, + DateTimeOffset CreatedAt, + DateTimeOffset UpdatedAt); + +public sealed record UserProfile( + Guid Id, + Guid UserId, + string Name, + string? Avatar, + string? PinHash, + bool IsDefault, + DateTimeOffset CreatedAt, + DateTimeOffset UpdatedAt); + +public sealed record UserSession( + Guid Id, + Guid UserId, + Guid ActiveProfileId, + string RefreshTokenHash, + string? DeviceName, + DateTimeOffset AuthenticatedAt, + DateTimeOffset CreatedAt, + DateTimeOffset LastSeenAt, + DateTimeOffset ExpiresAt, + DateTimeOffset? RevokedAt); + +public sealed record AuthenticatedSession( + UserAccount User, + UserProfile Profile, + UserSession Session); + +public sealed record UserAccountWithProfiles( + UserAccount User, + IReadOnlyList Profiles); + +public sealed record UserSessionSummary( + UserSession Session, + string Username, + string ProfileName); diff --git a/SecondDimensionWatcherReDive.Framework/DataRepository/WebDavToken.cs b/SecondDimensionWatcherReDive.Framework/DataRepository/WebDavToken.cs index beede59..9ae0a00 100644 --- a/SecondDimensionWatcherReDive.Framework/DataRepository/WebDavToken.cs +++ b/SecondDimensionWatcherReDive.Framework/DataRepository/WebDavToken.cs @@ -2,7 +2,12 @@ namespace SecondDimensionWatcherReDive.Framework.DataRepository; public sealed record WebDavToken( Guid Id, + Guid UserId, string Username, string TokenHash, string? Description, - DateTimeOffset CreatedAt); + DateTimeOffset CreatedAt, + string Scope, + string VirtualRoot, + DateTimeOffset? ExpiresAt, + DateTimeOffset? RevokedAt); diff --git a/SecondDimensionWatcherReDive.IntegrationTest/Auth/RoleAuthorizationTests.cs b/SecondDimensionWatcherReDive.IntegrationTest/Auth/RoleAuthorizationTests.cs new file mode 100644 index 0000000..83652d5 --- /dev/null +++ b/SecondDimensionWatcherReDive.IntegrationTest/Auth/RoleAuthorizationTests.cs @@ -0,0 +1,118 @@ +using System.Net; +using System.Net.Http.Json; +using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.IntegrationTest.TestData; + +namespace SecondDimensionWatcherReDive.IntegrationTest.Auth; + +[TestClass] +public sealed class RoleAuthorizationTests +{ + [TestMethod] + public async Task Viewer_CanReadFiles_ButCannotWriteContentPlaybackOrTasks() + { + using var factory = new WebDavWebApplicationFactory(role: UserRole.Viewer); + factory.ResetState(); + factory.Mappings.Add(WebDavMappingFixtures.NewMapping( + "/shows/episode.mkv", "/disk/episode.mkv")); + using var client = factory.CreateJwtClient(); + + using var read = await client.GetAsync("/api/vfs/stat?path=/shows/episode.mkv"); + using var addFeed = await client.PostAsJsonAsync("/api/feed", new + { + url = "https://example.test/feed.xml", + name = "test" + }); + using var playback = await client.PutAsJsonAsync("/api/playback/preferences", new + { + autoPlayNext = true + }); + using var task = await client.PostAsync("/api/tasks/SyncFeed/run", null); + + Assert.AreEqual(HttpStatusCode.OK, read.StatusCode); + Assert.AreEqual(HttpStatusCode.Forbidden, addFeed.StatusCode); + Assert.AreEqual(HttpStatusCode.Forbidden, playback.StatusCode); + Assert.AreEqual(HttpStatusCode.Forbidden, task.StatusCode); + } + + [TestMethod] + public async Task Member_CannotRunAdministratorTask() + { + using var factory = new WebDavWebApplicationFactory(role: UserRole.Member); + using var client = factory.CreateJwtClient(); + + using var response = await client.PostAsync("/api/tasks/SyncFeed/run", null); + + Assert.AreEqual(HttpStatusCode.Forbidden, response.StatusCode); + } + + [TestMethod] + public async Task Member_CannotReadOrWriteSettingsManageUsersOrDeleteDownloadedFiles() + { + using var factory = new WebDavWebApplicationFactory(role: UserRole.Member); + using var client = factory.CreateJwtClient(); + var animationId = Guid.NewGuid(); + + using var readSettings = await client.GetAsync("/api/settings"); + using var writeSettings = await client.PatchAsJsonAsync("/api/settings", new { }); + using var manageUsers = await client.GetAsync("/api/accounts/users"); + using var deleteFiles = await client.DeleteAsync( + $"/api/animationinfo/cancel/{animationId}?removeFile=true"); + + Assert.AreEqual(HttpStatusCode.Forbidden, readSettings.StatusCode); + Assert.AreEqual(HttpStatusCode.Forbidden, writeSettings.StatusCode); + Assert.AreEqual(HttpStatusCode.Forbidden, manageUsers.StatusCode); + Assert.AreEqual(HttpStatusCode.Forbidden, deleteFiles.StatusCode); + } + + [TestMethod] + public async Task Viewer_CannotStartOrCancelDownloadsOrWriteChat() + { + using var factory = new WebDavWebApplicationFactory(role: UserRole.Viewer); + using var client = factory.CreateJwtClient(); + var animationId = Guid.NewGuid(); + var conversationId = Guid.NewGuid(); + + using var start = await client.PostAsync( + $"/api/animationinfo/download/{animationId}", null); + using var cancel = await client.DeleteAsync( + $"/api/animationinfo/cancel/{animationId}"); + using var createChat = await client.PostAsJsonAsync( + "/api/chat/conversations", new { title = "blocked" }); + using var sendChat = await client.PostAsJsonAsync( + $"/api/chat/conversations/{conversationId}/messages", + new { content = "blocked" }); + + Assert.AreEqual(HttpStatusCode.Forbidden, start.StatusCode); + Assert.AreEqual(HttpStatusCode.Forbidden, cancel.StatusCode); + Assert.AreEqual(HttpStatusCode.Forbidden, createChat.StatusCode); + Assert.AreEqual(HttpStatusCode.Forbidden, sendChat.StatusCode); + } + + [TestMethod] + public async Task MissingSessionToken_IsUnauthorized() + { + using var factory = new WebDavWebApplicationFactory(); + using var client = factory.CreateUnauthenticatedClient(); + + using var response = await client.GetAsync("/api/feed"); + + Assert.AreEqual(HttpStatusCode.Unauthorized, response.StatusCode); + } + + [TestMethod] + public async Task RevokedLoginSession_InvalidatesExistingAccessTokenImmediately() + { + using var factory = new WebDavWebApplicationFactory(); + factory.Mappings.Add(WebDavMappingFixtures.NewMapping( + "/shows/episode.mkv", "/disk/episode.mkv")); + using var client = factory.CreateJwtClient(); + using var before = await client.GetAsync("/api/vfs/stat?path=/shows/episode.mkv"); + Assert.AreEqual(HttpStatusCode.OK, before.StatusCode); + + factory.RevokeLoginSession(); + using var after = await client.GetAsync("/api/vfs/stat?path=/shows/episode.mkv"); + + Assert.AreEqual(HttpStatusCode.Unauthorized, after.StatusCode); + } +} diff --git a/SecondDimensionWatcherReDive.IntegrationTest/Helpers/FakeWebDavTokenRepository.cs b/SecondDimensionWatcherReDive.IntegrationTest/Helpers/FakeWebDavTokenRepository.cs index 3d9480a..409a46f 100644 --- a/SecondDimensionWatcherReDive.IntegrationTest/Helpers/FakeWebDavTokenRepository.cs +++ b/SecondDimensionWatcherReDive.IntegrationTest/Helpers/FakeWebDavTokenRepository.cs @@ -4,11 +4,34 @@ namespace SecondDimensionWatcherReDive.IntegrationTest.Helpers; internal sealed class FakeWebDavTokenRepository : IWebDavTokenRepository { - private readonly WebDavToken _seeded; + private WebDavToken _seeded; - public FakeWebDavTokenRepository(string username, string tokenHash) + public Guid TokenId => _seeded.Id; + + public void Expire() => _seeded = _seeded with + { + ExpiresAt = DateTimeOffset.UtcNow.AddMinutes(-1) + }; + + public void SetScope(string scope) => _seeded = _seeded with { Scope = scope }; + + public FakeWebDavTokenRepository( + Guid userId, + string username, + string tokenHash, + string virtualRoot = "/") { - _seeded = new WebDavToken(Guid.NewGuid(), username, tokenHash, "integration-test", DateTimeOffset.UtcNow); + _seeded = new WebDavToken( + Guid.NewGuid(), + userId, + username, + tokenHash, + "integration-test", + DateTimeOffset.UtcNow, + "read", + virtualRoot, + DateTimeOffset.UtcNow.AddDays(1), + null); } public Task> GetAllOrderedAsync(CancellationToken cancellationToken) @@ -23,6 +46,13 @@ public Task ExistsByUsernameAsync(string username, CancellationToken cance public Task AddAsync(WebDavToken token, CancellationToken cancellationToken) => throw new NotSupportedException(); - public Task RemoveByIdAsync(Guid id, CancellationToken cancellationToken) - => throw new NotSupportedException(); + public Task RevokeByIdAsync( + Guid id, + DateTimeOffset revokedAt, + CancellationToken cancellationToken) + { + if (_seeded.Id != id) return Task.FromResult(false); + _seeded = _seeded with { RevokedAt = revokedAt }; + return Task.FromResult(true); + } } diff --git a/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/HouseholdMigrationPostgreSqlTests.cs b/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/HouseholdMigrationPostgreSqlTests.cs new file mode 100644 index 0000000..8d1975f --- /dev/null +++ b/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/HouseholdMigrationPostgreSqlTests.cs @@ -0,0 +1,188 @@ +using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Repositories; +using Testcontainers.PostgreSql; + +namespace SecondDimensionWatcherReDive.IntegrationTest.PostgreSql; + +[TestClass] +public sealed class HouseholdMigrationPostgreSqlTests +{ + private static readonly PostgreSqlContainer Database = new PostgreSqlBuilder("postgres:17-alpine") + .WithDatabase("sdw_identity_tests") + .WithUsername("postgres") + .WithPassword("postgres") + .Build(); + + private static HouseholdMigrationPostgreSqlTestFixture Fixture = null!; + + [ClassInitialize] + public static async Task InitializeAsync(TestContext _) + { + await Database.StartAsync(); + Fixture = new HouseholdMigrationPostgreSqlTestFixture(Database.GetConnectionString()); + } + + [ClassCleanup] + public static async Task CleanupAsync() => await Database.DisposeAsync(); + + [TestInitialize] + public async Task ResetAsync() => await Fixture.RecreateAsync(CancellationToken.None); + + [TestMethod] + public async Task LegacyHistory_IsAssignedToDefaultProfile_BeforeForeignKeysAreAdded() + { + await Fixture.SeedLegacyAndUpgradeAsync(CancellationToken.None); + + var snapshot = await Fixture.InspectAsync(CancellationToken.None); + Assert.AreEqual(1, snapshot.UserCount); + Assert.AreEqual(IdentityDefaults.UserId, snapshot.UserId); + Assert.AreEqual("admin", snapshot.Username); + Assert.AreEqual(UserRole.Admin, snapshot.Role); + Assert.AreEqual(IdentityDefaults.ProfileId, snapshot.ProfileId); + Assert.AreEqual("Home", snapshot.ProfileName); + Assert.AreEqual(IdentityDefaults.ProfileId, snapshot.ProgressProfileId); + Assert.AreEqual(123d, snapshot.PositionSeconds); + Assert.AreEqual(IdentityDefaults.ProfileId, snapshot.PreferenceProfileId); + Assert.AreEqual("zh-Hans", snapshot.SubtitleLanguage); + Assert.AreEqual(IdentityDefaults.ProfileId, snapshot.ConversationProfileId); + Assert.AreEqual("legacy chat", snapshot.ConversationTitle); + Assert.AreEqual(IdentityDefaults.UserId, snapshot.DeviceUserId); + Assert.AreEqual("read", snapshot.DeviceScope); + Assert.AreEqual("/", snapshot.DeviceRoot); + Assert.IsNull(snapshot.DeviceExpiresAt); + Assert.IsNull(snapshot.DeviceRevokedAt); + + var down = await Fixture.MigrateDownAsync(CancellationToken.None); + Assert.AreEqual(1, down.PlaybackCount); + Assert.AreEqual(1, down.PreferenceCount); + Assert.AreEqual(1, down.ConversationCount); + Assert.AreEqual(1, down.DeviceTokenCount); + Assert.IsFalse(down.UsersTableExists); + Assert.IsFalse(down.ProfilesTableExists); + + // Re-upgrade proves Down left the legacy rows in a valid, recoverable state. + await Fixture.UpgradeAsync(CancellationToken.None); + var reupgraded = await Fixture.InspectAsync(CancellationToken.None); + Assert.AreEqual(123d, reupgraded.PositionSeconds); + } + + [TestMethod] + public async Task CleanMigration_LeavesRegistrationOpen_AndCanDowngrade() + { + await Fixture.MigrateCleanDatabaseAsync(CancellationToken.None); + Assert.AreEqual(0, await Fixture.GetUserCountAsync(CancellationToken.None)); + + var registration = await Fixture.RegisterAndCreateSessionAsync(CancellationToken.None); + Assert.AreEqual(IdentityDefaults.ProfileId, registration.PersistedProfileId); + Assert.AreEqual(IdentityDefaults.ProfileId, registration.IssuedProfileId); + Assert.IsTrue(registration.SessionIsActive); + Assert.AreEqual(2, registration.UserCount); + + var registeredDown = await Fixture.AttemptUnsafeDowngradeAsync( + CancellationToken.None); + Assert.IsTrue(registeredDown.Rejected); + Assert.IsTrue(registeredDown.CurrentMigrationStillApplied); + Assert.AreEqual(2, registeredDown.UserCount); + + await Fixture.RecreateAsync(CancellationToken.None); + await Fixture.MigrateCleanDatabaseAsync(CancellationToken.None); + var down = await Fixture.MigrateDownAsync(CancellationToken.None); + Assert.AreEqual(0, down.PlaybackCount); + Assert.AreEqual(0, down.PreferenceCount); + Assert.AreEqual(0, down.ConversationCount); + Assert.AreEqual(0, down.DeviceTokenCount); + Assert.IsFalse(down.UsersTableExists); + Assert.IsFalse(down.ProfilesTableExists); + } + + [TestMethod] + public async Task ConcurrentAdminDemotions_CannotRemoveLastEnabledAdministrator() + { + await Fixture.MigrateCleanDatabaseAsync(CancellationToken.None); + + var result = await Fixture.DemoteTwoAdminsConcurrentlyAsync(CancellationToken.None); + + Assert.AreEqual(1, result.Results.Count(item => item == UpdateUserAccessResult.Updated)); + Assert.AreEqual(1, result.Results.Count(item => item == UpdateUserAccessResult.LastAdministrator)); + Assert.AreEqual(1, result.EnabledAdministratorCount); + } + + [TestMethod] + public async Task Profiles_HaveIndependentPlaybackPreferencesAndConversations() + { + await Fixture.MigrateCleanDatabaseAsync(CancellationToken.None); + + var result = await Fixture.ExerciseProfileIsolationAsync(CancellationToken.None); + + Assert.AreEqual(10d, result.FirstPosition); + Assert.AreEqual(70d, result.SecondPosition); + Assert.AreEqual("zh-Hans", result.FirstSubtitleLanguage); + Assert.AreEqual("en", result.SecondSubtitleLanguage); + Assert.AreEqual(1, result.FirstConversationCount); + Assert.AreEqual(1, result.SecondConversationCount); + Assert.IsTrue(result.CrossProfileConversationHidden); + Assert.AreEqual(10d, result.FirstContinuePosition); + Assert.AreEqual(70d, result.SecondContinuePosition); + } + + [TestMethod] + public async Task ProfileSwitchLogoutRevokeAndRefreshRotation_InvalidateOldCredentials() + { + await Fixture.MigrateCleanDatabaseAsync(CancellationToken.None); + + var result = await Fixture.ExerciseSessionLifecycleAsync(CancellationToken.None); + + Assert.IsTrue(result.WrongPinRejected); + Assert.IsTrue(result.CorrectPinRotated); + Assert.IsTrue(result.OldAccessRejected); + Assert.IsTrue(result.NewProfileClaimIsActive); + Assert.IsTrue(result.OldRefreshReplayRejected); + Assert.IsTrue(result.LogoutRejectedAccess); + Assert.IsTrue(result.LogoutRejectedRefresh); + Assert.IsTrue(result.AdministratorRevokeRejectedAccess); + Assert.IsTrue(result.AdministratorRevokeRejectedRefresh); + Assert.AreEqual(1, result.ConcurrentRefreshSuccessCount); + } + + [TestMethod] + public async Task Downgrade_WithMultipleProfilesAndHistory_IsRejectedWithoutMutation() + { + await Fixture.MigrateCleanDatabaseAsync(CancellationToken.None); + await Fixture.ExerciseProfileIsolationAsync(CancellationToken.None); + + var result = await Fixture.AttemptUnsafeDowngradeAsync(CancellationToken.None); + + Assert.IsTrue(result.Rejected); + Assert.IsTrue(result.CurrentMigrationStillApplied); + Assert.AreEqual(1, result.UserCount); + Assert.AreEqual(2, result.ProfileCount); + Assert.AreEqual(2, result.ProgressCount); + Assert.AreEqual(2, result.PreferenceCount); + } + + [TestMethod] + public async Task Downgrade_WithScopedExpiringDeviceToken_IsRejectedWithoutWideningIt() + { + await Fixture.MigrateCleanDatabaseAsync(CancellationToken.None); + await Fixture.SeedUnsafeScopedDeviceTokenAsync(CancellationToken.None); + + var result = await Fixture.AttemptUnsafeDowngradeAsync(CancellationToken.None); + + Assert.IsTrue(result.Rejected); + Assert.IsTrue(result.CurrentMigrationStillApplied); + Assert.AreEqual(1, result.DeviceTokenCount); + } + + [TestMethod] + public async Task ConcurrentFirstRegistration_ReturnsConflictInsteadOfServerError() + { + await Fixture.MigrateCleanDatabaseAsync(CancellationToken.None); + + var result = await Fixture.RegisterConcurrentlyAsync(CancellationToken.None); + + Assert.AreEqual(1, result.SuccessCount); + Assert.AreEqual(1, result.ConflictCount); + Assert.AreEqual(1, result.UserCount); + Assert.AreEqual(1, result.SessionCount); + } +} diff --git a/SecondDimensionWatcherReDive.IntegrationTest/Repositories/HouseholdMigrationPostgreSqlTestFixture.cs b/SecondDimensionWatcherReDive.IntegrationTest/Repositories/HouseholdMigrationPostgreSqlTestFixture.cs new file mode 100644 index 0000000..66948d2 --- /dev/null +++ b/SecondDimensionWatcherReDive.IntegrationTest/Repositories/HouseholdMigrationPostgreSqlTestFixture.cs @@ -0,0 +1,786 @@ +using System.IdentityModel.Tokens.Jwt; +using System.Security.Claims; +using System.Text; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.IdentityModel.Tokens; +using Moq; +using Npgsql; +using SecondDimensionWatcherReDive.Auth; +using SecondDimensionWatcherReDive.Controllers; +using SecondDimensionWatcherReDive.Framework.Authorization; +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Repositories; + +internal sealed record HouseholdMigrationSnapshot( + int UserCount, + Guid? UserId, + string? Username, + UserRole? Role, + Guid? ProfileId, + string? ProfileName, + Guid? ProgressProfileId, + double? PositionSeconds, + Guid? PreferenceProfileId, + string? SubtitleLanguage, + Guid? ConversationProfileId, + string? ConversationTitle, + Guid? DeviceUserId, + string? DeviceScope, + string? DeviceRoot, + DateTimeOffset? DeviceExpiresAt, + DateTimeOffset? DeviceRevokedAt); + +internal sealed record HouseholdMigrationDownSnapshot( + int PlaybackCount, + int PreferenceCount, + int ConversationCount, + int DeviceTokenCount, + bool UsersTableExists, + bool ProfilesTableExists); + +internal sealed record CleanRegistrationSnapshot( + Guid PersistedProfileId, + Guid IssuedProfileId, + bool SessionIsActive, + int UserCount); + +internal sealed record ConcurrentAdminUpdateSnapshot( + IReadOnlyList Results, + int EnabledAdministratorCount); + +internal sealed record ProfileIsolationSnapshot( + double? FirstPosition, + double? SecondPosition, + string? FirstSubtitleLanguage, + string? SecondSubtitleLanguage, + int FirstConversationCount, + int SecondConversationCount, + bool CrossProfileConversationHidden, + double? FirstContinuePosition, + double? SecondContinuePosition); + +internal sealed record SessionLifecycleSnapshot( + bool WrongPinRejected, + bool CorrectPinRotated, + bool OldAccessRejected, + bool NewProfileClaimIsActive, + bool OldRefreshReplayRejected, + bool LogoutRejectedAccess, + bool LogoutRejectedRefresh, + bool AdministratorRevokeRejectedAccess, + bool AdministratorRevokeRejectedRefresh, + int ConcurrentRefreshSuccessCount); + +internal sealed record ConcurrentRegistrationSnapshot( + int SuccessCount, + int ConflictCount, + int UserCount, + int SessionCount); + +internal sealed record DowngradeSafetySnapshot( + bool Rejected, + bool CurrentMigrationStillApplied, + int UserCount, + int ProfileCount, + int ProgressCount, + int PreferenceCount, + int DeviceTokenCount); + +/// +/// PostgreSQL-only migration fixture. It lives in the integration-test repository boundary so EF entities and +/// ApplicationContext never escape the permitted data-access boundary. +/// +internal sealed class HouseholdMigrationPostgreSqlTestFixture(string connectionString) +{ + internal const string PreviousMigration = "20260828164158_AddApplicationSettings"; + + private readonly DbContextOptions _contextOptions = + new DbContextOptionsBuilder() + .UseNpgsql(connectionString, options => options.EnableRetryOnFailure()) + .Options; + + public async Task RecreateAsync(CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + await context.Database.EnsureDeletedAsync(cancellationToken); + } + + public async Task SeedLegacyAndUpgradeAsync(CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + await context.Database.MigrateAsync(PreviousMigration, cancellationToken); + + var now = DateTimeOffset.UtcNow; + var animationInfoId = Guid.Parse("41000000-0000-0000-0000-000000000001"); + context.AnimationInfo.Add(new Models.AnimationInfo + { + Id = animationInfoId, + Title = "legacy episode", + Description = string.Empty, + PublishTime = now, + DownloadUrl = string.Empty, + DownloadType = string.Empty, + CachedDownloadData = [], + AdditionalDownloadInfo = string.Empty, + IsDownloadFinished = true, + FileStore = "local", + StorePath = "/legacy" + }); + await context.SaveChangesAsync(cancellationToken); + + var progressId = Guid.Parse("42000000-0000-0000-0000-000000000001"); + var conversationId = Guid.Parse("43000000-0000-0000-0000-000000000001"); + var tokenId = Guid.Parse("44000000-0000-0000-0000-000000000001"); + await context.Database.ExecuteSqlInterpolatedAsync( + $""" + INSERT INTO "PlaybackProgresses" + ("Id", "UserId", "AnimationInfoId", "VirtualPath", "PositionSeconds", + "DurationSeconds", "IsWatched", "UpdatedAt", "WatchedAt") + VALUES + ({progressId}, {Guid.Empty}, {animationInfoId}, {'/' + "legacy/episode.mkv"}, + {123d}, {1500d}, {false}, {now}, {null}); + """, + cancellationToken); + await context.Database.ExecuteSqlInterpolatedAsync( + $""" + INSERT INTO "PlaybackPreferences" + ("UserId", "SubtitleLanguage", "SubtitleTrackLabel", "AudioLanguage", + "AudioTrackLabel", "AutoPlayNext", "UpdatedAt") + VALUES ({Guid.Empty}, {"zh-Hans"}, {null}, {"ja"}, {null}, {true}, {now}); + """, + cancellationToken); + await context.Database.ExecuteSqlInterpolatedAsync( + $""" + INSERT INTO "ChatConversations" ("Id", "Title", "CreatedAt", "UpdatedAt") + VALUES ({conversationId}, {"legacy chat"}, {now}, {now}); + """, + cancellationToken); + await context.Database.ExecuteSqlInterpolatedAsync( + $""" + INSERT INTO "WebDavTokens" + ("Id", "Username", "TokenHash", "Description", "CreatedAt") + VALUES ({tokenId}, {"legacy-device"}, {"legacy-hash"}, {"old client"}, {now}); + """, + cancellationToken); + + await context.Database.MigrateAsync(cancellationToken); + } + + public async Task InspectAsync( + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + var user = await context.Users.AsNoTracking().SingleOrDefaultAsync(cancellationToken); + var profile = await context.Profiles.AsNoTracking().SingleOrDefaultAsync(cancellationToken); + var progress = await context.PlaybackProgresses.AsNoTracking() + .SingleOrDefaultAsync(cancellationToken); + var preference = await context.PlaybackPreferences.AsNoTracking() + .SingleOrDefaultAsync(cancellationToken); + var conversation = await context.ChatConversations.AsNoTracking() + .SingleOrDefaultAsync(cancellationToken); + var token = await context.WebDavTokens.AsNoTracking() + .SingleOrDefaultAsync(cancellationToken); + return new HouseholdMigrationSnapshot( + await context.Users.CountAsync(cancellationToken), + user?.Id, + user?.Username, + user?.Role, + profile?.Id, + profile?.Name, + progress?.UserId, + progress?.PositionSeconds, + preference?.UserId, + preference?.SubtitleLanguage, + conversation?.ProfileId, + conversation?.Title, + token?.UserId, + token?.Scope, + token?.VirtualRoot, + token?.ExpiresAt, + token?.RevokedAt); + } + + public async Task MigrateCleanDatabaseAsync(CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + await context.Database.MigrateAsync(cancellationToken); + } + + public Task UpgradeAsync(CancellationToken cancellationToken) => + MigrateCleanDatabaseAsync(cancellationToken); + + public async Task GetUserCountAsync(CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + return await context.Users.CountAsync(cancellationToken); + } + + public async Task RegisterAndCreateSessionAsync( + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + var repository = new IdentityRepository(context); + var now = DateTimeOffset.UtcNow; + var user = new UserAccount( + IdentityDefaults.UserId, + IdentityDefaults.Username, + BCrypt.Net.BCrypt.HashPassword("integration-password"), + UserRole.Admin, + false, + now, + now); + var profile = new UserProfile( + IdentityDefaults.ProfileId, + user.Id, + IdentityDefaults.ProfileName, + null, + null, + true, + now, + now); + await repository.CreateUserWithProfileAsync(user, profile, cancellationToken); + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["JwtSecret"] = "postgres-integration-secret-long-enough-123456" + }) + .Build(); + var issuer = new SessionTokenIssuer(configuration, repository); + var issued = await issuer.CreateSessionAsync( + user, profile, "integration", cancellationToken); + var persistedProfileId = await context.Profiles + .Select(candidate => candidate.Id) + .SingleAsync(cancellationToken); + var active = await repository.GetAuthenticatedSessionAsync( + issued.SessionId, DateTimeOffset.UtcNow, cancellationToken); + var secondUser = new UserAccount( + Guid.NewGuid(), + "family-member", + BCrypt.Net.BCrypt.HashPassword("member-password"), + UserRole.Member, + false, + now, + now); + var secondProfile = new UserProfile( + Guid.NewGuid(), + secondUser.Id, + "Member Home", + null, + null, + true, + now, + now); + await repository.CreateUserWithProfileAsync( + secondUser, secondProfile, cancellationToken); + return new CleanRegistrationSnapshot( + persistedProfileId, + issued.ProfileId, + active is not null, + await context.Users.CountAsync(cancellationToken)); + } + + public async Task DemoteTwoAdminsConcurrentlyAsync( + CancellationToken cancellationToken) + { + var firstUserId = Guid.Parse("51000000-0000-0000-0000-000000000001"); + var secondUserId = Guid.Parse("51000000-0000-0000-0000-000000000002"); + await using (var seedContext = new Models.ApplicationContext(_contextOptions)) + { + var now = DateTimeOffset.UtcNow; + seedContext.Users.AddRange( + UserEntity(firstUserId, "first-admin", now), + UserEntity(secondUserId, "second-admin", now)); + seedContext.Profiles.AddRange( + ProfileEntity(Guid.Parse("52000000-0000-0000-0000-000000000001"), firstUserId, now), + ProfileEntity(Guid.Parse("52000000-0000-0000-0000-000000000002"), secondUserId, now)); + await seedContext.SaveChangesAsync(cancellationToken); + } + + async Task DemoteAsync(Guid id) + { + await using var updateContext = new Models.ApplicationContext(_contextOptions); + var repository = new IdentityRepository(updateContext); + return await repository.UpdateUserAccessAsync( + id, + UserRole.Member, + false, + DateTimeOffset.UtcNow, + cancellationToken); + } + + var results = await Task.WhenAll( + DemoteAsync(firstUserId), + DemoteAsync(secondUserId)); + await using var inspectContext = new Models.ApplicationContext(_contextOptions); + var enabledAdmins = await inspectContext.Users.CountAsync( + user => user.Role == UserRole.Admin && !user.IsDisabled, + cancellationToken); + return new ConcurrentAdminUpdateSnapshot(results, enabledAdmins); + } + + public async Task ExerciseSessionLifecycleAsync( + CancellationToken cancellationToken) + { + var configuration = CreateJwtConfiguration(); + var validationParameters = CreateTokenValidationParameters(configuration); + var now = DateTimeOffset.UtcNow; + var user = new UserAccount( + Guid.Parse("61000000-0000-0000-0000-000000000001"), + "session-user", + BCrypt.Net.BCrypt.HashPassword("session-password"), + UserRole.Admin, + false, + now, + now); + var firstProfile = new UserProfile( + Guid.Parse("62000000-0000-0000-0000-000000000001"), + user.Id, + "First", + null, + null, + true, + now, + now); + var secondProfile = new UserProfile( + Guid.Parse("62000000-0000-0000-0000-000000000002"), + user.Id, + "Second", + null, + BCrypt.Net.BCrypt.HashPassword("2468"), + false, + now, + now); + + await using var context = new Models.ApplicationContext(_contextOptions); + var repository = new IdentityRepository(context); + await repository.CreateUserWithProfileAsync(user, firstProfile, cancellationToken); + await repository.AddProfileAsync(secondProfile, cancellationToken); + var issuer = new SessionTokenIssuer(configuration, repository); + var initial = await issuer.CreateSessionAsync( + user, firstProfile, "profile-switch-test", cancellationToken); + var oldPrincipal = ValidateToken(initial.AccessToken, validationParameters); + var authorization = new Mock(); + var accounts = CreateAccountsController( + repository, issuer, authorization.Object, oldPrincipal); + + var wrongPin = await accounts.SwitchProfile( + new Controllers.External.SwitchProfileRequest( + secondProfile.Id, "0000", initial.RefreshToken), + cancellationToken); + var afterWrongPin = await repository.GetAuthenticatedSessionAsync( + initial.SessionId, DateTimeOffset.UtcNow, cancellationToken); + var wrongPinRejected = wrongPin is UnauthorizedResult + && afterWrongPin?.Profile.Id == firstProfile.Id; + + var correctPin = await accounts.SwitchProfile( + new Controllers.External.SwitchProfileRequest( + secondProfile.Id, "2468", initial.RefreshToken), + cancellationToken); + var rotated = (correctPin as OkObjectResult)?.Value + as Controllers.External.LoginResult; + if (rotated?.Token is null || rotated.RefreshToken is null) + throw new InvalidOperationException("Profile switch did not issue tokens."); + var newPrincipal = ValidateToken(rotated.Token, validationParameters); + var correctPinRotated = rotated.ProfileId == secondProfile.Id + && rotated.RefreshToken != initial.RefreshToken; + var oldAccessRejected = !await IsPrincipalCurrentAsync( + oldPrincipal, repository, cancellationToken); + var newProfileClaimIsActive = newPrincipal.TryGetProfileId(out var newProfileId) + && newProfileId == secondProfile.Id + && await IsPrincipalCurrentAsync( + newPrincipal, repository, cancellationToken); + + var auth = CreateAuthController( + configuration, validationParameters, repository, issuer); + var oldReplay = await auth.Refresh( + new Controllers.External.AuthRequest( + initial.AccessToken, initial.RefreshToken), + cancellationToken); + var oldRefreshReplayRejected = IsUnauthorized(oldReplay); + + auth.ControllerContext = new ControllerContext + { + HttpContext = new DefaultHttpContext { User = newPrincipal } + }; + await auth.Logout(cancellationToken); + var logoutRejectedAccess = !await IsPrincipalCurrentAsync( + newPrincipal, repository, cancellationToken); + var logoutRefresh = await auth.Refresh( + new Controllers.External.AuthRequest( + rotated.Token, rotated.RefreshToken), + cancellationToken); + var logoutRejectedRefresh = IsUnauthorized(logoutRefresh); + + var adminRevoked = await issuer.CreateSessionAsync( + user, secondProfile, "administrator-revoke-test", cancellationToken); + var adminRevokedPrincipal = ValidateToken( + adminRevoked.AccessToken, validationParameters); + var adminAccounts = CreateAccountsController( + repository, issuer, authorization.Object, newPrincipal); + await adminAccounts.RevokeAnySession( + adminRevoked.SessionId, cancellationToken); + var administratorRevokeRejectedAccess = !await IsPrincipalCurrentAsync( + adminRevokedPrincipal, repository, cancellationToken); + var administratorRevokeRefresh = await auth.Refresh( + new Controllers.External.AuthRequest( + adminRevoked.AccessToken, adminRevoked.RefreshToken), + cancellationToken); + var administratorRevokeRejectedRefresh = IsUnauthorized( + administratorRevokeRefresh); + + var concurrent = await issuer.CreateSessionAsync( + user, secondProfile, "concurrent-refresh-test", cancellationToken); + async Task RefreshConcurrentlyAsync() + { + await using var refreshContext = new Models.ApplicationContext(_contextOptions); + var refreshRepository = new IdentityRepository(refreshContext); + var refreshIssuer = new SessionTokenIssuer(configuration, refreshRepository); + var refreshController = CreateAuthController( + configuration, + validationParameters, + refreshRepository, + refreshIssuer); + var result = await refreshController.Refresh( + new Controllers.External.AuthRequest( + concurrent.AccessToken, concurrent.RefreshToken), + cancellationToken); + return result is OkObjectResult; + } + + var concurrentResults = await Task.WhenAll( + RefreshConcurrentlyAsync(), + RefreshConcurrentlyAsync()); + return new SessionLifecycleSnapshot( + wrongPinRejected, + correctPinRotated, + oldAccessRejected, + newProfileClaimIsActive, + oldRefreshReplayRejected, + logoutRejectedAccess, + logoutRejectedRefresh, + administratorRevokeRejectedAccess, + administratorRevokeRejectedRefresh, + concurrentResults.Count(result => result)); + } + + public async Task RegisterConcurrentlyAsync( + CancellationToken cancellationToken) + { + var configuration = CreateJwtConfiguration(); + var validationParameters = CreateTokenValidationParameters(configuration); + async Task RegisterAsync() + { + await using var registerContext = new Models.ApplicationContext(_contextOptions); + var repository = new IdentityRepository(registerContext); + var issuer = new SessionTokenIssuer(configuration, repository); + var controller = CreateAuthController( + configuration, validationParameters, repository, issuer); + return await controller.Register( + new Controllers.External.LoginData( + "concurrent-password", + IdentityDefaults.Username, + "concurrent-registration", + IdentityDefaults.ProfileName), + cancellationToken); + } + + var results = await Task.WhenAll(RegisterAsync(), RegisterAsync()); + await using var inspect = new Models.ApplicationContext(_contextOptions); + return new ConcurrentRegistrationSnapshot( + results.Count(result => result is OkObjectResult), + results.Count(result => result is ConflictResult), + await inspect.Users.CountAsync(cancellationToken), + await inspect.LoginSessions.CountAsync(cancellationToken)); + } + + public async Task SeedUnsafeScopedDeviceTokenAsync( + CancellationToken cancellationToken) + { + var now = DateTimeOffset.UtcNow; + await using var context = new Models.ApplicationContext(_contextOptions); + context.Users.Add(UserEntity( + IdentityDefaults.UserId, IdentityDefaults.Username, now)); + context.Users.Local.Single().PasswordHash = null; + context.Profiles.Add(ProfileEntity( + IdentityDefaults.ProfileId, IdentityDefaults.UserId, now)); + context.WebDavTokens.Add(new Models.WebDavToken + { + Id = Guid.NewGuid(), + UserId = IdentityDefaults.UserId, + Username = "scoped-device", + TokenHash = "hash", + CreatedAt = now, + Scope = "read", + VirtualRoot = "/Anime", + ExpiresAt = now.AddDays(30) + }); + await context.SaveChangesAsync(cancellationToken); + } + + public async Task AttemptUnsafeDowngradeAsync( + CancellationToken cancellationToken) + { + var rejected = false; + try + { + await using var downContext = new Models.ApplicationContext(_contextOptions); + await downContext.Database.MigrateAsync( + PreviousMigration, cancellationToken); + } + catch (Exception exception) when (IsSafetyRejection(exception)) + { + rejected = true; + } + + await using var inspect = new Models.ApplicationContext(_contextOptions); + var applied = await inspect.Database.GetAppliedMigrationsAsync(cancellationToken); + return new DowngradeSafetySnapshot( + rejected, + applied.Contains("20260829155550_AddHouseholdIdentityAndAccessScopes"), + await inspect.Users.CountAsync(cancellationToken), + await inspect.Profiles.CountAsync(cancellationToken), + await inspect.PlaybackProgresses.CountAsync(cancellationToken), + await inspect.PlaybackPreferences.CountAsync(cancellationToken), + await inspect.WebDavTokens.CountAsync(cancellationToken)); + } + + public async Task ExerciseProfileIsolationAsync( + CancellationToken cancellationToken) + { + var userId = Guid.Parse("71000000-0000-0000-0000-000000000001"); + var firstProfileId = Guid.Parse("72000000-0000-0000-0000-000000000001"); + var secondProfileId = Guid.Parse("72000000-0000-0000-0000-000000000002"); + var animationInfoId = Guid.Parse("73000000-0000-0000-0000-000000000001"); + const string VirtualPath = "/unknown/episode.mkv"; + var now = DateTimeOffset.UtcNow; + await using var context = new Models.ApplicationContext(_contextOptions); + context.Users.Add(new Models.UserAccount + { + Id = userId, + Username = "family", + PasswordHash = "hash", + Role = UserRole.Member, + CreatedAt = now, + UpdatedAt = now + }); + context.Profiles.AddRange( + ProfileEntity(firstProfileId, userId, now, "First"), + ProfileEntity(secondProfileId, userId, now, "Second")); + context.AnimationInfo.Add(new Models.AnimationInfo + { + Id = animationInfoId, + Title = "profile isolation", + Description = string.Empty, + PublishTime = now, + DownloadUrl = string.Empty, + DownloadType = string.Empty, + CachedDownloadData = [], + AdditionalDownloadInfo = string.Empty, + IsDownloadFinished = true, + FileStore = "local", + StorePath = "/profile-isolation" + }); + context.FileMappings.Add(new Models.FileMapping + { + Id = Guid.NewGuid(), + AnimationInfoId = animationInfoId, + VirtualPath = VirtualPath, + PhysicalPath = "/disk/episode.mkv", + FileStore = "local" + }); + await context.SaveChangesAsync(cancellationToken); + + var playback = new PlaybackRepository(context, _contextOptions); + await playback.UpsertProgressAsync( + firstProfileId, animationInfoId, VirtualPath, + 10, 100, false, now, cancellationToken); + await playback.UpsertProgressAsync( + secondProfileId, animationInfoId, VirtualPath, + 70, 100, false, now.AddSeconds(1), cancellationToken); + await playback.UpsertPreferencesAsync(new PlaybackPreferences( + firstProfileId, "zh-Hans", null, "ja", null, true, now), cancellationToken); + await playback.UpsertPreferencesAsync(new PlaybackPreferences( + secondProfileId, "en", null, "en", null, false, now), cancellationToken); + + var chat = new ChatRepository(context); + var firstConversation = await chat.CreateConversationAsync( + firstProfileId, "first", cancellationToken); + await chat.CreateConversationAsync(secondProfileId, "second", cancellationToken); + var firstProgress = await playback.FindProgressAsync( + firstProfileId, animationInfoId, VirtualPath, cancellationToken); + var secondProgress = await playback.FindProgressAsync( + secondProfileId, animationInfoId, VirtualPath, cancellationToken); + var firstPreferences = await playback.GetPreferencesAsync( + firstProfileId, cancellationToken); + var secondPreferences = await playback.GetPreferencesAsync( + secondProfileId, cancellationToken); + var firstConversations = await chat.GetConversationsAsync( + firstProfileId, cancellationToken); + var secondConversations = await chat.GetConversationsAsync( + secondProfileId, cancellationToken); + var crossProfile = await chat.GetConversationWithMessagesAsync( + firstConversation.Id, secondProfileId, cancellationToken); + var firstContinue = await playback.GetContinueWatchingAsync( + firstProfileId, 10, cancellationToken); + var secondContinue = await playback.GetContinueWatchingAsync( + secondProfileId, 10, cancellationToken); + return new ProfileIsolationSnapshot( + firstProgress?.PositionSeconds, + secondProgress?.PositionSeconds, + firstPreferences.SubtitleLanguage, + secondPreferences.SubtitleLanguage, + firstConversations.Count, + secondConversations.Count, + crossProfile is null, + firstContinue.SingleOrDefault()?.Progress.PositionSeconds, + secondContinue.SingleOrDefault()?.Progress.PositionSeconds); + } + + private static Models.UserAccount UserEntity( + Guid id, + string username, + DateTimeOffset now) => new() + { + Id = id, + Username = username, + PasswordHash = "hash", + Role = UserRole.Admin, + IsDisabled = false, + CreatedAt = now, + UpdatedAt = now + }; + + private static IConfiguration CreateJwtConfiguration() => + new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["JwtSecret"] = "postgres-integration-secret-long-enough-123456" + }) + .Build(); + + private static TokenValidationParameters CreateTokenValidationParameters( + IConfiguration configuration) => new() + { + ValidateIssuerSigningKey = true, + IssuerSigningKey = new SymmetricSecurityKey( + Encoding.ASCII.GetBytes(configuration["JwtSecret"]!)), + ValidateIssuer = false, + ValidateAudience = false, + ValidateLifetime = true, + RequireExpirationTime = true + }; + + private static ClaimsPrincipal ValidateToken( + string token, + TokenValidationParameters validationParameters) => + new JwtSecurityTokenHandler().ValidateToken( + token, validationParameters, out _); + + private static AccountsController CreateAccountsController( + IIdentityRepository repository, + SessionTokenIssuer issuer, + IAuthorizationService authorizationService, + ClaimsPrincipal principal) => new(repository, issuer, authorizationService) + { + ControllerContext = new ControllerContext + { + HttpContext = new DefaultHttpContext { User = principal } + } + }; + + private static AuthController CreateAuthController( + IConfiguration configuration, + TokenValidationParameters validationParameters, + IIdentityRepository repository, + SessionTokenIssuer issuer) => new( + configuration, + validationParameters, + repository, + issuer, + NullLogger.Instance); + + private static async Task IsPrincipalCurrentAsync( + ClaimsPrincipal principal, + IIdentityRepository repository, + CancellationToken cancellationToken) + { + if (!principal.TryGetUserId(out var userId) + || !principal.TryGetProfileId(out var profileId) + || !principal.TryGetSessionId(out var sessionId)) + return false; + var authenticated = await repository.GetAuthenticatedSessionAsync( + sessionId, DateTimeOffset.UtcNow, cancellationToken); + return authenticated is not null + && authenticated.User.Id == userId + && authenticated.Profile.Id == profileId + && principal.IsInRole(authenticated.User.Role.ToString()); + } + + private static bool IsUnauthorized(IActionResult result) => + result is UnauthorizedResult or UnauthorizedObjectResult; + + private static bool IsSafetyRejection(Exception exception) => + exception is PostgresException { SqlState: "P0001" } + || exception.InnerException is not null && IsSafetyRejection(exception.InnerException); + + private static Models.UserProfile ProfileEntity( + Guid id, + Guid userId, + DateTimeOffset now, + string name = "Home") => new() + { + Id = id, + UserId = userId, + Name = name, + IsDefault = true, + CreatedAt = now, + UpdatedAt = now + }; + + public async Task MigrateDownAsync( + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + await context.Database.MigrateAsync(PreviousMigration, cancellationToken); + var usersExists = await TableExistsAsync(context, "Users", cancellationToken); + var profilesExists = await TableExistsAsync(context, "Profiles", cancellationToken); + return new HouseholdMigrationDownSnapshot( + await context.Database.SqlQueryRaw( + "SELECT COUNT(*)::integer AS \"Value\" FROM \"PlaybackProgresses\"") + .SingleAsync(cancellationToken), + await context.Database.SqlQueryRaw( + "SELECT COUNT(*)::integer AS \"Value\" FROM \"PlaybackPreferences\"") + .SingleAsync(cancellationToken), + await context.Database.SqlQueryRaw( + "SELECT COUNT(*)::integer AS \"Value\" FROM \"ChatConversations\"") + .SingleAsync(cancellationToken), + await context.Database.SqlQueryRaw( + "SELECT COUNT(*)::integer AS \"Value\" FROM \"WebDavTokens\"") + .SingleAsync(cancellationToken), + usersExists, + profilesExists); + } + + private static async Task TableExistsAsync( + Models.ApplicationContext context, + string tableName, + CancellationToken cancellationToken) + { + await context.Database.OpenConnectionAsync(cancellationToken); + await using var command = context.Database.GetDbConnection().CreateCommand(); + command.CommandText = + "SELECT EXISTS (SELECT 1 FROM information_schema.tables " + + "WHERE table_schema = 'public' AND table_name = @name)"; + var parameter = command.CreateParameter(); + parameter.ParameterName = "name"; + parameter.Value = tableName; + command.Parameters.Add(parameter); + return (bool)(await command.ExecuteScalarAsync(cancellationToken))!; + } +} diff --git a/SecondDimensionWatcherReDive.IntegrationTest/Vfs/ScopedDeviceTokenTests.cs b/SecondDimensionWatcherReDive.IntegrationTest/Vfs/ScopedDeviceTokenTests.cs new file mode 100644 index 0000000..a87a815 --- /dev/null +++ b/SecondDimensionWatcherReDive.IntegrationTest/Vfs/ScopedDeviceTokenTests.cs @@ -0,0 +1,137 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using Moq; +using SecondDimensionWatcherReDive.IntegrationTest.Helpers; +using SecondDimensionWatcherReDive.IntegrationTest.TestData; +using SecondDimensionWatcherReDive.WebDav; + +namespace SecondDimensionWatcherReDive.IntegrationTest.Vfs; + +[TestClass] +public sealed class ScopedDeviceTokenTests +{ + private static readonly HttpMethod PropFindMethod = new("PROPFIND"); + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); + private WebDavWebApplicationFactory _factory = null!; + private HttpClient _client = null!; + + [TestInitialize] + public void Setup() + { + _factory = new WebDavWebApplicationFactory("/Anime"); + _factory.ResetState(); + _client = _factory.CreateBasicAuthClient(); + var visible = WebDavMappingFixtures.NewMapping( + "/Anime/episode.mkv", "/disk/visible.mkv"); + var nested = WebDavMappingFixtures.NewMapping( + "/Anime/Sub/subtitle.srt", "/disk/subtitle.srt"); + var adjacent = WebDavMappingFixtures.NewMapping( + "/Anime2/private.mkv", "/disk/private.mkv"); + _factory.Mappings.AddRange([visible, nested, adjacent]); + _factory.FileStoreMock.Setup(store => store.FileInfoAsync( + visible.PhysicalPath, It.IsAny())) + .ReturnsAsync(WebDavMappingFixtures.InfoFor(visible, 42)); + _factory.FileStoreMock.Setup(store => store.OpenReadStreamAsync( + visible.PhysicalPath, It.IsAny())) + .ReturnsAsync(new MemoryStream([1, 2, 3])); + } + + [TestCleanup] + public void Cleanup() + { + _client.Dispose(); + _factory.Dispose(); + } + + [TestMethod] + public async Task VfsRoot_IsRewritten_AndAdjacentPrefixIsHidden() + { + using var response = await _client.GetAsync("/api/vfs/list?path=/"); + + Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); + var entries = await response.Content.ReadFromJsonAsync(JsonOptions); + Assert.IsNotNull(entries); + CollectionAssert.AreEquivalent( + new[] { "episode.mkv", "Sub" }, + entries.Select(entry => entry.Name).ToArray()); + Assert.IsFalse(entries.Any(entry => entry.Name.Contains("Anime2", StringComparison.Ordinal))); + + using var visible = await _client.GetAsync("/api/vfs/stat?path=/episode.mkv"); + using var adjacent = await _client.GetAsync("/api/vfs/stat?path=/../Anime2/private.mkv"); + Assert.AreEqual(HttpStatusCode.OK, visible.StatusCode); + Assert.AreEqual(HttpStatusCode.BadRequest, adjacent.StatusCode); + } + + [TestMethod] + public async Task WebDavRootAndHrefs_AreRewrittenToDeviceNamespace() + { + using var request = new HttpRequestMessage(PropFindMethod, "/webdav/"); + request.Headers.Add(WebDavConstants.Headers.Depth, "1"); + using var response = await _client.SendAsync(request); + + Assert.AreEqual((HttpStatusCode)207, response.StatusCode); + var multiStatus = await WebDavXmlAssertions.ReadMultiStatusAsync(response); + var hrefs = multiStatus.Responses.Select(item => item.Href).ToArray(); + CollectionAssert.Contains(hrefs, "/webdav/"); + CollectionAssert.Contains(hrefs, "/webdav/episode.mkv"); + CollectionAssert.Contains(hrefs, "/webdav/Sub/"); + Assert.IsFalse(hrefs.Any(href => href.Contains("/Anime", StringComparison.Ordinal))); + + using var file = await _client.GetAsync("/webdav/episode.mkv"); + Assert.AreEqual(HttpStatusCode.OK, file.StatusCode); + CollectionAssert.AreEqual(new byte[] { 1, 2, 3 }, await file.Content.ReadAsByteArrayAsync()); + } + + [TestMethod] + public async Task RevokedDeviceToken_IsRejectedImmediately() + { + Assert.IsTrue(await _factory.DeviceTokenRepository.RevokeByIdAsync( + _factory.DeviceTokenRepository.TokenId, + DateTimeOffset.UtcNow, + CancellationToken.None)); + + using var response = await _client.GetAsync("/api/vfs/stat?path=/episode.mkv"); + using var webDav = await SendRootPropFindAsync(); + + Assert.AreEqual(HttpStatusCode.Unauthorized, response.StatusCode); + Assert.AreEqual(HttpStatusCode.Unauthorized, webDav.StatusCode); + } + + [TestMethod] + public async Task ExpiredDeviceToken_IsRejectedByVfsAndWebDav() + { + _factory.DeviceTokenRepository.Expire(); + + using var vfs = await _client.GetAsync("/api/vfs/stat?path=/episode.mkv"); + using var webDav = await SendRootPropFindAsync(); + + Assert.AreEqual(HttpStatusCode.Unauthorized, vfs.StatusCode); + Assert.AreEqual(HttpStatusCode.Unauthorized, webDav.StatusCode); + } + + [TestMethod] + public async Task NonReadDeviceToken_IsRejectedByVfsAndWebDav() + { + _factory.DeviceTokenRepository.SetScope("write"); + + using var vfs = await _client.GetAsync("/api/vfs/stat?path=/episode.mkv"); + using var webDav = await SendRootPropFindAsync(); + + Assert.AreEqual(HttpStatusCode.Unauthorized, vfs.StatusCode); + Assert.AreEqual(HttpStatusCode.Unauthorized, webDav.StatusCode); + } + + private async Task SendRootPropFindAsync() + { + using var request = new HttpRequestMessage(PropFindMethod, "/webdav/"); + request.Headers.Add(WebDavConstants.Headers.Depth, "0"); + return await _client.SendAsync(request); + } + + private sealed record VfsEntryDto( + string Name, + bool IsDirectory, + long? Size, + DateTimeOffset? LastModifiedUtc); +} diff --git a/SecondDimensionWatcherReDive.IntegrationTest/WebDavWebApplicationFactory.cs b/SecondDimensionWatcherReDive.IntegrationTest/WebDavWebApplicationFactory.cs index ae5cf79..ed7f969 100644 --- a/SecondDimensionWatcherReDive.IntegrationTest/WebDavWebApplicationFactory.cs +++ b/SecondDimensionWatcherReDive.IntegrationTest/WebDavWebApplicationFactory.cs @@ -13,6 +13,7 @@ using Microsoft.Extensions.Hosting; using Microsoft.IdentityModel.Tokens; using Moq; +using SecondDimensionWatcherReDive.Framework.Authorization; using SecondDimensionWatcherReDive.Framework.DataRepository; using SecondDimensionWatcherReDive.Framework.FileStore; using SecondDimensionWatcherReDive.Framework.Tasks; @@ -28,6 +29,11 @@ internal sealed class WebDavWebApplicationFactory : WebApplicationFactory FileStoreMock { get; } = new(); public Mock FileStoreProviderMock { get; } = new(); public Helpers.FakeFileMappingRepository MappingRepository { get; } + public FakeWebDavTokenRepository DeviceTokenRepository { get; } private readonly object _mappingsLock = new(); + private readonly UserRole _role; + private bool _sessionRevoked; - public WebDavWebApplicationFactory() + public WebDavWebApplicationFactory( + string virtualRoot = "/", + UserRole role = UserRole.Admin) { + _role = role; MappingRepository = new Helpers.FakeFileMappingRepository(Mappings); + DeviceTokenRepository = new FakeWebDavTokenRepository( + UserId, + TestUserName, + BCrypt.Net.BCrypt.HashPassword(TestPassword), + virtualRoot); FileStoreMock.SetupGet(s => s.Name).Returns("local"); FileStoreProviderMock .Setup(p => p.GetRequiredClient(It.IsAny())) @@ -121,14 +138,52 @@ protected override void ConfigureWebHost(IWebHostBuilder builder) services.RemoveAll(); services.RemoveAll(); services.RemoveAll(); + services.RemoveAll(); services.RemoveAll(); services.AddSingleton(FileStoreMock.Object); services.AddSingleton(FileStoreProviderMock.Object); services.AddSingleton(_ => MappingRepository); services.AddSingleton(_ => new FakeFileExplorer(Mappings, FileStoreMock.Object, MappingRepository)); - services.AddSingleton(_ => - new FakeWebDavTokenRepository(TestUserName, BCrypt.Net.BCrypt.HashPassword(TestPassword))); + services.AddSingleton(_ => DeviceTokenRepository); + var identityRepository = new Mock(); + var user = new UserAccount( + UserId, + TestUserName, + "hash", + _role, + false, + AuthenticatedAt, + AuthenticatedAt); + var profile = new UserProfile( + ProfileId, + UserId, + "Test", + null, + null, + true, + AuthenticatedAt, + AuthenticatedAt); + var session = new UserSession( + SessionId, + UserId, + ProfileId, + "hash", + "integration-test", + AuthenticatedAt, + AuthenticatedAt, + AuthenticatedAt, + DateTimeOffset.UtcNow.AddDays(1), + null); + identityRepository.Setup(repository => repository.FindUserByIdAsync( + UserId, It.IsAny())) + .ReturnsAsync(user); + identityRepository.Setup(repository => repository.GetAuthenticatedSessionAsync( + SessionId, It.IsAny(), It.IsAny())) + .ReturnsAsync(() => _sessionRevoked + ? null + : new AuthenticatedSession(user, profile, session)); + services.AddSingleton(identityRepository.Object); services.AddSingleton(); }); } @@ -160,13 +215,24 @@ public HttpClient CreateBasicAuthClient(string user = TestUserName, string pass public HttpClient CreateUnauthenticatedClient() => CreateClient(new WebApplicationFactoryClientOptions { AllowAutoRedirect = false }); + public void RevokeLoginSession() => _sessionRevoked = true; + public HttpClient CreateJwtClient() { var client = CreateClient(new WebApplicationFactoryClientOptions { AllowAutoRedirect = false }); var keyBytes = Encoding.ASCII.GetBytes(JwtSecret); var creds = new SigningCredentials(new SymmetricSecurityKey(keyBytes), SecurityAlgorithms.HmacSha256); var token = new JwtSecurityToken( - claims: new[] { new Claim(ClaimTypes.Name, TestUserName) }, + claims: + [ + new Claim(ClaimTypes.Name, TestUserName), + new Claim(ClaimTypes.Role, _role.ToString()), + new Claim(IdentityClaimTypes.UserId, UserId.ToString()), + new Claim(IdentityClaimTypes.ProfileId, ProfileId.ToString()), + new Claim(IdentityClaimTypes.SessionId, SessionId.ToString()), + new Claim(IdentityClaimTypes.AuthenticatedAt, + AuthenticatedAt.ToUnixTimeSeconds().ToString()) + ], expires: DateTime.UtcNow.AddMinutes(10), signingCredentials: creds); var jwt = new JwtSecurityTokenHandler().WriteToken(token); diff --git a/SecondDimensionWatcherReDive.Test/AccountsControllerTests.cs b/SecondDimensionWatcherReDive.Test/AccountsControllerTests.cs new file mode 100644 index 0000000..e4f95cc --- /dev/null +++ b/SecondDimensionWatcherReDive.Test/AccountsControllerTests.cs @@ -0,0 +1,193 @@ +using System.Security.Claims; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Configuration; +using Moq; +using SecondDimensionWatcherReDive.Auth; +using SecondDimensionWatcherReDive.Controllers; +using SecondDimensionWatcherReDive.Controllers.External; +using SecondDimensionWatcherReDive.Framework.Authorization; +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Test; + +[TestClass] +public sealed class AccountsControllerTests +{ + private static readonly Guid UserId = Guid.Parse("61000000-0000-0000-0000-000000000001"); + private static readonly Guid ActiveProfileId = Guid.Parse("62000000-0000-0000-0000-000000000001"); + private static readonly Guid ProtectedProfileId = Guid.Parse("62000000-0000-0000-0000-000000000002"); + + [TestMethod] + public async Task ProfileA_CannotClearProfileBPin_WithoutPinOrRecentAuthentication() + { + var repository = new Mock(); + repository.Setup(candidate => candidate.FindProfileAsync( + ProtectedProfileId, It.IsAny())) + .ReturnsAsync(Profile(ProtectedProfileId, BCrypt.Net.BCrypt.HashPassword("2468"))); + var authorization = new Mock(); + authorization.Setup(service => service.AuthorizeAsync( + It.IsAny(), + It.IsAny(), + AccessPolicies.RecentAuthentication)) + .ReturnsAsync(AuthorizationResult.Failed()); + var controller = CreateController(repository, authorization); + + var result = await controller.UpdateProfile( + ProtectedProfileId, + new UpdateProfileRequest( + "Protected", + null, + Pin: string.Empty, + CurrentPin: null, + ReplacePin: true), + CancellationToken.None); + + Assert.IsInstanceOfType(result); + repository.Verify(candidate => candidate.UpdateProfileAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Never); + } + + [TestMethod] + public async Task ProfileBPin_AllowsIntentionalPinReplacement() + { + var repository = new Mock(); + repository.Setup(candidate => candidate.FindProfileAsync( + ProtectedProfileId, It.IsAny())) + .ReturnsAsync(Profile(ProtectedProfileId, BCrypt.Net.BCrypt.HashPassword("2468"))); + repository.Setup(candidate => candidate.UpdateProfileAsync( + ProtectedProfileId, + UserId, + "Protected", + null, + It.IsAny(), + true, + It.IsAny(), + It.IsAny())) + .ReturnsAsync(true); + var authorization = new Mock(); + authorization.Setup(service => service.AuthorizeAsync( + It.IsAny(), + It.IsAny(), + AccessPolicies.RecentAuthentication)) + .ReturnsAsync(AuthorizationResult.Failed()); + var controller = CreateController(repository, authorization); + + var result = await controller.UpdateProfile( + ProtectedProfileId, + new UpdateProfileRequest( + "Protected", + null, + Pin: "1357", + CurrentPin: "2468", + ReplacePin: true), + CancellationToken.None); + + Assert.IsInstanceOfType(result); + repository.Verify(candidate => candidate.UpdateProfileAsync( + ProtectedProfileId, + UserId, + "Protected", + null, + It.Is(hash => hash != null + && BCrypt.Net.BCrypt.Verify("1357", hash)), + true, + It.IsAny(), + It.IsAny()), + Times.Once); + } + + [TestMethod] + public async Task CreateProfile_DuplicateName_ReturnsConflict() + { + var repository = new Mock(); + repository.Setup(candidate => candidate.AddProfileAsync( + It.IsAny(), It.IsAny())) + .ThrowsAsync(new IdentityConflictException("duplicate")); + var controller = CreateController( + repository, new Mock()); + + var result = await controller.CreateProfile( + new CreateProfileRequest("Protected", null, null), + CancellationToken.None); + + Assert.IsInstanceOfType(result); + } + + [TestMethod] + public async Task UpdateProfile_DuplicateSiblingName_ReturnsConflict() + { + var repository = new Mock(); + repository.Setup(candidate => candidate.FindProfileAsync( + ActiveProfileId, It.IsAny())) + .ReturnsAsync(Profile(ActiveProfileId, null)); + repository.Setup(candidate => candidate.UpdateProfileAsync( + ActiveProfileId, + UserId, + "Protected", + null, + null, + false, + It.IsAny(), + It.IsAny())) + .ThrowsAsync(new IdentityConflictException("duplicate")); + var controller = CreateController( + repository, new Mock()); + + var result = await controller.UpdateProfile( + ActiveProfileId, + new UpdateProfileRequest("Protected", null, null), + CancellationToken.None); + + Assert.IsInstanceOfType(result); + } + + private static AccountsController CreateController( + Mock repository, + Mock authorization) + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["JwtSecret"] = "unit-test-secret-long-enough-for-hmac-1234" + }) + .Build(); + return new AccountsController( + repository.Object, + new SessionTokenIssuer(configuration, repository.Object), + authorization.Object) + { + ControllerContext = new ControllerContext + { + HttpContext = new DefaultHttpContext + { + User = new ClaimsPrincipal(new ClaimsIdentity( + [ + new Claim(IdentityClaimTypes.UserId, UserId.ToString()), + new Claim(IdentityClaimTypes.ProfileId, ActiveProfileId.ToString()), + new Claim(ClaimTypes.Role, nameof(UserRole.Member)) + ], "test")) + } + } + }; + } + + private static UserProfile Profile(Guid id, string? pinHash) => new( + id, + UserId, + "Protected", + null, + pinHash, + false, + DateTimeOffset.UtcNow, + DateTimeOffset.UtcNow); +} diff --git a/SecondDimensionWatcherReDive.Test/AnimationInfoControllerTests.cs b/SecondDimensionWatcherReDive.Test/AnimationInfoControllerTests.cs index 1cf8e25..0a43dff 100644 --- a/SecondDimensionWatcherReDive.Test/AnimationInfoControllerTests.cs +++ b/SecondDimensionWatcherReDive.Test/AnimationInfoControllerTests.cs @@ -1,8 +1,10 @@ +using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Caching.Distributed; using Moq; using SecondDimensionWatcherReDive.Controllers; +using SecondDimensionWatcherReDive.Framework.Authorization; using SecondDimensionWatcherReDive.Framework.FileDownload; using SecondDimensionWatcherReDive.Framework.DataRepository; using SecondDimensionWatcherReDive.Utils.FileStore; @@ -18,6 +20,7 @@ public class AnimationInfoControllerTests private Mock _providerMock = null!; private Mock _downloadClientMock = null!; private Mock _fileMapperMock = null!; + private Mock _authorizationServiceMock = null!; private AnimationInfoController _controller = null!; [TestInitialize] @@ -29,6 +32,7 @@ public void Setup() _providerMock = new Mock(); _downloadClientMock = new Mock(); _fileMapperMock = new Mock(); + _authorizationServiceMock = new Mock(); _providerMock .Setup(p => p.GetRequiredClient(It.IsAny())) @@ -39,7 +43,8 @@ public void Setup() _fileMappingRepoMock.Object, _cacheMock.Object, _providerMock.Object, - _fileMapperMock.Object) + _fileMapperMock.Object, + _authorizationServiceMock.Object) { ControllerContext = new ControllerContext { @@ -341,6 +346,35 @@ public async Task CancelDownload_Success_SetsIsDownloadTrackedFalseAndUpdates() cancellationAttemptId.Value, It.Is(token => token.CanBeCanceled && !token.IsCancellationRequested)), Times.Once); + _authorizationServiceMock.Verify(service => service.AuthorizeAsync( + It.IsAny(), + It.IsAny(), + It.IsAny()), Times.Never); + } + + [TestMethod] + public async Task CancelDownload_RemoveFileWithoutRecentAdministrator_ReturnsForbidBeforeLookup() + { + var id = Guid.NewGuid(); + _authorizationServiceMock.Setup(service => service.AuthorizeAsync( + It.IsAny(), + null, + AccessPolicies.RecentAdministrator)) + .ReturnsAsync(AuthorizationResult.Failed()); + + var result = await _controller.CancelDownload( + id, removeFile: true, CancellationToken.None); + + Assert.IsInstanceOfType(result); + _repoMock.Verify(repository => repository.FindByIdAsync( + It.IsAny(), It.IsAny()), Times.Never); + _downloadClientMock.Verify(client => client.CancelDownloadTaskAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), Times.Never); } [TestMethod] diff --git a/SecondDimensionWatcherReDive.Test/BasicAuthenticationHandlerTests.cs b/SecondDimensionWatcherReDive.Test/BasicAuthenticationHandlerTests.cs index fa0926d..fb5d830 100644 --- a/SecondDimensionWatcherReDive.Test/BasicAuthenticationHandlerTests.cs +++ b/SecondDimensionWatcherReDive.Test/BasicAuthenticationHandlerTests.cs @@ -7,6 +7,7 @@ using Microsoft.Extensions.Options; using Moq; using SecondDimensionWatcherReDive.Auth; +using SecondDimensionWatcherReDive.Framework.Authorization; using SecondDimensionWatcherReDive.Framework.DataRepository; namespace SecondDimensionWatcherReDive.Test; @@ -16,6 +17,7 @@ public class BasicAuthenticationHandlerTests { private const string ValidUser = "alice"; private const string ValidPassword = "correct-horse"; + private static readonly Guid UserId = Guid.Parse("10000000-0000-0000-0000-000000000001"); private string _hash = null!; [TestInitialize] @@ -35,6 +37,12 @@ public void Setup() var services = new ServiceCollection(); services.AddSingleton(repo.Object); + var identityRepo = new Mock(); + identityRepo.Setup(r => r.FindUserByIdAsync(UserId, It.IsAny())) + .ReturnsAsync(new UserAccount( + UserId, "admin", "hash", UserRole.Admin, false, + DateTimeOffset.UtcNow, DateTimeOffset.UtcNow)); + services.AddSingleton(identityRepo.Object); var provider = services.BuildServiceProvider(); var optionsMonitor = new Mock>(); @@ -53,8 +61,14 @@ public void Setup() return (handler, httpContext, repo); } - private WebDavToken SeededToken(string username = ValidUser) => - new(Guid.NewGuid(), username, _hash, null, DateTimeOffset.UtcNow); + private WebDavToken SeededToken( + string username = ValidUser, + string scope = "read", + string root = "/Anime", + DateTimeOffset? expiresAt = null, + DateTimeOffset? revokedAt = null) => + new(Guid.NewGuid(), UserId, username, _hash, null, DateTimeOffset.UtcNow, + scope, root, expiresAt ?? DateTimeOffset.UtcNow.AddDays(1), revokedAt); private static string BasicHeader(string user, string password) => "Basic " + Convert.ToBase64String(Encoding.UTF8.GetBytes($"{user}:{password}")); @@ -116,6 +130,28 @@ public async Task ValidCredentials_Succeed() var result = await handler.AuthenticateAsync(); Assert.IsTrue(result.Succeeded); Assert.AreEqual(ValidUser, result.Principal!.Identity!.Name); + Assert.AreEqual("/Anime", result.Principal.FindFirst(IdentityClaimTypes.VirtualRoot)?.Value); + Assert.AreEqual("read", result.Principal.FindFirst(IdentityClaimTypes.DeviceScope)?.Value); + Assert.AreEqual(UserId.ToString(), result.Principal.FindFirst(IdentityClaimTypes.UserId)?.Value); + } + + [TestMethod] + public async Task RevokedExpiredOrNonReadToken_Fails() + { + var revoked = await CreateHandlerAsync( + BasicHeader(ValidUser, ValidPassword), + SeededToken(revokedAt: DateTimeOffset.UtcNow)); + Assert.IsFalse((await revoked.handler.AuthenticateAsync()).Succeeded); + + var expired = await CreateHandlerAsync( + BasicHeader(ValidUser, ValidPassword), + SeededToken(expiresAt: DateTimeOffset.UtcNow.AddMinutes(-1))); + Assert.IsFalse((await expired.handler.AuthenticateAsync()).Succeeded); + + var write = await CreateHandlerAsync( + BasicHeader(ValidUser, ValidPassword), + SeededToken(scope: "write")); + Assert.IsFalse((await write.handler.AuthenticateAsync()).Succeeded); } [TestMethod] diff --git a/SecondDimensionWatcherReDive.Test/ConversationTitleGeneratorTests.cs b/SecondDimensionWatcherReDive.Test/ConversationTitleGeneratorTests.cs index 709ae78..404954a 100644 --- a/SecondDimensionWatcherReDive.Test/ConversationTitleGeneratorTests.cs +++ b/SecondDimensionWatcherReDive.Test/ConversationTitleGeneratorTests.cs @@ -11,6 +11,8 @@ namespace SecondDimensionWatcherReDive.Test; [TestClass] public class ConversationTitleGeneratorTests { + private static readonly Guid ProfileId = Guid.Parse("10000000-0000-0000-0000-000000000001"); + private static IServiceProvider ServicesWith(IAIEngine? engine) { var services = new ServiceCollection(); @@ -125,15 +127,18 @@ public async Task TryAutoTitle_SavesTitleWhenEligible() var convId = Guid.NewGuid(); var engine = EngineReturning("Anime subscription"); var repo = new Mock(); - repo.Setup(r => r.GetConversationWithMessagesAsync(convId, It.IsAny())) + repo.Setup(r => r.GetConversationWithMessagesAsync( + convId, ProfileId, It.IsAny())) .ReturnsAsync(new ChatConversationDetail(convId, null, DateTimeOffset.Now, DateTimeOffset.Now, [])); var gen = new ConversationTitleGenerator(ServicesWith(engine.Object), repo.Object, NullLogger.Instance); - await gen.TryAutoTitleAsync(convId, "请订阅新番", "好的", null, CancellationToken.None); + await gen.TryAutoTitleAsync( + convId, ProfileId, "请订阅新番", "好的", null, CancellationToken.None); - repo.Verify(r => r.UpdateConversationTitleAsync(convId, "Anime subscription", It.IsAny()), + repo.Verify(r => r.UpdateConversationTitleAsync( + convId, ProfileId, "Anime subscription", It.IsAny()), Times.Once); } @@ -143,15 +148,17 @@ public async Task TryAutoTitle_SkipsWhenTitleAlreadySet() var convId = Guid.NewGuid(); var engine = EngineReturning("Generated"); var repo = new Mock(); - repo.Setup(r => r.GetConversationWithMessagesAsync(convId, It.IsAny())) + repo.Setup(r => r.GetConversationWithMessagesAsync( + convId, ProfileId, It.IsAny())) .ReturnsAsync(new ChatConversationDetail(convId, "User chose this", DateTimeOffset.Now, DateTimeOffset.Now, [])); var gen = new ConversationTitleGenerator(ServicesWith(engine.Object), repo.Object, NullLogger.Instance); - await gen.TryAutoTitleAsync(convId, "u", "a", null, CancellationToken.None); + await gen.TryAutoTitleAsync(convId, ProfileId, "u", "a", null, CancellationToken.None); - repo.Verify(r => r.UpdateConversationTitleAsync(It.IsAny(), It.IsAny(), It.IsAny()), + repo.Verify(r => r.UpdateConversationTitleAsync( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); } @@ -161,15 +168,17 @@ public async Task TryAutoTitle_DoesNotSaveOnEmptyOutput() var convId = Guid.NewGuid(); var engine = EngineReturning(" "); var repo = new Mock(); - repo.Setup(r => r.GetConversationWithMessagesAsync(convId, It.IsAny())) + repo.Setup(r => r.GetConversationWithMessagesAsync( + convId, ProfileId, It.IsAny())) .ReturnsAsync(new ChatConversationDetail(convId, null, DateTimeOffset.Now, DateTimeOffset.Now, [])); var gen = new ConversationTitleGenerator(ServicesWith(engine.Object), repo.Object, NullLogger.Instance); - await gen.TryAutoTitleAsync(convId, "u", "a", null, CancellationToken.None); + await gen.TryAutoTitleAsync(convId, ProfileId, "u", "a", null, CancellationToken.None); - repo.Verify(r => r.UpdateConversationTitleAsync(It.IsAny(), It.IsAny(), It.IsAny()), + repo.Verify(r => r.UpdateConversationTitleAsync( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); } @@ -182,15 +191,17 @@ public async Task TryAutoTitle_SwallowsEngineExceptions() .Returns(Throwing()); var repo = new Mock(); - repo.Setup(r => r.GetConversationWithMessagesAsync(convId, It.IsAny())) + repo.Setup(r => r.GetConversationWithMessagesAsync( + convId, ProfileId, It.IsAny())) .ReturnsAsync(new ChatConversationDetail(convId, null, DateTimeOffset.Now, DateTimeOffset.Now, [])); var gen = new ConversationTitleGenerator(ServicesWith(engine.Object), repo.Object, NullLogger.Instance); - await gen.TryAutoTitleAsync(convId, "u", "a", null, CancellationToken.None); + await gen.TryAutoTitleAsync(convId, ProfileId, "u", "a", null, CancellationToken.None); - repo.Verify(r => r.UpdateConversationTitleAsync(It.IsAny(), It.IsAny(), It.IsAny()), + repo.Verify(r => r.UpdateConversationTitleAsync( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); static async IAsyncEnumerable Throwing() diff --git a/SecondDimensionWatcherReDive.Test/DevicePathScopeTests.cs b/SecondDimensionWatcherReDive.Test/DevicePathScopeTests.cs new file mode 100644 index 0000000..a460404 --- /dev/null +++ b/SecondDimensionWatcherReDive.Test/DevicePathScopeTests.cs @@ -0,0 +1,41 @@ +using SecondDimensionWatcherReDive.Auth; + +namespace SecondDimensionWatcherReDive.Test; + +[TestClass] +public sealed class DevicePathScopeTests +{ + [TestMethod] + public void ScopedRoot_MapsPublicRootAndChildren() + { + Assert.IsTrue(DevicePathScope.TryMapPublicToInternal( + "/", "/Anime", out var publicRoot, out var internalRoot)); + Assert.AreEqual("/", publicRoot); + Assert.AreEqual("/Anime", internalRoot); + + Assert.IsTrue(DevicePathScope.TryMapPublicToInternal( + "/Season/episode.mkv", "/Anime", out var publicChild, out var internalChild)); + Assert.AreEqual("/Season/episode.mkv", publicChild); + Assert.AreEqual("/Anime/Season/episode.mkv", internalChild); + } + + [TestMethod] + public void InternalMapping_UsesPathSegments_NotStringPrefixes() + { + Assert.IsTrue(DevicePathScope.TryMapInternalToPublic( + "/Anime/episode.mkv", "/Anime", out var publicPath)); + Assert.AreEqual("/episode.mkv", publicPath); + + Assert.IsFalse(DevicePathScope.TryMapInternalToPublic( + "/Anime2/private.mkv", "/Anime", out _)); + } + + [TestMethod] + public void TraversalOrBackslash_IsRejected() + { + Assert.IsFalse(DevicePathScope.TryMapPublicToInternal( + "/../Anime2/private.mkv", "/Anime", out _, out _)); + Assert.IsFalse(DevicePathScope.TryMapPublicToInternal( + "/Season\\private.mkv", "/Anime", out _, out _)); + } +} diff --git a/SecondDimensionWatcherReDive.Test/ManageDownloadsToolAuthorizationTests.cs b/SecondDimensionWatcherReDive.Test/ManageDownloadsToolAuthorizationTests.cs new file mode 100644 index 0000000..918c2b2 --- /dev/null +++ b/SecondDimensionWatcherReDive.Test/ManageDownloadsToolAuthorizationTests.cs @@ -0,0 +1,154 @@ +using System.Security.Claims; +using System.Text.Json; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http; +using Moq; +using SecondDimensionWatcherReDive.AI.Models; +using SecondDimensionWatcherReDive.Chat.Tools; +using SecondDimensionWatcherReDive.Framework.Authorization; +using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Framework.FileDownload; + +namespace SecondDimensionWatcherReDive.Test; + +[TestClass] +public class ManageDownloadsToolAuthorizationTests +{ + [TestMethod] + public async Task CancelWithRemoveFile_MemberIsRejectedBeforeDownloadClientCall() + { + var (tool, repository, client, authorization, animation) = CreateTool(); + authorization.Setup(service => service.AuthorizeAsync( + It.IsAny(), + null, + AccessPolicies.RecentAdministrator)) + .ReturnsAsync(AuthorizationResult.Failed()); + + var result = await tool.ExecuteAsync(Arguments( + animation.Id, removeFile: true), CancellationToken.None); + + var failure = Assert.IsInstanceOfType(result); + StringAssert.Contains(failure.Error, "administrator"); + repository.Verify(repo => repo.TryBeginCancelDownloadAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), Times.Never); + client.Verify(downloadClient => downloadClient.CancelDownloadTaskAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), Times.Never); + } + + [TestMethod] + public async Task CancelWithoutRemoveFile_MemberCanCancel() + { + var (tool, repository, client, authorization, animation) = CreateTool(); + repository.Setup(repo => repo.TryBeginCancelDownloadAsync( + animation.Id, + animation.DownloadAttemptId, + It.IsAny(), + It.IsAny())) + .ReturnsAsync(true); + client.Setup(downloadClient => downloadClient.CancelDownloadTaskAsync( + animation.Id, + animation.DownloadUrl, + animation.CachedDownloadData, + animation.AdditionalDownloadInfo, + false, + CancellationToken.None)) + .ReturnsAsync(new CancelDownloadResult(true, false)); + var result = await tool.ExecuteAsync(Arguments( + animation.Id, removeFile: false), CancellationToken.None); + + var success = Assert.IsInstanceOfType>(result); + Assert.IsTrue(success.Result); + authorization.Verify(service => service.AuthorizeAsync( + It.IsAny(), + It.IsAny(), + It.IsAny()), Times.Never); + client.Verify(downloadClient => downloadClient.CancelDownloadTaskAsync( + animation.Id, + animation.DownloadUrl, + animation.CachedDownloadData, + animation.AdditionalDownloadInfo, + false, + CancellationToken.None), Times.Once); + } + + private static ( + ManageDownloadsTool Tool, + Mock Repository, + Mock Client, + Mock Authorization, + AnimationInfo Animation) CreateTool() + { + var animation = new AnimationInfo( + Guid.NewGuid(), + "Title", + "Description", + DateTimeOffset.UtcNow, + "https://example.invalid/item.torrent", + "torrent", + [], + "cached", + true, + default, + default, + false, + null, + null, + null, + null, + null, + null, + false, + 0) + { + DownloadAttemptId = Guid.NewGuid() + }; + var repository = new Mock(); + repository.Setup(repo => repo.FindByIdAsync( + animation.Id, CancellationToken.None)) + .ReturnsAsync(animation); + var mappingRepository = new Mock(); + mappingRepository.Setup(repo => repo.TryFinalizeDownloadCancellationAsync( + animation.Id, + animation.DownloadAttemptId, + It.IsAny(), + It.IsAny())) + .ReturnsAsync(true); + var client = new Mock(); + var provider = new Mock(); + provider.Setup(value => value.GetClient(animation.DownloadType)) + .Returns(client.Object); + var httpContextAccessor = new HttpContextAccessor + { + HttpContext = new DefaultHttpContext + { + User = new ClaimsPrincipal(new ClaimsIdentity( + [new Claim(ClaimTypes.Role, nameof(UserRole.Member))], + "test")) + } + }; + var authorization = new Mock(); + var tool = new ManageDownloadsTool( + repository.Object, + mappingRepository.Object, + provider.Object, + httpContextAccessor, + authorization.Object); + return (tool, repository, client, authorization, animation); + } + + private static JsonElement Arguments(Guid animationId, bool removeFile) => + JsonSerializer.SerializeToElement( + new ManageDownloadsParams( + ManageDownloadsAction.Cancel, + animationId.ToString(), + removeFile), + ToolJsonOptions.Options); +} diff --git a/SecondDimensionWatcherReDive.Test/PlaybackControllerTests.cs b/SecondDimensionWatcherReDive.Test/PlaybackControllerTests.cs index 31486bf..0275277 100644 --- a/SecondDimensionWatcherReDive.Test/PlaybackControllerTests.cs +++ b/SecondDimensionWatcherReDive.Test/PlaybackControllerTests.cs @@ -4,6 +4,7 @@ using Moq; using SecondDimensionWatcherReDive.Controllers; using SecondDimensionWatcherReDive.Controllers.External; +using SecondDimensionWatcherReDive.Framework.Authorization; using SecondDimensionWatcherReDive.Framework.DataRepository; using DataAnimationInfo = SecondDimensionWatcherReDive.Framework.DataRepository.AnimationInfo; using DataAnimation = SecondDimensionWatcherReDive.Framework.DataRepository.Animation; @@ -36,7 +37,7 @@ public void Setup() HttpContext = new DefaultHttpContext { User = new ClaimsPrincipal(new ClaimsIdentity( - [new Claim("Id", UserId.ToString())], + [new Claim(IdentityClaimTypes.ProfileId, UserId.ToString())], "test")) } } diff --git a/SecondDimensionWatcherReDive.Test/WebDavTokenControllerTests.cs b/SecondDimensionWatcherReDive.Test/WebDavTokenControllerTests.cs index e8eed94..9449e21 100644 --- a/SecondDimensionWatcherReDive.Test/WebDavTokenControllerTests.cs +++ b/SecondDimensionWatcherReDive.Test/WebDavTokenControllerTests.cs @@ -1,7 +1,10 @@ +using System.Security.Claims; +using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Moq; using SecondDimensionWatcherReDive.Controllers; using SecondDimensionWatcherReDive.Controllers.External; +using SecondDimensionWatcherReDive.Framework.Authorization; using SecondDimensionWatcherReDive.Framework.DataRepository; namespace SecondDimensionWatcherReDive.Test; @@ -10,7 +13,10 @@ namespace SecondDimensionWatcherReDive.Test; public class WebDavTokenControllerTests { private Mock _repo = null!; + private Mock _identityRepo = null!; + private Mock _mappingRepo = null!; private WebDavTokenController _controller = null!; + private readonly Guid _userId = Guid.Parse("10000000-0000-0000-0000-000000000001"); [TestInitialize] public void Setup() @@ -18,7 +24,27 @@ public void Setup() _repo = new Mock(); _repo.Setup(r => r.ExistsByUsernameAsync(It.IsAny(), It.IsAny())) .ReturnsAsync(false); - _controller = new WebDavTokenController(_repo.Object); + _identityRepo = new Mock(); + _identityRepo.Setup(r => r.FindUserByIdAsync(_userId, It.IsAny())) + .ReturnsAsync(new UserAccount( + _userId, "admin", "hash", UserRole.Admin, false, + DateTimeOffset.UtcNow, DateTimeOffset.UtcNow)); + _mappingRepo = new Mock(); + _controller = new WebDavTokenController( + _repo.Object, _identityRepo.Object, _mappingRepo.Object) + { + ControllerContext = new ControllerContext + { + HttpContext = new DefaultHttpContext + { + User = new ClaimsPrincipal(new ClaimsIdentity( + [ + new Claim(IdentityClaimTypes.UserId, _userId.ToString()), + new Claim(ClaimTypes.Role, nameof(UserRole.Admin)) + ], "test")) + } + } + }; } [TestMethod] @@ -26,8 +52,8 @@ public async Task ListTokens_ReturnsSummariesWithoutHash() { var seeded = new List { - new(Guid.NewGuid(), "alice", "hash-1", "first key", DateTimeOffset.UtcNow.AddMinutes(-1)), - new(Guid.NewGuid(), "bob", "hash-2", null, DateTimeOffset.UtcNow) + Token("alice", "hash-1", "first key", DateTimeOffset.UtcNow.AddMinutes(-1)), + Token("bob", "hash-2", null, DateTimeOffset.UtcNow) }; _repo.Setup(r => r.GetAllOrderedAsync(It.IsAny())) .ReturnsAsync(seeded); @@ -63,6 +89,10 @@ public async Task CreateToken_AutoGeneratesUsernameWhenMissing() Assert.AreNotEqual(payload.Token, captured.TokenHash, "TokenHash must not be plaintext."); Assert.IsTrue(BCrypt.Net.BCrypt.Verify(payload.Token, captured.TokenHash)); Assert.IsNull(captured.Description); + Assert.AreEqual(_userId, captured.UserId); + Assert.AreEqual("read", captured.Scope); + Assert.AreEqual("/", captured.VirtualRoot); + Assert.IsTrue(captured.ExpiresAt > DateTimeOffset.UtcNow.AddDays(364)); } [TestMethod] @@ -116,7 +146,8 @@ public async Task CreateToken_ReturnsConflictWhenUsernameTaken() [TestMethod] public async Task DeleteToken_ReturnsNotFoundWhenMissing() { - _repo.Setup(r => r.RemoveByIdAsync(It.IsAny(), It.IsAny())) + _repo.Setup(r => r.RevokeByIdAsync( + It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(false); var response = await _controller.DeleteToken(Guid.NewGuid(), CancellationToken.None); Assert.IsInstanceOfType(response, typeof(NotFoundResult)); @@ -125,9 +156,18 @@ public async Task DeleteToken_ReturnsNotFoundWhenMissing() [TestMethod] public async Task DeleteToken_ReturnsNoContentOnSuccess() { - _repo.Setup(r => r.RemoveByIdAsync(It.IsAny(), It.IsAny())) + _repo.Setup(r => r.RevokeByIdAsync( + It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(true); var response = await _controller.DeleteToken(Guid.NewGuid(), CancellationToken.None); Assert.IsInstanceOfType(response, typeof(NoContentResult)); } + + private WebDavToken Token( + string username, + string hash, + string? description, + DateTimeOffset createdAt) => + new(Guid.NewGuid(), _userId, username, hash, description, createdAt, + "read", "/", createdAt.AddYears(1), null); } diff --git a/SecondDimensionWatcherReDive/Auth/BasicAuthenticationHandler.cs b/SecondDimensionWatcherReDive/Auth/BasicAuthenticationHandler.cs index ef43345..0fd6778 100644 --- a/SecondDimensionWatcherReDive/Auth/BasicAuthenticationHandler.cs +++ b/SecondDimensionWatcherReDive/Auth/BasicAuthenticationHandler.cs @@ -4,6 +4,7 @@ using System.Text.Encodings.Web; using Microsoft.AspNetCore.Authentication; using Microsoft.Extensions.Options; +using SecondDimensionWatcherReDive.Framework.Authorization; using SecondDimensionWatcherReDive.Framework.DataRepository; namespace SecondDimensionWatcherReDive.Auth; @@ -49,7 +50,17 @@ protected override async Task HandleAuthenticateAsync() var repository = Context.RequestServices.GetRequiredService(); var record = await repository.FindByUsernameAsync(username, Context.RequestAborted); - if (record is null) + var now = DateTimeOffset.UtcNow; + if (record is null + || record.RevokedAt is not null + || record.ExpiresAt is { } expiresAt && expiresAt <= now + || !string.Equals(record.Scope, "read", StringComparison.Ordinal) + || !DevicePathScope.TryNormalizeAbsolutePath(record.VirtualRoot, out var virtualRoot)) + return AuthenticateResult.Fail("Invalid credentials."); + + var identityRepository = Context.RequestServices.GetRequiredService(); + var user = await identityRepository.FindUserByIdAsync(record.UserId, Context.RequestAborted); + if (user is null || user.IsDisabled) return AuthenticateResult.Fail("Invalid credentials."); bool verified; @@ -65,7 +76,15 @@ protected override async Task HandleAuthenticateAsync() if (!verified) return AuthenticateResult.Fail("Invalid credentials."); - var identity = new ClaimsIdentity([new Claim(ClaimTypes.Name, username)], Scheme.Name); + var identity = new ClaimsIdentity( + [ + new Claim(ClaimTypes.Name, username), + new Claim(ClaimTypes.Role, user.Role.ToString()), + new Claim(IdentityClaimTypes.UserId, user.Id.ToString()), + new Claim(IdentityClaimTypes.DeviceTokenId, record.Id.ToString()), + new Claim(IdentityClaimTypes.DeviceScope, record.Scope), + new Claim(IdentityClaimTypes.VirtualRoot, virtualRoot) + ], Scheme.Name); var principal = new ClaimsPrincipal(identity); return AuthenticateResult.Success(new AuthenticationTicket(principal, Scheme.Name)); } diff --git a/SecondDimensionWatcherReDive/Auth/DevicePathScope.cs b/SecondDimensionWatcherReDive/Auth/DevicePathScope.cs new file mode 100644 index 0000000..10fd1fc --- /dev/null +++ b/SecondDimensionWatcherReDive/Auth/DevicePathScope.cs @@ -0,0 +1,95 @@ +using System.Security.Claims; +using SecondDimensionWatcherReDive.Framework.Authorization; + +namespace SecondDimensionWatcherReDive.Auth; + +internal static class DevicePathScope +{ + public static string GetVirtualRoot(ClaimsPrincipal principal) => + TryNormalizeAbsolutePath( + principal.FindFirst(IdentityClaimTypes.VirtualRoot)?.Value, + out var root) + ? root + : "/"; + + public static bool TryNormalizeAbsolutePath(string? raw, out string normalized) + { + if (string.IsNullOrEmpty(raw)) + { + normalized = "/"; + return true; + } + + if (!raw.StartsWith("/", StringComparison.Ordinal) + || raw.Contains('\\', StringComparison.Ordinal) + || raw.Any(char.IsControl)) + { + normalized = string.Empty; + return false; + } + + var segments = raw.Split('/', StringSplitOptions.RemoveEmptyEntries); + if (segments.Any(segment => segment is "." or "..")) + { + normalized = string.Empty; + return false; + } + + normalized = segments.Length == 0 ? "/" : "/" + string.Join('/', segments); + return true; + } + + public static bool TryMapPublicToInternal( + string? publicPath, + string virtualRoot, + out string normalizedPublicPath, + out string internalPath) + { + if (!TryNormalizeAbsolutePath(publicPath, out normalizedPublicPath) + || !TryNormalizeAbsolutePath(virtualRoot, out var root)) + { + internalPath = string.Empty; + return false; + } + + internalPath = root switch + { + "/" => normalizedPublicPath, + _ when normalizedPublicPath == "/" => root, + _ => root + normalizedPublicPath + }; + return true; + } + + public static bool TryMapInternalToPublic( + string internalPath, + string virtualRoot, + out string publicPath) + { + publicPath = string.Empty; + if (!TryNormalizeAbsolutePath(internalPath, out var normalizedInternal) + || !TryNormalizeAbsolutePath(virtualRoot, out var root)) + return false; + + if (root == "/") + { + publicPath = normalizedInternal; + return true; + } + + if (normalizedInternal == root) + { + publicPath = "/"; + return true; + } + + // Include the slash in the prefix: /Anime is a parent of /Anime/file, + // but never of /Anime2/file. + var rootedPrefix = root + "/"; + if (!normalizedInternal.StartsWith(rootedPrefix, StringComparison.Ordinal)) + return false; + + publicPath = normalizedInternal[root.Length..]; + return true; + } +} diff --git a/SecondDimensionWatcherReDive/Auth/SessionTokenIssuer.cs b/SecondDimensionWatcherReDive/Auth/SessionTokenIssuer.cs new file mode 100644 index 0000000..b9df7d3 --- /dev/null +++ b/SecondDimensionWatcherReDive/Auth/SessionTokenIssuer.cs @@ -0,0 +1,133 @@ +using System.IdentityModel.Tokens.Jwt; +using System.Security.Claims; +using System.Security.Cryptography; +using System.Text; +using Microsoft.IdentityModel.Tokens; +using SecondDimensionWatcherReDive.Framework.Authorization; +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Auth; + +internal sealed record IssuedSessionTokens( + string AccessToken, + string RefreshToken, + Guid SessionId, + Guid ProfileId); + +internal sealed class SessionTokenIssuer( + IConfiguration configuration, + IIdentityRepository identityRepository) +{ + internal static readonly TimeSpan AccessTokenLifetime = TimeSpan.FromMinutes(10); + internal static readonly TimeSpan RefreshTokenLifetime = TimeSpan.FromDays(30); + + public async Task CreateSessionAsync( + UserAccount user, + UserProfile profile, + string? deviceName, + CancellationToken cancellationToken) + { + var now = DateTimeOffset.UtcNow; + var refreshToken = GenerateRefreshToken(); + var session = new UserSession( + Guid.NewGuid(), + user.Id, + profile.Id, + HashRefreshToken(refreshToken), + NormalizeDeviceName(deviceName), + now, + now, + now, + now + RefreshTokenLifetime, + null); + await identityRepository.AddSessionAsync(session, cancellationToken); + return new IssuedSessionTokens( + GenerateAccessToken(user, profile, session), + refreshToken, + session.Id, + profile.Id); + } + + public async Task RotateSessionAsync( + AuthenticatedSession authenticatedSession, + UserProfile profile, + string expectedRefreshToken, + bool reauthenticated, + CancellationToken cancellationToken) + { + var now = DateTimeOffset.UtcNow; + var refreshToken = GenerateRefreshToken(); + var authenticatedAt = reauthenticated ? now : (DateTimeOffset?)null; + if (!await identityRepository.TryRotateSessionAsync( + authenticatedSession.Session.Id, + HashRefreshToken(expectedRefreshToken), + HashRefreshToken(refreshToken), + profile.Id, + authenticatedAt, + now, + now + RefreshTokenLifetime, + cancellationToken)) + return null; + + var session = authenticatedSession.Session with + { + ActiveProfileId = profile.Id, + RefreshTokenHash = HashRefreshToken(refreshToken), + AuthenticatedAt = authenticatedAt ?? authenticatedSession.Session.AuthenticatedAt, + LastSeenAt = now, + ExpiresAt = now + RefreshTokenLifetime + }; + return new IssuedSessionTokens( + GenerateAccessToken(authenticatedSession.User, profile, session), + refreshToken, + session.Id, + profile.Id); + } + + public static string HashRefreshToken(string refreshToken) => + Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(refreshToken))); + + private string GenerateAccessToken( + UserAccount user, + UserProfile profile, + UserSession session) + { + var key = Encoding.ASCII.GetBytes(configuration["JwtSecret"]!); + var descriptor = new SecurityTokenDescriptor + { + Subject = new ClaimsIdentity( + [ + new Claim(JwtRegisteredClaimNames.Sub, user.Id.ToString()), + new Claim(ClaimTypes.NameIdentifier, user.Id.ToString()), + new Claim(ClaimTypes.Name, user.Username), + new Claim(ClaimTypes.Role, user.Role.ToString()), + new Claim(IdentityClaimTypes.UserId, user.Id.ToString()), + new Claim(IdentityClaimTypes.ProfileId, profile.Id.ToString()), + new Claim(IdentityClaimTypes.SessionId, session.Id.ToString()), + new Claim(IdentityClaimTypes.AuthenticatedAt, + session.AuthenticatedAt.ToUnixTimeSeconds().ToString()), + new Claim("Id", profile.Id.ToString()), + new Claim(JwtRegisteredClaimNames.Jti, Guid.NewGuid().ToString()) + ]), + Expires = DateTime.UtcNow.Add(AccessTokenLifetime), + SigningCredentials = new SigningCredentials( + new SymmetricSecurityKey(key), + SecurityAlgorithms.HmacSha256Signature) + }; + var handler = new JwtSecurityTokenHandler(); + return handler.WriteToken(handler.CreateToken(descriptor)); + } + + private static string GenerateRefreshToken() => + Convert.ToBase64String(RandomNumberGenerator.GetBytes(48)) + .TrimEnd('=') + .Replace('+', '-') + .Replace('/', '_'); + + private static string? NormalizeDeviceName(string? value) + { + var trimmed = value?.Trim(); + if (string.IsNullOrEmpty(trimmed)) return null; + return trimmed.Length <= 128 ? trimmed : trimmed[..128]; + } +} diff --git a/SecondDimensionWatcherReDive/Controllers/AccountsController.cs b/SecondDimensionWatcherReDive/Controllers/AccountsController.cs new file mode 100644 index 0000000..26f6c34 --- /dev/null +++ b/SecondDimensionWatcherReDive/Controllers/AccountsController.cs @@ -0,0 +1,319 @@ +using System.Text.RegularExpressions; +using Microsoft.AspNetCore.Authentication.JwtBearer; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using SecondDimensionWatcherReDive.Auth; +using SecondDimensionWatcherReDive.Framework.Authorization; +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Controllers; + +[ApiController] +[Route("api/accounts")] +[Authorize(AuthenticationSchemes = JwtBearerDefaults.AuthenticationScheme)] +internal sealed partial class AccountsController( + IIdentityRepository identityRepository, + SessionTokenIssuer tokenIssuer, + IAuthorizationService authorizationService) : ControllerBase +{ + [GeneratedRegex("^[a-z0-9._-]{3,64}$")] + private static partial Regex UsernamePattern(); + + [GeneratedRegex("^[0-9]{4,8}$")] + private static partial Regex PinPattern(); + + [HttpGet("profiles")] + public async Task GetProfiles(CancellationToken cancellationToken) + { + if (!User.TryGetUserId(out var userId)) return Unauthorized(); + var profiles = await identityRepository.GetProfilesAsync(userId, cancellationToken); + return Ok(profiles.Select(AuthController.ToProfileResponse).ToList()); + } + + [HttpPost("profiles")] + [Authorize(Policy = AccessPolicies.ContentWrite)] + public async Task CreateProfile( + [FromBody] External.CreateProfileRequest request, + CancellationToken cancellationToken) + { + if (!User.TryGetUserId(out var userId)) return Unauthorized(); + if (!TryNormalizeProfile(request.Name, request.Avatar, request.Pin, + out var name, out var avatar, out var pinHash)) + return BadRequest(); + var now = DateTimeOffset.UtcNow; + UserProfile profile; + try + { + profile = await identityRepository.AddProfileAsync( + new UserProfile( + Guid.NewGuid(), userId, name, avatar, pinHash, false, now, now), + cancellationToken); + } + catch (IdentityConflictException) + { + return Conflict(); + } + return Ok(AuthController.ToProfileResponse(profile)); + } + + [HttpPatch("profiles/{id:guid}")] + [Authorize(Policy = AccessPolicies.ContentWrite)] + public async Task UpdateProfile( + [FromRoute] Guid id, + [FromBody] External.UpdateProfileRequest request, + CancellationToken cancellationToken) + { + if (!User.TryGetUserId(out var userId)) return Unauthorized(); + if (!User.TryGetProfileId(out var activeProfileId)) return Unauthorized(); + var target = await identityRepository.FindProfileAsync(id, cancellationToken); + if (target is null || target.UserId != userId) return NotFound(); + + var needsStepUp = id != activeProfileId || request.ReplacePin; + if (needsStepUp) + { + var pinVerified = target.PinHash is not null + && VerifyPin(request.CurrentPin, target.PinHash); + var recentlyAuthenticated = (await authorizationService.AuthorizeAsync( + User, resource: null, AccessPolicies.RecentAuthentication)).Succeeded; + if (!pinVerified && !recentlyAuthenticated) return Forbid(); + } + + if (!TryNormalizeProfile( + request.Name, + request.Avatar, + request.ReplacePin ? request.Pin : null, + out var name, + out var avatar, + out var pinHash)) + return BadRequest(); + bool updated; + try + { + updated = await identityRepository.UpdateProfileAsync( + id, + userId, + name, + avatar, + pinHash, + request.ReplacePin, + DateTimeOffset.UtcNow, + cancellationToken); + } + catch (IdentityConflictException) + { + return Conflict(); + } + return updated ? NoContent() : NotFound(); + } + + [HttpPost("profiles/switch")] + public async Task SwitchProfile( + [FromBody] External.SwitchProfileRequest request, + CancellationToken cancellationToken) + { + if (!User.TryGetUserId(out var userId) + || !User.TryGetSessionId(out var sessionId)) + return Unauthorized(); + var profile = await identityRepository.FindProfileAsync( + request.ProfileId, cancellationToken); + if (profile is null || profile.UserId != userId) + return NotFound(); + if (profile.PinHash is not null && !VerifyPin(request.Pin, profile.PinHash)) + return Unauthorized(); + + var authenticated = await identityRepository.GetAuthenticatedSessionAsync( + sessionId, DateTimeOffset.UtcNow, cancellationToken); + if (authenticated is null || authenticated.User.Id != userId) + return Unauthorized(); + var rotated = await tokenIssuer.RotateSessionAsync( + authenticated, + profile, + request.RefreshToken, + reauthenticated: false, + cancellationToken); + return rotated is null + ? Unauthorized() + : Ok(new External.LoginResult( + rotated.AccessToken, + rotated.RefreshToken, + true, + rotated.SessionId, + rotated.ProfileId)); + } + + [HttpGet("sessions")] + public async Task GetOwnSessions(CancellationToken cancellationToken) + { + if (!User.TryGetUserId(out var userId)) return Unauthorized(); + User.TryGetSessionId(out var currentSessionId); + var sessions = await identityRepository.GetSessionsAsync(userId, cancellationToken); + return Ok(sessions.Select(session => ToResponse(session, currentSessionId)).ToList()); + } + + [HttpDelete("sessions/{id:guid}")] + public async Task RevokeOwnSession( + [FromRoute] Guid id, + CancellationToken cancellationToken) + { + if (!User.TryGetUserId(out var userId)) return Unauthorized(); + var revoked = await identityRepository.RevokeSessionAsync( + id, userId, DateTimeOffset.UtcNow, cancellationToken); + return revoked ? NoContent() : NotFound(); + } + + [HttpGet("users")] + [Authorize(Policy = AccessPolicies.Administrator)] + public async Task GetUsers(CancellationToken cancellationToken) + { + var users = await identityRepository.GetUsersAsync(cancellationToken); + return Ok(users.Select(ToResponse).ToList()); + } + + [HttpPost("users")] + [Authorize(Policy = AccessPolicies.RecentAdministrator)] + public async Task CreateUser( + [FromBody] External.CreateUserRequest request, + CancellationToken cancellationToken) + { + var username = request.Username.Trim().ToLowerInvariant(); + if (!UsernamePattern().IsMatch(username) + || string.IsNullOrEmpty(request.Password) + || !TryParseRole(request.Role, out var role) + || !TryNormalizeProfile(request.ProfileName, null, null, + out var profileName, out _, out _)) + return BadRequest(); + if (await identityRepository.FindUserByUsernameAsync(username, cancellationToken) is not null) + return Conflict(); + + var now = DateTimeOffset.UtcNow; + var user = new UserAccount( + Guid.NewGuid(), + username, + BCrypt.Net.BCrypt.HashPassword(request.Password), + role, + false, + now, + now); + var profile = new UserProfile( + Guid.NewGuid(), + user.Id, + profileName, + null, + null, + true, + now, + now); + try + { + return Ok(ToResponse(await identityRepository.CreateUserWithProfileAsync( + user, profile, cancellationToken))); + } + catch (IdentityConflictException) + { + return Conflict(); + } + } + + [HttpPatch("users/{id:guid}")] + [Authorize(Policy = AccessPolicies.RecentAdministrator)] + public async Task UpdateUserAccess( + [FromRoute] Guid id, + [FromBody] External.UpdateUserAccessRequest request, + CancellationToken cancellationToken) + { + if (!TryParseRole(request.Role, out var role)) return BadRequest(); + var result = await identityRepository.UpdateUserAccessAsync( + id, role, request.IsDisabled, DateTimeOffset.UtcNow, cancellationToken); + return result switch + { + UpdateUserAccessResult.Updated => NoContent(), + UpdateUserAccessResult.NotFound => NotFound(), + UpdateUserAccessResult.LastAdministrator => Conflict(new + { + message = "At least one enabled administrator is required." + }), + _ => throw new ArgumentOutOfRangeException(nameof(result), result, null) + }; + } + + [HttpGet("sessions/all")] + [Authorize(Policy = AccessPolicies.Administrator)] + public async Task GetAllSessions(CancellationToken cancellationToken) + { + User.TryGetSessionId(out var currentSessionId); + var sessions = await identityRepository.GetSessionsAsync(null, cancellationToken); + return Ok(sessions.Select(session => ToResponse(session, currentSessionId)).ToList()); + } + + [HttpDelete("sessions/{id:guid}/admin")] + [Authorize(Policy = AccessPolicies.RecentAdministrator)] + public async Task RevokeAnySession( + [FromRoute] Guid id, + CancellationToken cancellationToken) + { + var revoked = await identityRepository.RevokeSessionAsync( + id, null, DateTimeOffset.UtcNow, cancellationToken); + return revoked ? NoContent() : NotFound(); + } + + private static bool TryNormalizeProfile( + string rawName, + string? rawAvatar, + string? rawPin, + out string name, + out string? avatar, + out string? pinHash) + { + name = rawName.Trim(); + avatar = string.IsNullOrWhiteSpace(rawAvatar) ? null : rawAvatar.Trim(); + pinHash = null; + if (name.Length is < 1 or > 64 || avatar?.Length > 512) + return false; + if (rawPin is null) return true; + if (rawPin.Length == 0) return true; + if (!PinPattern().IsMatch(rawPin)) return false; + pinHash = BCrypt.Net.BCrypt.HashPassword(rawPin); + return true; + } + + private static bool VerifyPin(string? pin, string hash) + { + if (pin is null) return false; + try + { + return BCrypt.Net.BCrypt.Verify(pin, hash); + } + catch (BCrypt.Net.SaltParseException) + { + return false; + } + } + + private static bool TryParseRole(string raw, out UserRole role) => + Enum.TryParse(raw, ignoreCase: true, out role) + && Enum.IsDefined(role); + + private static External.UserResponse ToResponse(UserAccountWithProfiles item) => + new(item.User.Id, + item.User.Username, + item.User.Role.ToString(), + item.User.IsDisabled, + item.User.CreatedAt, + item.Profiles.Select(AuthController.ToProfileResponse).ToList()); + + private static External.SessionResponse ToResponse( + UserSessionSummary item, + Guid currentSessionId) => + new(item.Session.Id, + item.Session.UserId, + item.Username, + item.Session.ActiveProfileId, + item.ProfileName, + item.Session.DeviceName, + item.Session.AuthenticatedAt, + item.Session.CreatedAt, + item.Session.LastSeenAt, + item.Session.ExpiresAt, + item.Session.RevokedAt, + item.Session.Id == currentSessionId); +} diff --git a/SecondDimensionWatcherReDive/Controllers/AnimationInfoController.cs b/SecondDimensionWatcherReDive/Controllers/AnimationInfoController.cs index 8e3ee5b..53e3ec6 100644 --- a/SecondDimensionWatcherReDive/Controllers/AnimationInfoController.cs +++ b/SecondDimensionWatcherReDive/Controllers/AnimationInfoController.cs @@ -2,6 +2,7 @@ using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; +using SecondDimensionWatcherReDive.Framework.Authorization; using Microsoft.Extensions.Caching.Distributed; using SecondDimensionWatcherReDive.Framework.DataRepository; using SecondDimensionWatcherReDive.Framework.FileDownload; @@ -19,6 +20,7 @@ internal class AnimationInfoController( IDistributedCache distributedCache, IFileDownloadClientProvider fileDownloadClientProvider, IFileMapper fileMapper, + IAuthorizationService authorizationService, IIncidentReporter? incidentReporter = null) : ControllerBase { @@ -68,6 +70,7 @@ public async Task GetDownloadStatus([FromRoute] Guid id, Cancella } [HttpPost("download/{id:guid}")] + [Authorize(Policy = AccessPolicies.ContentWrite)] public async Task StartDownload([FromRoute] Guid id, CancellationToken cancellationToken) { var info = await animationInfoRepository.FindByIdAsync(id, cancellationToken); @@ -129,6 +132,7 @@ await CompensateFailedStartAsync( } [HttpPost("pause/{id:guid}")] + [Authorize(Policy = AccessPolicies.ContentWrite)] public async Task PauseDownload([FromRoute] Guid id, CancellationToken cancellationToken) { var info = await animationInfoRepository.FindByIdAsync(id, cancellationToken); @@ -152,6 +156,7 @@ public async Task PauseDownload([FromRoute] Guid id, Cancellation } [HttpPost("resume/{id:guid}")] + [Authorize(Policy = AccessPolicies.ContentWrite)] public async Task ResumeDownload([FromRoute] Guid id, CancellationToken cancellationToken) { var info = await animationInfoRepository.FindByIdAsync(id, cancellationToken); @@ -175,9 +180,14 @@ public async Task ResumeDownload([FromRoute] Guid id, Cancellatio } [HttpDelete("cancel/{id:guid}")] + [Authorize(Policy = AccessPolicies.ContentWrite)] public async Task CancelDownload([FromRoute] Guid id, [FromQuery] bool removeFile = false, CancellationToken cancellationToken = default) { + if (removeFile && !(await authorizationService.AuthorizeAsync( + User, resource: null, AccessPolicies.RecentAdministrator)).Succeeded) + return Forbid(); + var info = await animationInfoRepository.FindByIdAsync(id, cancellationToken); if (info is null) @@ -298,6 +308,7 @@ private static CancellationTokenSource CreateDownloadSagaTokenSource() => new(TimeSpan.FromSeconds(10)); [HttpPost("{id:guid}/retry-inference")] + [Authorize(Policy = AccessPolicies.Administrator)] public async Task RetryInference([FromRoute] Guid id, CancellationToken cancellationToken) { var info = await animationInfoRepository.FindByIdAsync(id, cancellationToken); @@ -319,6 +330,7 @@ public async Task RetryInference([FromRoute] Guid id, Cancellatio } [HttpPost("{id:guid}/reidentify-files/ai")] + [Authorize(Policy = AccessPolicies.Administrator)] public async Task ReidentifyFilesWithAi( [FromRoute] Guid id, CancellationToken cancellationToken) diff --git a/SecondDimensionWatcherReDive/Controllers/AuthController.cs b/SecondDimensionWatcherReDive/Controllers/AuthController.cs index 217b4f4..f4108e4 100644 --- a/SecondDimensionWatcherReDive/Controllers/AuthController.cs +++ b/SecondDimensionWatcherReDive/Controllers/AuthController.cs @@ -1,151 +1,307 @@ using System.IdentityModel.Tokens.Jwt; using System.Security.Claims; -using System.Security.Cryptography; -using System.Text; -using System.Text.Json; +using System.Text.RegularExpressions; using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Caching.Distributed; using Microsoft.IdentityModel.Tokens; +using SecondDimensionWatcherReDive.Auth; +using SecondDimensionWatcherReDive.Framework.Authorization; +using SecondDimensionWatcherReDive.Framework.DataRepository; namespace SecondDimensionWatcherReDive.Controllers; [ApiController] [Route("api/[controller]")] -internal partial class AuthController : ControllerBase +internal partial class AuthController( + IConfiguration configuration, + TokenValidationParameters tokenValidationParams, + IIdentityRepository identityRepository, + SessionTokenIssuer tokenIssuer, + ILogger logger) : ControllerBase { - private readonly IConfiguration _configuration; - private readonly ILogger _logger; - private readonly IDistributedCache _distributedCache; + [GeneratedRegex("^[a-z0-9._-]{3,64}$")] + private static partial Regex UsernamePattern(); - private readonly TokenValidationParameters _tokenValidationParams; - - public AuthController(IConfiguration configuration, TokenValidationParameters tokenValidationParams, - IDistributedCache distributedCache, ILogger logger) + [HttpPost("register")] + public async Task Register( + [FromBody] External.LoginData data, + CancellationToken cancellationToken) { - _configuration = configuration; - _tokenValidationParams = tokenValidationParams; - _distributedCache = distributedCache; - _logger = logger; - } + if (await identityRepository.AnyUsersAsync(cancellationToken) + || HasLegacyPassword()) + return Conflict(); + if (!TryNormalizeUsername(data.Username, out var username) + || string.IsNullOrEmpty(data.Password)) + return BadRequest(); - private static string RandomString(int length) - { - const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; - return RandomNumberGenerator.GetString(chars, length); + var now = DateTimeOffset.UtcNow; + var user = new UserAccount( + IdentityDefaults.UserId, + username, + BCrypt.Net.BCrypt.HashPassword(data.Password), + UserRole.Admin, + false, + now, + now); + var profile = new UserProfile( + IdentityDefaults.ProfileId, + user.Id, + NormalizeProfileName(data.ProfileName), + null, + null, + true, + now, + now); + try + { + await identityRepository.CreateUserWithProfileAsync(user, profile, cancellationToken); + } + catch (IdentityConflictException) + { + return Conflict(); + } + return Ok(ToResult(await tokenIssuer.CreateSessionAsync( + user, profile, data.DeviceName, cancellationToken))); } - private async Task GenerateJwtTokenAsync() + [HttpPost("login")] + public async Task Login( + [FromBody] External.LoginData data, + CancellationToken cancellationToken) { - var handler = new JwtSecurityTokenHandler(); - var key = Encoding.ASCII.GetBytes(_configuration["JwtSecret"]!); + if (!TryNormalizeUsername(data.Username, out var username)) + return Unauthorized(); - var tokenDescriptor = new SecurityTokenDescriptor - { - Subject = new ClaimsIdentity(new[] - { - new Claim("Id", Guid.Empty.ToString()), - new Claim(JwtRegisteredClaimNames.Jti, Guid.NewGuid().ToString()) - }), - Expires = DateTime.UtcNow.AddMinutes(10), - SigningCredentials = - new SigningCredentials(new SymmetricSecurityKey(key), SecurityAlgorithms.HmacSha256Signature) - }; + var user = await identityRepository.FindUserByUsernameAsync(username, cancellationToken); + if (user is null + && string.Equals(username, IdentityDefaults.Username, StringComparison.Ordinal) + && VerifyLegacyPassword(data.Password)) + user = await CreateLegacyAdminAsync(data.Password, cancellationToken); + if (user is null || user.IsDisabled || !await VerifyPasswordAsync( + user, data.Password, cancellationToken)) + return Unauthorized(); + + var profiles = await identityRepository.GetProfilesAsync(user.Id, cancellationToken); + var profile = profiles.FirstOrDefault(candidate => candidate.IsDefault) + ?? profiles.FirstOrDefault(); + if (profile is null) return Unauthorized(); - var token = handler.CreateToken(tokenDescriptor); - var jwtToken = handler.WriteToken(token); + return Ok(ToResult(await tokenIssuer.CreateSessionAsync( + user, profile, data.DeviceName, cancellationToken))); + } - var refreshToken = new External.RefreshToken(RandomString(25) + Guid.NewGuid(), token.Id); + [HttpPost("refresh")] + public async Task Refresh( + [FromBody] External.AuthRequest request, + CancellationToken cancellationToken) + { + var principal = ValidateExpiredAccessToken(request.Token); + if (principal is null + || !principal.TryGetUserId(out var userId) + || !principal.TryGetSessionId(out var sessionId)) + return Unauthorized(new External.LoginResult(null, null, false)); - await _distributedCache.SetStringAsync(refreshToken.Token, - JsonSerializer.Serialize(refreshToken, External.AppJsonSerializerContext.Default.RefreshToken)); + var authenticated = await identityRepository.GetAuthenticatedSessionAsync( + sessionId, DateTimeOffset.UtcNow, cancellationToken); + if (authenticated is null || authenticated.User.Id != userId) + return Unauthorized(new External.LoginResult(null, null, false)); - return new External.LoginResult(jwtToken, refreshToken.Token); + var rotated = await tokenIssuer.RotateSessionAsync( + authenticated, + authenticated.Profile, + request.RefreshToken, + reauthenticated: false, + cancellationToken); + return rotated is null + ? Unauthorized(new External.LoginResult(null, null, false)) + : Ok(ToResult(rotated)); } - [HttpPost("register")] - public async Task Register([FromBody] External.LoginData data) + [Authorize(AuthenticationSchemes = JwtBearerDefaults.AuthenticationScheme)] + [HttpPost("reauthenticate")] + public async Task Reauthenticate( + [FromBody] External.ReauthenticateRequest request, + CancellationToken cancellationToken) { - if (!string.IsNullOrWhiteSpace(_configuration["Password:Value"])) - return BadRequest(); + var authenticated = await GetCurrentSessionAsync(cancellationToken); + if (authenticated is null + || !await VerifyPasswordAsync(authenticated.User, request.Password, cancellationToken)) + return Unauthorized(); - var passwordFile = _configuration["PasswordFile"] ?? "password.json"; - await System.IO.File.WriteAllBytesAsync(passwordFile, - JsonSerializer.SerializeToUtf8Bytes( - new External.PasswordConfig(new External.PasswordHash(BCrypt.Net.BCrypt.HashPassword(data.Password))), - External.AppJsonSerializerContext.Default.PasswordConfig)); + var rotated = await tokenIssuer.RotateSessionAsync( + authenticated, + authenticated.Profile, + request.RefreshToken, + reauthenticated: true, + cancellationToken); + return rotated is null ? Unauthorized() : Ok(ToResult(rotated)); + } - return Ok(await GenerateJwtTokenAsync()); + [Authorize(AuthenticationSchemes = JwtBearerDefaults.AuthenticationScheme)] + [HttpPost("logout")] + public async Task Logout(CancellationToken cancellationToken) + { + if (!User.TryGetSessionId(out var sessionId)) return Unauthorized(); + await identityRepository.RevokeSessionAsync( + sessionId, + requiredUserId: null, + DateTimeOffset.UtcNow, + cancellationToken); + return NoContent(); } - [HttpPost("login")] - public async Task Login([FromBody] External.LoginData data) + [Authorize(AuthenticationSchemes = JwtBearerDefaults.AuthenticationScheme)] + [HttpGet("verify")] + public async Task Verify(CancellationToken cancellationToken) { - var storedValue = _configuration["Password:Value"]; - if (string.IsNullOrWhiteSpace(storedValue)) - return BadRequest(); + var authenticated = await GetCurrentSessionAsync(cancellationToken); + if (authenticated is null) return Unauthorized(); + var profiles = await identityRepository.GetProfilesAsync( + authenticated.User.Id, cancellationToken); + return Ok(new External.AuthStateResponse( + authenticated.User.Id, + authenticated.User.Username, + authenticated.User.Role.ToString(), + authenticated.Session.Id, + authenticated.Profile.Id, + profiles.Select(ToProfileResponse).ToList())); + } - if (!BCrypt.Net.BCrypt.Verify(data.Password, storedValue)) - return BadRequest(); + [HttpGet("allowRegister")] + public async Task CanRegister(CancellationToken cancellationToken) => + Ok(new + { + Allow = !HasLegacyPassword() + && !await identityRepository.AnyUsersAsync(cancellationToken) + }); - return Ok(await GenerateJwtTokenAsync()); + private async Task GetCurrentSessionAsync( + CancellationToken cancellationToken) + { + if (!User.TryGetSessionId(out var sessionId)) return null; + return await identityRepository.GetAuthenticatedSessionAsync( + sessionId, DateTimeOffset.UtcNow, cancellationToken); } - [HttpPost("refresh")] - public async Task Refresh([FromBody] External.AuthRequest request) + private ClaimsPrincipal? ValidateExpiredAccessToken(string token) { - var result = await VerifyAndGenerateTokenAsync(request); - return result.Success ? Ok(result) : BadRequest(result); + try + { + var parameters = tokenValidationParams.Clone(); + parameters.ValidateLifetime = false; + var principal = new JwtSecurityTokenHandler().ValidateToken( + token, parameters, out var validatedToken); + return validatedToken is JwtSecurityToken securityToken + && string.Equals( + securityToken.Header.Alg, + SecurityAlgorithms.HmacSha256, + StringComparison.OrdinalIgnoreCase) + ? principal + : null; + } + catch (Exception exception) + { + LogTokenVerificationFailed(logger, exception); + return null; + } } - private async Task VerifyAndGenerateTokenAsync(External.AuthRequest request) + private async Task CreateLegacyAdminAsync( + string password, + CancellationToken cancellationToken) { + var now = DateTimeOffset.UtcNow; + var user = new UserAccount( + IdentityDefaults.UserId, + IdentityDefaults.Username, + BCrypt.Net.BCrypt.HashPassword(password), + UserRole.Admin, + false, + now, + now); + var profile = new UserProfile( + IdentityDefaults.ProfileId, + user.Id, + IdentityDefaults.ProfileName, + null, + null, + true, + now, + now); try { - var handler = new JwtSecurityTokenHandler(); - var param = _tokenValidationParams.Clone(); - param.ValidateLifetime = false; - var tokenInVerification = - handler.ValidateToken(request.Token, param, out var validatedToken); - + await identityRepository.CreateUserWithProfileAsync(user, profile, cancellationToken); + return user; + } + catch (IdentityConflictException) + { + var existing = await identityRepository.FindUserByUsernameAsync( + IdentityDefaults.Username, cancellationToken); + if (existing is null) throw; + return existing; + } + } - if (validatedToken is JwtSecurityToken securityToken && !securityToken.Header.Alg.Equals( - SecurityAlgorithms.HmacSha256, - StringComparison.InvariantCultureIgnoreCase)) - return new External.LoginResult(null, null, false); + private async Task VerifyPasswordAsync( + UserAccount user, + string password, + CancellationToken cancellationToken) + { + if (user.PasswordHash is not null) + return VerifyHash(password, user.PasswordHash); + if (user.Id != IdentityDefaults.UserId || !VerifyLegacyPassword(password)) + return false; - var storedJson = await _distributedCache.GetStringAsync(request.RefreshToken); - var storedToken = storedJson is null ? null : JsonSerializer.Deserialize(storedJson, External.AppJsonSerializerContext.Default.RefreshToken); - if (storedToken is null) return new External.LoginResult(null, null, false); + return await identityRepository.SetPasswordHashAsync( + user.Id, + BCrypt.Net.BCrypt.HashPassword(password), + DateTimeOffset.UtcNow, + cancellationToken); + } - if (tokenInVerification.FindFirst(c => c.Type == JwtRegisteredClaimNames.Jti)?.Value != storedToken.JwtId) - return new External.LoginResult(null, null, false); + private bool VerifyLegacyPassword(string password) + { + var value = configuration["Password:Value"]; + return !string.IsNullOrWhiteSpace(value) && VerifyHash(password, value); + } - await _distributedCache.RemoveAsync(request.RefreshToken); + private bool HasLegacyPassword() => + !string.IsNullOrWhiteSpace(configuration["Password:Value"]); - return await GenerateJwtTokenAsync(); + private static bool VerifyHash(string password, string hash) + { + try + { + return BCrypt.Net.BCrypt.Verify(password, hash); } - catch (Exception exception) + catch (BCrypt.Net.SaltParseException) { - LogTokenVerificationFailed(_logger, exception); - return new External.LoginResult(null, null, false); + return false; } } - [HttpGet("verify")] - [Authorize(AuthenticationSchemes = JwtBearerDefaults.AuthenticationScheme)] - public IActionResult Verify() + private static bool TryNormalizeUsername(string? value, out string username) { - return Ok(HttpContext.User.Claims.Select(c => new { c.Type, c.Value })); + username = string.IsNullOrWhiteSpace(value) + ? IdentityDefaults.Username + : value.Trim().ToLowerInvariant(); + return UsernamePattern().IsMatch(username); } - [HttpGet("allowRegister")] - public IActionResult CanRegister() + private static string NormalizeProfileName(string? value) { - return Ok(new { Allow = string.IsNullOrWhiteSpace(_configuration["Password:Value"]) }); + var name = value?.Trim(); + if (string.IsNullOrEmpty(name)) return IdentityDefaults.ProfileName; + return name.Length <= 64 ? name : name[..64]; } - [LoggerMessage(Level = LogLevel.Error, Message = "Token verification failed")] - private static partial void LogTokenVerificationFailed(ILogger logger, Exception ex); + private static External.LoginResult ToResult(IssuedSessionTokens tokens) => + new(tokens.AccessToken, tokens.RefreshToken, true, tokens.SessionId, tokens.ProfileId); + + internal static External.AuthProfileResponse ToProfileResponse(UserProfile profile) => + new(profile.Id, profile.Name, profile.Avatar, profile.PinHash is not null, profile.IsDefault); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Token verification failed")] + private static partial void LogTokenVerificationFailed(ILogger logger, Exception exception); } diff --git a/SecondDimensionWatcherReDive/Controllers/Converter.cs b/SecondDimensionWatcherReDive/Controllers/Converter.cs index 742d740..a011b05 100644 --- a/SecondDimensionWatcherReDive/Controllers/Converter.cs +++ b/SecondDimensionWatcherReDive/Controllers/Converter.cs @@ -83,7 +83,15 @@ public static External.SubscriptionAutomationSimulationResult ToExternal( explanation.Message)).ToList())).ToList()); public static External.WebDavTokenSummary ToExternal(this WebDavToken record) => - new(record.Id, record.Username, record.Description, record.CreatedAt); + new(record.Id, + record.UserId, + record.Username, + record.Description, + record.CreatedAt, + record.Scope, + record.VirtualRoot, + record.ExpiresAt, + record.RevokedAt); public static External.SeasonBangumi ToExternal(this SeasonBangumi record) => new(record.Id, diff --git a/SecondDimensionWatcherReDive/Controllers/External/Accounts.cs b/SecondDimensionWatcherReDive/Controllers/External/Accounts.cs new file mode 100644 index 0000000..af01689 --- /dev/null +++ b/SecondDimensionWatcherReDive/Controllers/External/Accounts.cs @@ -0,0 +1,52 @@ +using System.ComponentModel.DataAnnotations; + +namespace SecondDimensionWatcherReDive.Controllers.External; + +internal sealed record CreateProfileRequest( + [Required] string Name, + string? Avatar, + string? Pin); + +internal sealed record UpdateProfileRequest( + [Required] string Name, + string? Avatar, + string? Pin, + string? CurrentPin = null, + bool ReplacePin = false); + +internal sealed record SwitchProfileRequest( + Guid ProfileId, + string? Pin, + [Required] string RefreshToken); + +internal sealed record SessionResponse( + Guid Id, + Guid UserId, + string Username, + Guid ProfileId, + string ProfileName, + string? DeviceName, + DateTimeOffset AuthenticatedAt, + DateTimeOffset CreatedAt, + DateTimeOffset LastSeenAt, + DateTimeOffset ExpiresAt, + DateTimeOffset? RevokedAt, + bool IsCurrent); + +internal sealed record UserResponse( + Guid Id, + string Username, + string Role, + bool IsDisabled, + DateTimeOffset CreatedAt, + IReadOnlyList Profiles); + +internal sealed record CreateUserRequest( + [Required] string Username, + [Required] string Password, + [Required] string Role, + [Required] string ProfileName); + +internal sealed record UpdateUserAccessRequest( + [Required] string Role, + bool IsDisabled); diff --git a/SecondDimensionWatcherReDive/Controllers/External/AppJsonSerializerContext.cs b/SecondDimensionWatcherReDive/Controllers/External/AppJsonSerializerContext.cs index 0f20309..f1b042b 100644 --- a/SecondDimensionWatcherReDive/Controllers/External/AppJsonSerializerContext.cs +++ b/SecondDimensionWatcherReDive/Controllers/External/AppJsonSerializerContext.cs @@ -21,7 +21,18 @@ namespace SecondDimensionWatcherReDive.Controllers.External; [JsonSerializable(typeof(IEnumerable))] [JsonSerializable(typeof(FileStoreListResult[]))] [JsonSerializable(typeof(FileStoreToken))] -[JsonSerializable(typeof(RefreshToken))] +[JsonSerializable(typeof(ReauthenticateRequest))] +[JsonSerializable(typeof(AuthProfileResponse))] +[JsonSerializable(typeof(AuthStateResponse))] +[JsonSerializable(typeof(CreateProfileRequest))] +[JsonSerializable(typeof(UpdateProfileRequest))] +[JsonSerializable(typeof(SwitchProfileRequest))] +[JsonSerializable(typeof(SessionResponse))] +[JsonSerializable(typeof(List))] +[JsonSerializable(typeof(UserResponse))] +[JsonSerializable(typeof(List))] +[JsonSerializable(typeof(CreateUserRequest))] +[JsonSerializable(typeof(UpdateUserAccessRequest))] [JsonSerializable(typeof(AddFeedRequest))] [JsonSerializable(typeof(UpsertSubscriptionAutomationPolicyRequest))] [JsonSerializable(typeof(SubscriptionAutomationPolicy))] diff --git a/SecondDimensionWatcherReDive/Controllers/External/Auth.cs b/SecondDimensionWatcherReDive/Controllers/External/Auth.cs index c72c585..d6b4e7c 100644 --- a/SecondDimensionWatcherReDive/Controllers/External/Auth.cs +++ b/SecondDimensionWatcherReDive/Controllers/External/Auth.cs @@ -2,14 +2,40 @@ namespace SecondDimensionWatcherReDive.Controllers.External; -internal sealed record LoginData([Required] string Password); +internal sealed record LoginData( + [Required] string Password, + string? Username = null, + string? DeviceName = null, + string? ProfileName = null); -internal sealed record LoginResult(string? Token, string? RefreshToken, bool Success = true); +internal sealed record LoginResult( + string? Token, + string? RefreshToken, + bool Success = true, + Guid? SessionId = null, + Guid? ProfileId = null); internal sealed record AuthRequest([Required] string Token, [Required] string RefreshToken); -internal sealed record RefreshToken(string Token, string JwtId); +internal sealed record ReauthenticateRequest( + [Required] string Password, + [Required] string RefreshToken); internal sealed record PasswordConfig(PasswordHash Password); internal sealed record PasswordHash(string Value); + +internal sealed record AuthProfileResponse( + Guid Id, + string Name, + string? Avatar, + bool HasPin, + bool IsDefault); + +internal sealed record AuthStateResponse( + Guid UserId, + string Username, + string Role, + Guid SessionId, + Guid ProfileId, + IReadOnlyList Profiles); diff --git a/SecondDimensionWatcherReDive/Controllers/External/File.cs b/SecondDimensionWatcherReDive/Controllers/External/File.cs index e9a0e14..40fbd0a 100644 --- a/SecondDimensionWatcherReDive/Controllers/External/File.cs +++ b/SecondDimensionWatcherReDive/Controllers/External/File.cs @@ -6,6 +6,12 @@ internal sealed record FileLinkResultResponse(string Url); internal sealed record FileLinkResultRequest([Required] Guid Id, string Path); -internal sealed record FileStoreToken(string Path, string FileStore); +internal sealed record FileStoreToken( + string Path, + string FileStore, + Guid SessionId, + Guid UserId, + Guid ProfileId, + string VirtualRoot); internal sealed record FileStoreListResult(string FileName, bool IsDirectory, string? Relative); diff --git a/SecondDimensionWatcherReDive/Controllers/External/WebDavToken.cs b/SecondDimensionWatcherReDive/Controllers/External/WebDavToken.cs index c2fb900..5282c6e 100644 --- a/SecondDimensionWatcherReDive/Controllers/External/WebDavToken.cs +++ b/SecondDimensionWatcherReDive/Controllers/External/WebDavToken.cs @@ -2,15 +2,29 @@ namespace SecondDimensionWatcherReDive.Controllers.External; internal sealed record WebDavTokenSummary( Guid Id, + Guid UserId, string Username, string? Description, - DateTimeOffset CreatedAt); + DateTimeOffset CreatedAt, + string Scope, + string VirtualRoot, + DateTimeOffset? ExpiresAt, + DateTimeOffset? RevokedAt); -internal sealed record CreateWebDavTokenRequest(string? Username, string? Description); +internal sealed record CreateWebDavTokenRequest( + string? Username, + string? Description, + Guid? UserId = null, + string? VirtualRoot = null, + DateTimeOffset? ExpiresAt = null); internal sealed record CreateWebDavTokenResponse( Guid Id, string Username, string Token, string? Description, - DateTimeOffset CreatedAt); + DateTimeOffset CreatedAt, + Guid UserId, + string Scope, + string VirtualRoot, + DateTimeOffset ExpiresAt); diff --git a/SecondDimensionWatcherReDive/Controllers/FeedController.cs b/SecondDimensionWatcherReDive/Controllers/FeedController.cs index e651964..8b95aee 100644 --- a/SecondDimensionWatcherReDive/Controllers/FeedController.cs +++ b/SecondDimensionWatcherReDive/Controllers/FeedController.cs @@ -1,6 +1,7 @@ using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; +using SecondDimensionWatcherReDive.Framework.Authorization; using SecondDimensionWatcherReDive.Framework.DataRepository; namespace SecondDimensionWatcherReDive.Controllers; @@ -18,6 +19,7 @@ public async Task GetFeeds(CancellationToken cancellationToken) } [HttpPost] + [Authorize(Policy = AccessPolicies.ContentWrite)] public async Task AddFeed([FromBody] External.AddFeedRequest request, CancellationToken cancellationToken) { @@ -28,6 +30,7 @@ public async Task AddFeed([FromBody] External.AddFeedRequest requ } [HttpDelete("{id:guid}")] + [Authorize(Policy = AccessPolicies.ContentWrite)] public async Task RemoveFeed([FromRoute] Guid id, CancellationToken cancellationToken) { var feed = await feedRepository.FindByIdAsync(id, cancellationToken); diff --git a/SecondDimensionWatcherReDive/Controllers/FileController.cs b/SecondDimensionWatcherReDive/Controllers/FileController.cs index 0d37050..a1dbc1a 100644 --- a/SecondDimensionWatcherReDive/Controllers/FileController.cs +++ b/SecondDimensionWatcherReDive/Controllers/FileController.cs @@ -6,6 +6,8 @@ using Microsoft.AspNetCore.StaticFiles; using System.Text.Json; using Microsoft.Extensions.Caching.Distributed; +using SecondDimensionWatcherReDive.Auth; +using SecondDimensionWatcherReDive.Framework.Authorization; using SecondDimensionWatcherReDive.Framework.DataRepository; using SecondDimensionWatcherReDive.Framework.FileStore; @@ -17,6 +19,7 @@ namespace SecondDimensionWatcherReDive.Controllers; internal partial class FileController( IAnimationInfoRepository animationInfoRepository, IFileExplorer fileExplorer, + IIdentityRepository identityRepository, IDistributedCache distributedCache, IContentTypeProvider contentTypeProvider, ILogger logger) : ControllerBase @@ -32,6 +35,10 @@ private static string GenerateToken(int length) public async Task GetFileLink([FromBody] External.FileLinkResultRequest payload, CancellationToken cancellationToken) { + if (!User.TryGetUserId(out var userId) + || !User.TryGetProfileId(out var profileId) + || !User.TryGetSessionId(out var sessionId)) + return Unauthorized(); LogGenerateLinkRequest(logger, payload.Id, payload.Path); var info = await animationInfoRepository.FindByIdWithAnimationAsync(payload.Id, cancellationToken); @@ -41,12 +48,18 @@ public async Task GetFileLink([FromBody] External.FileLinkResultR return NotFound(); } - var virtualPath = ResolveVirtualPath(info, payload.Path); + if (!TryResolveVirtualPath(info, payload.Path, out var virtualPath)) + return BadRequest(); + var virtualRoot = DevicePathScope.GetVirtualRoot(User); + if (!DevicePathScope.TryMapInternalToPublic( + virtualPath, virtualRoot, out _)) + return Forbid(); LogResolvedTargetPath(logger, virtualPath, "virtual path"); var token = GenerateToken(64); await distributedCache.SetStringAsync(token, - JsonSerializer.Serialize(new External.FileStoreToken(virtualPath, string.Empty), + JsonSerializer.Serialize(new External.FileStoreToken( + virtualPath, string.Empty, sessionId, userId, profileId, virtualRoot), External.AppJsonSerializerContext.Default.FileStoreToken), new DistributedCacheEntryOptions { AbsoluteExpirationRelativeToNow = TimeSpan.FromDays(1) }, cancellationToken); @@ -68,6 +81,18 @@ public async Task GetFile([FromQuery] [Required] string token, return NotFound(); } + var authenticated = await identityRepository.GetAuthenticatedSessionAsync( + fileStoreToken.SessionId, DateTimeOffset.UtcNow, cancellationToken); + if (authenticated is null + || authenticated.User.Id != fileStoreToken.UserId + || authenticated.Profile.Id != fileStoreToken.ProfileId + || !DevicePathScope.TryMapInternalToPublic( + fileStoreToken.Path, fileStoreToken.VirtualRoot, out _)) + { + LogPlayTokenInvalid(logger); + return NotFound(); + } + var fileName = Path.GetFileName(fileStoreToken.Path); var contentType = contentTypeProvider.TryGetContentType(fileName, out var type) ? type @@ -93,7 +118,11 @@ public async Task GetSubDir([FromQuery] [Required] Guid id, return NotFound(); } - var virtualPath = ResolveVirtualPath(info, relativeDir); + if (!TryResolveVirtualPath(info, relativeDir, out var virtualPath)) + return BadRequest(); + if (!DevicePathScope.TryMapInternalToPublic( + virtualPath, DevicePathScope.GetVirtualRoot(User), out _)) + return Forbid(); LogListPathInfo(logger, virtualPath, true); var tokens = await fileExplorer.EnumerateDirectoryAsync( @@ -109,12 +138,30 @@ public async Task GetSubDir([FromQuery] [Required] Guid id, return Ok(results); } - private static string ResolveVirtualPath(AnimationInfo info, string? relative) + private static bool TryResolveVirtualPath( + AnimationInfo info, + string? relative, + out string virtualPath) { var root = GetAnimationVirtualRoot(info); - if (string.IsNullOrWhiteSpace(relative)) return root; + if (string.IsNullOrWhiteSpace(relative)) + { + virtualPath = root; + return true; + } + var trimmed = relative.Trim('/'); - return string.IsNullOrEmpty(trimmed) ? root : $"{root}/{trimmed}"; + if (trimmed.Length > 2048 + || trimmed.Contains('\\') + || trimmed.Any(char.IsControl) + || trimmed.Split('/').Any(segment => segment.Length == 0 || segment is "." or "..")) + { + virtualPath = string.Empty; + return false; + } + + virtualPath = string.IsNullOrEmpty(trimmed) ? root : $"{root}/{trimmed}"; + return true; } private static string GetAnimationVirtualRoot(AnimationInfo info) diff --git a/SecondDimensionWatcherReDive/Controllers/IncidentsController.cs b/SecondDimensionWatcherReDive/Controllers/IncidentsController.cs index fe2f785..4e62cc6 100644 --- a/SecondDimensionWatcherReDive/Controllers/IncidentsController.cs +++ b/SecondDimensionWatcherReDive/Controllers/IncidentsController.cs @@ -1,6 +1,7 @@ using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; +using SecondDimensionWatcherReDive.Framework.Authorization; using SecondDimensionWatcherReDive.Framework.DataRepository; using SecondDimensionWatcherReDive.Utils.Incidents; @@ -9,6 +10,7 @@ namespace SecondDimensionWatcherReDive.Controllers; [ApiController] [Route("api/incidents")] [Authorize(AuthenticationSchemes = JwtBearerDefaults.AuthenticationScheme)] +[Authorize(Policy = AccessPolicies.Administrator)] internal sealed class IncidentsController( IIncidentRepository incidentRepository, IIncidentRetryService retryService) : ControllerBase diff --git a/SecondDimensionWatcherReDive/Controllers/MediaLibraryController.cs b/SecondDimensionWatcherReDive/Controllers/MediaLibraryController.cs index 17e851b..2c54668 100644 --- a/SecondDimensionWatcherReDive/Controllers/MediaLibraryController.cs +++ b/SecondDimensionWatcherReDive/Controllers/MediaLibraryController.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Options; using Npgsql; using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Framework.Authorization; using SecondDimensionWatcherReDive.Services; using SecondDimensionWatcherReDive.Utils.FileStore; @@ -13,6 +14,7 @@ namespace SecondDimensionWatcherReDive.Controllers; [ApiController] [Route("api/media-library/sources")] [Authorize(AuthenticationSchemes = JwtBearerDefaults.AuthenticationScheme)] +[Authorize(Policy = AccessPolicies.Administrator)] internal sealed class MediaLibraryController( IMediaLibrarySourceRepository repository, IMediaLibraryScanQueue scanQueue, diff --git a/SecondDimensionWatcherReDive/Controllers/MetadataReviewController.cs b/SecondDimensionWatcherReDive/Controllers/MetadataReviewController.cs index 0820151..b107ba1 100644 --- a/SecondDimensionWatcherReDive/Controllers/MetadataReviewController.cs +++ b/SecondDimensionWatcherReDive/Controllers/MetadataReviewController.cs @@ -1,6 +1,7 @@ using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; +using SecondDimensionWatcherReDive.Framework.Authorization; using SecondDimensionWatcherReDive.Framework.DataRepository; using SecondDimensionWatcherReDive.Utils.MetadataReview; using External = SecondDimensionWatcherReDive.Controllers.External; @@ -10,6 +11,7 @@ namespace SecondDimensionWatcherReDive.Controllers; [ApiController] [Route("api/metadata-review")] [Authorize(AuthenticationSchemes = JwtBearerDefaults.AuthenticationScheme)] +[Authorize(Policy = AccessPolicies.Administrator)] internal sealed class MetadataReviewController( IMetadataReviewRepository metadataReviewRepository, IMetadataReviewService metadataReviewService) : ControllerBase diff --git a/SecondDimensionWatcherReDive/Controllers/PlaybackController.cs b/SecondDimensionWatcherReDive/Controllers/PlaybackController.cs index 2d4e5f3..1c9c741 100644 --- a/SecondDimensionWatcherReDive/Controllers/PlaybackController.cs +++ b/SecondDimensionWatcherReDive/Controllers/PlaybackController.cs @@ -1,9 +1,9 @@ using System.ComponentModel.DataAnnotations; -using System.Security.Claims; using System.Text.RegularExpressions; using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; +using SecondDimensionWatcherReDive.Framework.Authorization; using SecondDimensionWatcherReDive.Framework.DataRepository; using DataAnimationInfo = SecondDimensionWatcherReDive.Framework.DataRepository.AnimationInfo; @@ -36,7 +36,7 @@ public async Task ContinueWatching( [FromQuery, Range(1, MaxContinueLimit)] int limit = DefaultContinueLimit, CancellationToken cancellationToken = default) { - if (!TryGetUserId(out var userId)) return Unauthorized(); + if (!User.TryGetProfileId(out var userId)) return Unauthorized(); var items = await playbackRepository.GetContinueWatchingAsync(userId, limit, cancellationToken); var response = items @@ -52,7 +52,7 @@ public async Task GetStates( [FromQuery] Guid animationInfoId, CancellationToken cancellationToken) { - if (!TryGetUserId(out var userId)) return Unauthorized(); + if (!User.TryGetProfileId(out var userId)) return Unauthorized(); if (animationInfoId == Guid.Empty) return BadRequest(); var info = await animationInfoRepository.FindByIdWithAnimationAsync(animationInfoId, cancellationToken); @@ -86,7 +86,7 @@ public async Task GetContext( [FromQuery, Required] string? path, CancellationToken cancellationToken) { - if (!TryGetUserId(out var userId)) return Unauthorized(); + if (!User.TryGetProfileId(out var userId)) return Unauthorized(); var resolution = await ResolveVideoAsync(animationInfoId, path, cancellationToken); if (resolution.Status is ResolutionStatus.Invalid) return BadRequest(); if (resolution.Status is ResolutionStatus.Missing) return NotFound(); @@ -111,11 +111,12 @@ public async Task GetContext( } [HttpPut("progress")] + [Authorize(Policy = AccessPolicies.PlaybackWrite)] public async Task UpdateProgress( [FromBody] External.PlaybackProgressRequest request, CancellationToken cancellationToken) { - if (!TryGetUserId(out var userId)) return Unauthorized(); + if (!User.TryGetProfileId(out var userId)) return Unauthorized(); if (!double.IsFinite(request.PositionSeconds) || !double.IsFinite(request.DurationSeconds) || request.PositionSeconds < 0 @@ -152,11 +153,12 @@ public async Task UpdateProgress( } [HttpPut("watched")] + [Authorize(Policy = AccessPolicies.PlaybackWrite)] public async Task SetWatched( [FromBody] External.PlaybackWatchedRequest request, CancellationToken cancellationToken) { - if (!TryGetUserId(out var userId)) return Unauthorized(); + if (!User.TryGetProfileId(out var userId)) return Unauthorized(); var resolution = await ResolveVideoAsync(request.AnimationInfoId, request.Path, cancellationToken); if (resolution.Status is ResolutionStatus.Invalid) return BadRequest(); @@ -183,17 +185,18 @@ public async Task SetWatched( [HttpGet("preferences")] public async Task GetPreferences(CancellationToken cancellationToken) { - if (!TryGetUserId(out var userId)) return Unauthorized(); + if (!User.TryGetProfileId(out var userId)) return Unauthorized(); var preferences = await playbackRepository.GetPreferencesAsync(userId, cancellationToken); return Ok(ToPreferencesResponse(preferences)); } [HttpPut("preferences")] + [Authorize(Policy = AccessPolicies.PlaybackWrite)] public async Task UpdatePreferences( [FromBody] External.PlaybackPreferencesRequest request, CancellationToken cancellationToken) { - if (!TryGetUserId(out var userId)) return Unauthorized(); + if (!User.TryGetProfileId(out var userId)) return Unauthorized(); var preferences = new PlaybackPreferences( userId, @@ -355,14 +358,6 @@ private static External.PlaybackPreferencesResponse ToPreferencesResponse(Playba preferences.AutoPlayNext, preferences.UpdatedAt == DateTimeOffset.UnixEpoch ? null : preferences.UpdatedAt); - private bool TryGetUserId(out Guid userId) - { - var raw = User.FindFirstValue("Id") - ?? User.FindFirstValue(ClaimTypes.NameIdentifier) - ?? User.FindFirstValue("sub"); - return Guid.TryParse(raw, out userId); - } - private static bool TryNormalizeRelativePath(string? raw, out string normalized) { normalized = string.Empty; diff --git a/SecondDimensionWatcherReDive/Controllers/SeasonController.cs b/SecondDimensionWatcherReDive/Controllers/SeasonController.cs index 1fa0cb1..ee4eb50 100644 --- a/SecondDimensionWatcherReDive/Controllers/SeasonController.cs +++ b/SecondDimensionWatcherReDive/Controllers/SeasonController.cs @@ -1,6 +1,7 @@ using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; +using SecondDimensionWatcherReDive.Framework.Authorization; using SecondDimensionWatcherReDive.Framework.DataRepository; using SecondDimensionWatcherReDive.Framework.Tasks; using SecondDimensionWatcherReDive.Utils.Scraper; @@ -106,6 +107,7 @@ await bangumiSubgroupRepository.AddAsync(new BangumiSubgroup( /// Manually refresh the season anime list. [HttpPost("refresh")] + [Authorize(Policy = AccessPolicies.Administrator)] public async Task Refresh(CancellationToken cancellationToken) { // Rate limit: reject if last scrape < 10 minutes ago @@ -124,6 +126,7 @@ public async Task Refresh(CancellationToken cancellationToken) /// Subscribe to a bangumi by creating a Feed record. [HttpPost("subscribe")] + [Authorize(Policy = AccessPolicies.ContentWrite)] public async Task Subscribe([FromBody] External.SubscribeRequest request, CancellationToken cancellationToken) { diff --git a/SecondDimensionWatcherReDive/Controllers/SettingsController.cs b/SecondDimensionWatcherReDive/Controllers/SettingsController.cs index 5bab853..5e79d78 100644 --- a/SecondDimensionWatcherReDive/Controllers/SettingsController.cs +++ b/SecondDimensionWatcherReDive/Controllers/SettingsController.cs @@ -3,6 +3,7 @@ using Microsoft.AspNetCore.Mvc; using SecondDimensionWatcherReDive.Configuration; using SecondDimensionWatcherReDive.Controllers.External; +using SecondDimensionWatcherReDive.Framework.Authorization; namespace SecondDimensionWatcherReDive.Controllers; @@ -13,6 +14,7 @@ namespace SecondDimensionWatcherReDive.Controllers; internal sealed class SettingsController(IRuntimeSettingsService settingsService) : ControllerBase { [HttpGet] + [Authorize(Policy = AccessPolicies.Administrator)] public async Task> GetSettingsAsync( CancellationToken cancellationToken) { @@ -21,6 +23,7 @@ public async Task> GetSettingsAsync( } [HttpPatch] + [Authorize(Policy = AccessPolicies.RecentAdministrator)] public async Task PatchSettingsAsync( [FromBody] PatchApplicationSettingsRequest request, CancellationToken cancellationToken) diff --git a/SecondDimensionWatcherReDive/Controllers/SubscriptionPoliciesController.cs b/SecondDimensionWatcherReDive/Controllers/SubscriptionPoliciesController.cs index cf1a204..2dbc1c1 100644 --- a/SecondDimensionWatcherReDive/Controllers/SubscriptionPoliciesController.cs +++ b/SecondDimensionWatcherReDive/Controllers/SubscriptionPoliciesController.cs @@ -1,6 +1,7 @@ using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; +using SecondDimensionWatcherReDive.Framework.Authorization; using SecondDimensionWatcherReDive.Framework.DataRepository; using SecondDimensionWatcherReDive.Framework.Feed; @@ -36,6 +37,7 @@ public async Task GetPolicy( } [HttpPut("{feedId:guid}")] + [Authorize(Policy = AccessPolicies.ContentWrite)] public async Task UpsertPolicy( [FromRoute] Guid feedId, [FromBody] External.UpsertSubscriptionAutomationPolicyRequest request, @@ -73,6 +75,7 @@ public async Task SimulatePolicy( } [HttpDelete("{feedId:guid}")] + [Authorize(Policy = AccessPolicies.ContentWrite)] public async Task DeletePolicy( [FromRoute] Guid feedId, CancellationToken cancellationToken) diff --git a/SecondDimensionWatcherReDive/Controllers/TasksController.cs b/SecondDimensionWatcherReDive/Controllers/TasksController.cs index 50dd947..d0a789c 100644 --- a/SecondDimensionWatcherReDive/Controllers/TasksController.cs +++ b/SecondDimensionWatcherReDive/Controllers/TasksController.cs @@ -1,6 +1,7 @@ using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; +using SecondDimensionWatcherReDive.Framework.Authorization; using SecondDimensionWatcherReDive.Framework.Tasks; namespace SecondDimensionWatcherReDive.Controllers; @@ -24,6 +25,7 @@ public IActionResult GetTasks() } [HttpPost("{id}/run")] + [Authorize(Policy = AccessPolicies.Administrator)] public IActionResult RunTask([FromRoute] string id) { var task = scheduledTasks.FirstOrDefault(t => diff --git a/SecondDimensionWatcherReDive/Controllers/VfsController.cs b/SecondDimensionWatcherReDive/Controllers/VfsController.cs index bed4548..40ab1f9 100644 --- a/SecondDimensionWatcherReDive/Controllers/VfsController.cs +++ b/SecondDimensionWatcherReDive/Controllers/VfsController.cs @@ -22,12 +22,13 @@ internal sealed partial class VfsController( [HttpGet("stat")] public async Task Stat([FromQuery] string? path, CancellationToken cancellationToken) { - if (!TryNormalize(path, out var virtualPath)) return BadRequest(); + if (!TryGetScopedPaths(path, out var publicPath, out var internalPath)) + return BadRequest(); - var resource = await ResolveAsync(virtualPath, cancellationToken); + var resource = await ResolveAsync(publicPath, internalPath, cancellationToken); if (resource is null) { - LogResourceMissing(logger, virtualPath); + LogResourceMissing(logger, publicPath); return NotFound(); } @@ -38,25 +39,26 @@ public async Task Stat([FromQuery] string? path, CancellationToke [HttpGet("list")] public async Task List([FromQuery] string? path, CancellationToken cancellationToken) { - if (!TryNormalize(path, out var virtualPath)) return BadRequest(); + if (!TryGetScopedPaths(path, out var publicPath, out var internalPath)) + return BadRequest(); - var resource = await ResolveAsync(virtualPath, cancellationToken); + var resource = await ResolveAsync(publicPath, internalPath, cancellationToken); if (resource is null) { - LogResourceMissing(logger, virtualPath); + LogResourceMissing(logger, publicPath); return NotFound(); } if (!resource.IsDirectory) { - LogListOnFile(logger, virtualPath); + LogListOnFile(logger, publicPath); return BadRequest(); } - var directoryPath = EnsureTrailingSlash(resource.VirtualPath); - var directoryName = resource.VirtualPath == "/" + var directoryPath = EnsureTrailingSlash(resource.InternalPath); + var directoryName = resource.PublicPath == "/" ? string.Empty - : Path.GetFileName(resource.VirtualPath.TrimEnd('/')); + : Path.GetFileName(resource.PublicPath.TrimEnd('/')); var children = await fileExplorer.EnumerateDirectoryAsync( new DirectoryToken(directoryPath, directoryName), cancellationToken); @@ -73,17 +75,18 @@ public async Task List([FromQuery] string? path, CancellationToke [HttpGet("read")] public async Task Read([FromQuery] string? path, CancellationToken cancellationToken) { - if (!TryNormalize(path, out var virtualPath)) return BadRequest(); + if (!TryGetScopedPaths(path, out var publicPath, out var internalPath)) + return BadRequest(); - var resource = await ResolveAsync(virtualPath, cancellationToken); + var resource = await ResolveAsync(publicPath, internalPath, cancellationToken); if (resource is null || resource.IsDirectory || resource.Mapping is null) { - LogResourceMissing(logger, virtualPath); + LogResourceMissing(logger, publicPath); return NotFound(); } var mapping = resource.Mapping; - var fileName = Path.GetFileName(mapping.VirtualPath); + var fileName = Path.GetFileName(resource.PublicPath); var contentType = contentTypeProvider.TryGetContentType(fileName, out var ct) ? ct : "application/octet-stream"; @@ -95,9 +98,9 @@ public async Task Read([FromQuery] string? path, CancellationToke private async Task BuildEntryAsync(ResolvedResource resource, CancellationToken cancellationToken) { - var name = resource.VirtualPath == "/" + var name = resource.PublicPath == "/" ? string.Empty - : Path.GetFileName(resource.VirtualPath.TrimEnd('/')); + : Path.GetFileName(resource.PublicPath.TrimEnd('/')); if (resource.IsDirectory || resource.Mapping is null) return new External.VfsEntry(name, IsDirectory: true, Size: null, LastModifiedUtc: null); @@ -137,53 +140,46 @@ public async Task Read([FromQuery] string? path, CancellationToke } } - private async Task ResolveAsync(string virtualPath, CancellationToken cancellationToken) + private async Task ResolveAsync( + string publicPath, + string internalPath, + CancellationToken cancellationToken) { - if (virtualPath == "/") return new ResolvedResource("/", IsDirectory: true, null); + if (internalPath == "/") + return new ResolvedResource(publicPath, internalPath, IsDirectory: true, null); - var trimmed = virtualPath.TrimEnd('/'); - if (trimmed.Length == 0) return new ResolvedResource("/", IsDirectory: true, null); + var trimmed = internalPath.TrimEnd('/'); + if (trimmed.Length == 0) + return new ResolvedResource(publicPath, "/", IsDirectory: true, null); var mapping = await fileMappingRepository.FindByVirtualPathAsync(trimmed, cancellationToken); - if (mapping is not null) return new ResolvedResource(trimmed, IsDirectory: false, mapping); + if (mapping is not null) + return new ResolvedResource(publicPath, trimmed, IsDirectory: false, mapping); var prefix = trimmed + "/"; var children = await fileMappingRepository.GetByVirtualPathPrefixAsync(prefix, cancellationToken); - return children.Count > 0 ? new ResolvedResource(trimmed, IsDirectory: true, null) : null; + return children.Count > 0 + ? new ResolvedResource(publicPath, trimmed, IsDirectory: true, null) + : null; } - private static bool TryNormalize(string? raw, out string normalized) - { - if (string.IsNullOrEmpty(raw)) - { - normalized = "/"; - return true; - } - - if (!raw.StartsWith('/')) - { - normalized = string.Empty; - return false; - } - - // Reject path traversal segments. We never expect them in legitimate virtual paths. - foreach (var segment in raw.Split('/', StringSplitOptions.RemoveEmptyEntries)) - { - if (segment == "." || segment == "..") - { - normalized = string.Empty; - return false; - } - } - - var trimmed = raw.TrimEnd('/'); - normalized = trimmed.Length == 0 ? "/" : trimmed; - return true; - } + private bool TryGetScopedPaths( + string? raw, + out string publicPath, + out string internalPath) => + DevicePathScope.TryMapPublicToInternal( + raw, + DevicePathScope.GetVirtualRoot(User), + out publicPath, + out internalPath); private static string EnsureTrailingSlash(string path) => path.EndsWith('/') ? path : path + "/"; - private sealed record ResolvedResource(string VirtualPath, bool IsDirectory, FileMapping? Mapping); + private sealed record ResolvedResource( + string PublicPath, + string InternalPath, + bool IsDirectory, + FileMapping? Mapping); [LoggerMessage(Level = LogLevel.Debug, Message = "VFS resource not found: {VirtualPath}")] private static partial void LogResourceMissing(ILogger logger, string virtualPath); diff --git a/SecondDimensionWatcherReDive/Controllers/WebDavController.cs b/SecondDimensionWatcherReDive/Controllers/WebDavController.cs index cdc437f..dea443b 100644 --- a/SecondDimensionWatcherReDive/Controllers/WebDavController.cs +++ b/SecondDimensionWatcherReDive/Controllers/WebDavController.cs @@ -43,13 +43,14 @@ public IActionResult Options() [HttpPropFind(RouteTemplate)] public async Task PropFind(string? path, CancellationToken cancellationToken) { - var virtualPath = NormalizeVirtualPath(path); + if (!TryGetScopedPaths(path, out var publicPath, out var internalPath)) + return BadRequest(); var depth = ParseDepth(Request.Headers[WebDavConstants.Headers.Depth].ToString()); - var resource = await ResolveAsync(virtualPath, cancellationToken); + var resource = await ResolveAsync(publicPath, internalPath, cancellationToken); if (resource is null) { - LogResourceMissing(logger, virtualPath); + LogResourceMissing(logger, publicPath); return NotFound(); } @@ -69,16 +70,30 @@ public async Task PropFind(string? path, CancellationToken cancel if (depth == DepthValue.One && resource.IsDirectory) { var children = await fileExplorer.EnumerateDirectoryAsync( - new DirectoryToken(EnsureTrailingSlash(resource.VirtualPath), Path.GetFileName(resource.VirtualPath.TrimEnd('/'))), + new DirectoryToken( + EnsureTrailingSlash(resource.InternalPath), + Path.GetFileName(resource.InternalPath.TrimEnd('/'))), cancellationToken); foreach (var child in children) { + var childInternalPath = child switch + { + FileToken file => file.Path, + DirectoryToken directory => directory.Path, + _ => null + }; + if (childInternalPath is null) continue; + if (!DevicePathScope.TryMapInternalToPublic( + childInternalPath, + DevicePathScope.GetVirtualRoot(User), + out var childPublicPath)) + continue; var childResource = child switch { - FileToken f => new ResolvedResource(f.Path, IsDirectory: false, + FileToken f => new ResolvedResource(childPublicPath, f.Path, IsDirectory: false, await fileMappingRepository.FindByVirtualPathAsync(f.Path, cancellationToken)), - DirectoryToken d => new ResolvedResource(d.Path, IsDirectory: true, null), + DirectoryToken d => new ResolvedResource(childPublicPath, d.Path, IsDirectory: true, null), _ => null }; if (childResource is null) continue; @@ -93,11 +108,12 @@ await fileMappingRepository.FindByVirtualPathAsync(f.Path, cancellationToken)), [HttpHead(RouteTemplate)] public async Task GetFile(string? path, CancellationToken cancellationToken) { - var virtualPath = NormalizeVirtualPath(path); - var resource = await ResolveAsync(virtualPath, cancellationToken); + if (!TryGetScopedPaths(path, out var publicPath, out var internalPath)) + return BadRequest(); + var resource = await ResolveAsync(publicPath, internalPath, cancellationToken); if (resource is null) { - LogResourceMissing(logger, virtualPath); + LogResourceMissing(logger, publicPath); return NotFound(); } @@ -109,7 +125,7 @@ public async Task GetFile(string? path, CancellationToken cancell } var mapping = resource.Mapping!; - var fileName = Path.GetFileName(mapping.VirtualPath); + var fileName = Path.GetFileName(resource.PublicPath); var contentType = ResolveContentType(fileName); var stream = await fileExplorer.OpenReadStreamAsync(new FileToken(mapping.VirtualPath, fileName), cancellationToken); @@ -134,14 +150,14 @@ private async Task BuildResponseAsync(ResolvedResource resource, Pr { var response = new DavResponse { - Href = BuildHref(resource.VirtualPath, resource.IsDirectory) + Href = BuildHref(resource.PublicPath, resource.IsDirectory) }; var prop = new Prop { - DisplayName = resource.VirtualPath == "/" + DisplayName = resource.PublicPath == "/" ? string.Empty - : Path.GetFileName(resource.VirtualPath.TrimEnd('/')) + : Path.GetFileName(resource.PublicPath.TrimEnd('/')) }; if (resource.IsDirectory) @@ -208,19 +224,27 @@ private async Task BuildResponseAsync(ResolvedResource resource, Pr return response; } - private async Task ResolveAsync(string virtualPath, CancellationToken cancellationToken) + private async Task ResolveAsync( + string publicPath, + string internalPath, + CancellationToken cancellationToken) { - if (virtualPath == "/") return new ResolvedResource("/", IsDirectory: true, null); + if (internalPath == "/") + return new ResolvedResource(publicPath, internalPath, IsDirectory: true, null); - var trimmed = virtualPath.TrimEnd('/'); - if (trimmed.Length == 0) return new ResolvedResource("/", IsDirectory: true, null); + var trimmed = internalPath.TrimEnd('/'); + if (trimmed.Length == 0) + return new ResolvedResource(publicPath, "/", IsDirectory: true, null); var mapping = await fileMappingRepository.FindByVirtualPathAsync(trimmed, cancellationToken); - if (mapping is not null) return new ResolvedResource(trimmed, IsDirectory: false, mapping); + if (mapping is not null) + return new ResolvedResource(publicPath, trimmed, IsDirectory: false, mapping); var prefix = trimmed + "/"; var children = await fileMappingRepository.GetByVirtualPathPrefixAsync(prefix, cancellationToken); - return children.Count > 0 ? new ResolvedResource(trimmed, IsDirectory: true, null) : null; + return children.Count > 0 + ? new ResolvedResource(publicPath, trimmed, IsDirectory: true, null) + : null; } private async Task TryReadPropFindRequestAsync(CancellationToken cancellationToken) @@ -246,11 +270,21 @@ private async Task BuildResponseAsync(ResolvedResource resource, Pr private string ResolveContentType(string fileName) => contentTypeProvider.TryGetContentType(fileName, out var ct) ? ct : "application/octet-stream"; - private static string NormalizeVirtualPath(string? routeValue) + private bool TryGetScopedPaths( + string? routeValue, + out string publicPath, + out string internalPath) { - if (string.IsNullOrEmpty(routeValue)) return "/"; - var trimmed = routeValue.Trim('/'); - return trimmed.Length == 0 ? "/" : "/" + trimmed; + var absolutePath = string.IsNullOrEmpty(routeValue) + ? "/" + : routeValue.StartsWith("/", StringComparison.Ordinal) + ? routeValue + : "/" + routeValue; + return DevicePathScope.TryMapPublicToInternal( + absolutePath, + DevicePathScope.GetVirtualRoot(User), + out publicPath, + out internalPath); } private static string EnsureTrailingSlash(string path) => path.EndsWith('/') ? path : path + "/"; @@ -405,7 +439,11 @@ private static void ApplyFilter(Prop prop, PropFilter filter) if ((filter.Keys & PropertyKeys.Executable) == 0) prop.Executable = null; } - private sealed record ResolvedResource(string VirtualPath, bool IsDirectory, FileMapping? Mapping); + private sealed record ResolvedResource( + string PublicPath, + string InternalPath, + bool IsDirectory, + FileMapping? Mapping); private static readonly object QuotaLock = new(); private static (string? Root, long Total, long Available, DateTime FetchedAt) _quotaCache; diff --git a/SecondDimensionWatcherReDive/Controllers/WebDavTokenController.cs b/SecondDimensionWatcherReDive/Controllers/WebDavTokenController.cs index 18c9cd6..4f8f93a 100644 --- a/SecondDimensionWatcherReDive/Controllers/WebDavTokenController.cs +++ b/SecondDimensionWatcherReDive/Controllers/WebDavTokenController.cs @@ -3,6 +3,8 @@ using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; +using SecondDimensionWatcherReDive.Auth; +using SecondDimensionWatcherReDive.Framework.Authorization; using SecondDimensionWatcherReDive.Framework.DataRepository; namespace SecondDimensionWatcherReDive.Controllers; @@ -10,16 +12,22 @@ namespace SecondDimensionWatcherReDive.Controllers; [ApiController] [Route("api/webdav-tokens")] [Authorize(AuthenticationSchemes = JwtBearerDefaults.AuthenticationScheme)] -internal partial class WebDavTokenController(IWebDavTokenRepository repository) : ControllerBase +internal partial class WebDavTokenController( + IWebDavTokenRepository repository, + IIdentityRepository identityRepository, + IFileMappingRepository fileMappingRepository) : ControllerBase { private const string UsernameAlphabet = "abcdefghijklmnopqrstuvwxyz0123456789"; private const int GeneratedUsernameLength = 8; private const int TokenByteLength = 32; + private static readonly TimeSpan DefaultLifetime = TimeSpan.FromDays(365); + private static readonly TimeSpan MaximumLifetime = TimeSpan.FromDays(365 * 5); [GeneratedRegex(@"^[A-Za-z0-9._-]{3,32}$")] private static partial Regex UsernamePattern(); [HttpGet] + [Authorize(Policy = AccessPolicies.Administrator)] public async Task ListTokens(CancellationToken cancellationToken) { var records = await repository.GetAllOrderedAsync(cancellationToken); @@ -27,6 +35,7 @@ public async Task ListTokens(CancellationToken cancellationToken) } [HttpPost] + [Authorize(Policy = AccessPolicies.RecentAdministrator)] public async Task CreateToken( [FromBody] External.CreateWebDavTokenRequest request, CancellationToken cancellationToken) @@ -42,10 +51,37 @@ public async Task CreateToken( if (await repository.ExistsByUsernameAsync(username, cancellationToken)) return Conflict(new { error = "Username already exists." }); + if (!User.TryGetUserId(out var currentUserId)) return Unauthorized(); + var userId = request.UserId ?? currentUserId; + var targetUser = await identityRepository.FindUserByIdAsync(userId, cancellationToken); + if (targetUser is null || targetUser.IsDisabled) return BadRequest(); + + if (!DevicePathScope.TryNormalizeAbsolutePath( + request.VirtualRoot, out var virtualRoot)) + return BadRequest(new { error = "VirtualRoot must be an absolute path without traversal segments." }); + if (!await IsDirectoryAsync(virtualRoot, cancellationToken)) + return BadRequest(new { error = "VirtualRoot must identify an existing directory." }); + + var now = DateTimeOffset.UtcNow; + var expiresAt = request.ExpiresAt ?? now + DefaultLifetime; + if (expiresAt <= now || expiresAt > now + MaximumLifetime) + return BadRequest(new { error = "ExpiresAt must be in the future and no more than five years away." }); + var plaintext = GenerateToken(); var hash = BCrypt.Net.BCrypt.HashPassword(plaintext); var description = string.IsNullOrWhiteSpace(request.Description) ? null : request.Description.Trim(); - var record = new WebDavToken(Guid.NewGuid(), username, hash, description, DateTimeOffset.UtcNow); + if (description?.Length > 256) return BadRequest(); + var record = new WebDavToken( + Guid.NewGuid(), + userId, + username, + hash, + description, + now, + "read", + virtualRoot, + expiresAt, + null); await repository.AddAsync(record, cancellationToken); @@ -54,16 +90,35 @@ public async Task CreateToken( record.Username, plaintext, record.Description, - record.CreatedAt)); + record.CreatedAt, + record.UserId, + record.Scope, + record.VirtualRoot, + expiresAt)); } [HttpDelete("{id:guid}")] + [Authorize(Policy = AccessPolicies.RecentAdministrator)] public async Task DeleteToken([FromRoute] Guid id, CancellationToken cancellationToken) { - var removed = await repository.RemoveByIdAsync(id, cancellationToken); + var removed = await repository.RevokeByIdAsync( + id, DateTimeOffset.UtcNow, cancellationToken); return removed ? NoContent() : NotFound(); } + private async Task IsDirectoryAsync( + string virtualRoot, + CancellationToken cancellationToken) + { + if (virtualRoot == "/") return true; + if (await fileMappingRepository.FindByVirtualPathAsync( + virtualRoot, cancellationToken) is not null) + return false; + var children = await fileMappingRepository.GetByVirtualPathPrefixAsync( + virtualRoot + "/", cancellationToken); + return children.Count > 0; + } + private static string GenerateUsername() => "sdw-" + RandomNumberGenerator.GetString(UsernameAlphabet, GeneratedUsernameLength); diff --git a/SecondDimensionWatcherReDive/Migrations/20260829155550_AddHouseholdIdentityAndAccessScopes.Designer.cs b/SecondDimensionWatcherReDive/Migrations/20260829155550_AddHouseholdIdentityAndAccessScopes.Designer.cs new file mode 100644 index 0000000..5614ced --- /dev/null +++ b/SecondDimensionWatcherReDive/Migrations/20260829155550_AddHouseholdIdentityAndAccessScopes.Designer.cs @@ -0,0 +1,1205 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using SecondDimensionWatcherReDive.Models; + +#nullable disable + +namespace SecondDimensionWatcherReDive.Migrations +{ + [DbContext(typeof(ApplicationContext))] + [Migration("20260829155550_AddHouseholdIdentityAndAccessScopes")] + partial class AddHouseholdIdentityAndAccessScopes + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.Animation", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("OriginalName") + .IsRequired() + .HasColumnType("text"); + + b.Property("PosterPath") + .HasColumnType("text"); + + b.Property("TmdbId") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("TmdbId") + .IsUnique(); + + b.ToTable("Animations"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.AnimationGroup", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("AnimationGroups"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.AnimationInfo", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AdditionalDownloadInfo") + .IsRequired() + .HasColumnType("text"); + + b.Property("AiRetryCount") + .HasColumnType("integer"); + + b.Property("AnimationId") + .HasColumnType("uuid"); + + b.Property("AutomationDisposition") + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("AutomationExplanationJson") + .HasColumnType("text"); + + b.Property("CachedDownloadData") + .IsRequired() + .HasColumnType("bytea"); + + b.Property("CurrentMetadataReviewOperationId") + .HasColumnType("uuid"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text"); + + b.Property("DownloadAttemptId") + .HasColumnType("uuid"); + + b.Property("DownloadCancellationId") + .HasColumnType("uuid"); + + b.Property("DownloadEndTime") + .HasColumnType("timestamp with time zone"); + + b.Property("DownloadStartTime") + .HasColumnType("timestamp with time zone"); + + b.Property("DownloadType") + .IsRequired() + .HasColumnType("text"); + + b.Property("DownloadUrl") + .IsRequired() + .HasColumnType("text"); + + b.Property("Episode") + .HasColumnType("integer"); + + b.Property("FileStore") + .HasColumnType("text"); + + b.Property("GroupId") + .HasColumnType("uuid"); + + b.Property("IsAiProcessed") + .HasColumnType("boolean"); + + b.Property("IsDownloadFinished") + .HasColumnType("boolean"); + + b.Property("IsDownloadTracked") + .HasColumnType("boolean"); + + b.Property("MediaLibraryMissingSince") + .HasColumnType("timestamp with time zone"); + + b.Property("MediaLibrarySourceId") + .HasColumnType("uuid"); + + b.Property("MetadataConfidence") + .HasColumnType("double precision"); + + b.Property("MetadataLastError") + .HasMaxLength(1024) + .HasColumnType("character varying(1024)"); + + b.Property("MetadataReviewedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("MetadataStatus") + .HasColumnType("integer"); + + b.Property("PublishTime") + .HasColumnType("timestamp with time zone"); + + b.Property("ReleaseSizeBytes") + .HasColumnType("bigint"); + + b.Property("Season") + .HasColumnType("integer"); + + b.Property("SourceFeedId") + .HasColumnType("uuid"); + + b.Property("StateVersion") + .IsConcurrencyToken() + .HasColumnType("bigint"); + + b.Property("StorePath") + .HasColumnType("text"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("AnimationId"); + + b.HasIndex("CurrentMetadataReviewOperationId") + .IsUnique(); + + b.HasIndex("GroupId"); + + b.HasIndex("MediaLibrarySourceId"); + + b.HasIndex("SourceFeedId"); + + b.HasIndex("FileStore", "StorePath") + .IsUnique() + .HasFilter("\"DownloadType\" = 'http://schemas.hcgstudio.com/ws/2023/06/sdw/downloadtype/media-library-import'"); + + b.HasIndex("MetadataStatus", "PublishTime"); + + b.ToTable("AnimationInfo", t => + { + t.HasCheckConstraint("CK_AnimationInfo_MetadataConfidence_Range", "\"MetadataConfidence\" IS NULL OR (\"MetadataConfidence\" >= 0 AND \"MetadataConfidence\" <= 1)"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ApplicationSettings", b => + { + b.Property("Id") + .HasColumnType("integer"); + + b.Property("ProtectedSecrets") + .HasColumnType("text"); + + b.Property("Revision") + .IsConcurrencyToken() + .HasColumnType("bigint"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("ValuesJson") + .IsRequired() + .HasColumnType("jsonb"); + + b.HasKey("Id"); + + b.ToTable("ApplicationSettings", t => + { + t.HasCheckConstraint("CK_ApplicationSettings_Revision_Positive", "\"Revision\" > 0"); + + t.HasCheckConstraint("CK_ApplicationSettings_Singleton", "\"Id\" = 1"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.BangumiSubgroup", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("MikanSubgroupId") + .HasColumnType("integer"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("ScrapedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("SeasonBangumiId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("SeasonBangumiId", "MikanSubgroupId") + .IsUnique(); + + b.ToTable("BangumiSubgroups"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ChatConversation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("ProfileId") + .HasColumnType("uuid"); + + b.Property("Title") + .HasColumnType("text"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("ProfileId", "UpdatedAt"); + + b.ToTable("ChatConversations"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ChatMessage", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Content") + .HasColumnType("text"); + + b.Property("ConversationId") + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Order") + .HasColumnType("integer"); + + b.Property("Role") + .IsRequired() + .HasColumnType("text"); + + b.Property("ToolCallId") + .HasColumnType("text"); + + b.Property("ToolCallsJson") + .HasColumnType("text"); + + b.Property("ToolName") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("ConversationId"); + + b.ToTable("ChatMessages"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.Feed", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Name") + .HasColumnType("text"); + + b.Property("Url") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("Feeds"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.FileMapping", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AnimationInfoId") + .HasColumnType("uuid"); + + b.Property("FileStore") + .IsRequired() + .HasColumnType("text"); + + b.Property("PhysicalPath") + .IsRequired() + .HasColumnType("text"); + + b.Property("VirtualPath") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("AnimationInfoId"); + + b.HasIndex("VirtualPath") + .IsUnique(); + + b.ToTable("FileMappings"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.FileNameRegexRule", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AnimationId") + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("Pattern") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("character varying(512)"); + + b.HasKey("Id"); + + b.HasIndex("AnimationId", "CreatedAt"); + + b.HasIndex("AnimationId", "Pattern") + .IsUnique(); + + b.ToTable("FileNameRegexRules"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.Incident", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Detail") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("DetectedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Fingerprint") + .IsRequired() + .HasMaxLength(96) + .HasColumnType("character varying(96)"); + + b.Property("LastRetryAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastRetryError") + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("ResolvedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("RetryCount") + .HasColumnType("integer"); + + b.Property("Severity") + .HasColumnType("integer"); + + b.Property("SourceId") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("Title") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("Type") + .HasColumnType("integer"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("Fingerprint") + .IsUnique(); + + b.HasIndex("ResolvedAt", "Type", "UpdatedAt"); + + b.ToTable("Incidents"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.LoginSession", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("ActiveProfileId") + .HasColumnType("uuid"); + + b.Property("AuthenticatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("DeviceName") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("ExpiresAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastSeenAt") + .HasColumnType("timestamp with time zone"); + + b.Property("RefreshTokenHash") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("RevokedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("ActiveProfileId"); + + b.HasIndex("UserId", "RevokedAt", "ExpiresAt"); + + b.ToTable("LoginSessions"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MediaLibrarySource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("IsMonitoring") + .HasColumnType("boolean"); + + b.Property("LastError") + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("LastImportedCount") + .HasColumnType("integer"); + + b.Property("LastRemovedCount") + .HasColumnType("integer"); + + b.Property("LastScanAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastSkippedCount") + .HasColumnType("integer"); + + b.Property("LastUpdatedCount") + .HasColumnType("integer"); + + b.Property("Path") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Path") + .IsUnique(); + + b.ToTable("MediaLibrarySources"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewMappingSnapshot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("FileStore") + .IsRequired() + .HasColumnType("text"); + + b.Property("Kind") + .HasColumnType("integer"); + + b.Property("OperationId") + .HasColumnType("uuid"); + + b.Property("PhysicalPath") + .IsRequired() + .HasColumnType("text"); + + b.Property("VirtualPath") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("OperationId", "Kind", "VirtualPath") + .IsUnique(); + + b.ToTable("MetadataReviewMappingSnapshots"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AnimationInfoId") + .HasColumnType("uuid"); + + b.Property("AppliedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("AppliedVersion") + .HasColumnType("bigint"); + + b.Property("BaseFileStore") + .HasColumnType("text"); + + b.Property("BaseIsDownloadFinished") + .HasColumnType("boolean"); + + b.Property("BaseStorePath") + .HasColumnType("text"); + + b.Property("BaseVersion") + .HasColumnType("bigint"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("ExpiresAt") + .HasColumnType("timestamp with time zone"); + + b.Property("PreviousAiRetryCount") + .HasColumnType("integer"); + + b.Property("PreviousAnimationId") + .HasColumnType("uuid"); + + b.Property("PreviousConfidence") + .HasColumnType("double precision"); + + b.Property("PreviousCurrentOperationId") + .HasColumnType("uuid"); + + b.Property("PreviousDescription") + .HasColumnType("text"); + + b.Property("PreviousEpisode") + .HasColumnType("integer"); + + b.Property("PreviousGroupId") + .HasColumnType("uuid"); + + b.Property("PreviousIsAiProcessed") + .HasColumnType("boolean"); + + b.Property("PreviousLastError") + .HasColumnType("text"); + + b.Property("PreviousMetadataStatus") + .HasColumnType("integer"); + + b.Property("PreviousReviewedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("PreviousSeason") + .HasColumnType("integer"); + + b.Property("ProposedAnimationName") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProposedAnimationOriginalName") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProposedAnimationPosterPath") + .HasColumnType("text"); + + b.Property("ProposedAnimationTmdbId") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProposedDescription") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProposedEpisode") + .HasColumnType("integer"); + + b.Property("ProposedGroupName") + .HasColumnType("text"); + + b.Property("ProposedSeason") + .HasColumnType("integer"); + + b.Property("State") + .HasColumnType("integer"); + + b.Property("UndoneAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("AnimationInfoId", "AppliedVersion") + .IsUnique(); + + b.HasIndex("AnimationInfoId", "State"); + + b.HasIndex("State", "ExpiresAt"); + + b.ToTable("MetadataReviewOperations", t => + { + t.HasCheckConstraint("CK_MetadataReviewOperations_Expiry", "\"ExpiresAt\" > \"CreatedAt\""); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MigrationMarker", b => + { + b.Property("Key") + .HasColumnType("text"); + + b.Property("AppliedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Key"); + + b.ToTable("MigrationMarkers"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.PlaybackPreference", b => + { + b.Property("UserId") + .HasColumnType("uuid"); + + b.Property("AudioLanguage") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("AudioTrackLabel") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("AutoPlayNext") + .HasColumnType("boolean"); + + b.Property("SubtitleLanguage") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("SubtitleTrackLabel") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("UserId"); + + b.ToTable("PlaybackPreferences"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.PlaybackProgress", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AnimationInfoId") + .HasColumnType("uuid"); + + b.Property("DurationSeconds") + .HasColumnType("double precision"); + + b.Property("IsWatched") + .HasColumnType("boolean"); + + b.Property("PositionSeconds") + .HasColumnType("double precision"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.Property("VirtualPath") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("WatchedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("AnimationInfoId"); + + b.HasIndex("UserId", "AnimationInfoId", "VirtualPath") + .IsUnique(); + + b.HasIndex("UserId", "IsWatched", "UpdatedAt"); + + b.ToTable("PlaybackProgresses", t => + { + t.HasCheckConstraint("CK_PlaybackProgresses_Duration_NonNegative", "\"DurationSeconds\" >= 0"); + + t.HasCheckConstraint("CK_PlaybackProgresses_Position_NonNegative", "\"PositionSeconds\" >= 0"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SeasonBangumi", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("DayOfWeek") + .HasColumnType("integer"); + + b.Property("ImageUrl") + .HasColumnType("text"); + + b.Property("MikanId") + .HasColumnType("integer"); + + b.Property("ScrapedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("MikanId") + .IsUnique(); + + b.ToTable("SeasonBangumis"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SubscriptionAutomationPolicy", b => + { + b.Property("FeedId") + .HasColumnType("uuid"); + + b.PrimitiveCollection("Codecs") + .IsRequired() + .HasColumnType("text[]"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.PrimitiveCollection("ExcludedKeywords") + .IsRequired() + .HasColumnType("text[]"); + + b.PrimitiveCollection("Languages") + .IsRequired() + .HasColumnType("text[]"); + + b.Property("MaxSizeBytes") + .HasColumnType("bigint"); + + b.Property("MinSizeBytes") + .HasColumnType("bigint"); + + b.Property("Mode") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.PrimitiveCollection("Resolutions") + .IsRequired() + .HasColumnType("text[]"); + + b.PrimitiveCollection("SubtitleGroups") + .IsRequired() + .HasColumnType("text[]"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("FeedId"); + + b.HasIndex("UpdatedAt"); + + b.ToTable("SubscriptionAutomationPolicies"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.UserAccount", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("IsDisabled") + .HasColumnType("boolean"); + + b.Property("PasswordHash") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("Role") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Username") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.HasKey("Id"); + + b.HasIndex("Username") + .IsUnique(); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.UserProfile", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("Avatar") + .HasMaxLength(512) + .HasColumnType("character varying(512)"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("IsDefault") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("PinHash") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("UserId", "Name") + .IsUnique(); + + b.ToTable("Profiles"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.WebDavToken", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("ExpiresAt") + .HasColumnType("timestamp with time zone"); + + b.Property("RevokedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Scope") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("TokenHash") + .IsRequired() + .HasColumnType("text"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.Property("Username") + .IsRequired() + .HasColumnType("text"); + + b.Property("VirtualRoot") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.HasKey("Id"); + + b.HasIndex("UserId"); + + b.HasIndex("Username") + .IsUnique(); + + b.ToTable("WebDavTokens"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.AnimationInfo", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.Animation", "Animation") + .WithMany() + .HasForeignKey("AnimationId"); + + b.HasOne("SecondDimensionWatcherReDive.Models.AnimationGroup", "Group") + .WithMany() + .HasForeignKey("GroupId"); + + b.HasOne("SecondDimensionWatcherReDive.Models.MediaLibrarySource", null) + .WithMany() + .HasForeignKey("MediaLibrarySourceId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("SecondDimensionWatcherReDive.Models.Feed", null) + .WithMany() + .HasForeignKey("SourceFeedId") + .OnDelete(DeleteBehavior.SetNull); + + b.Navigation("Animation"); + + b.Navigation("Group"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.BangumiSubgroup", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.SeasonBangumi", "SeasonBangumi") + .WithMany("Subgroups") + .HasForeignKey("SeasonBangumiId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("SeasonBangumi"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ChatConversation", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.UserProfile", "Profile") + .WithMany() + .HasForeignKey("ProfileId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Profile"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ChatMessage", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.ChatConversation", "Conversation") + .WithMany("Messages") + .HasForeignKey("ConversationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Conversation"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.FileNameRegexRule", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.Animation", null) + .WithMany() + .HasForeignKey("AnimationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.LoginSession", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.UserProfile", "ActiveProfile") + .WithMany() + .HasForeignKey("ActiveProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("SecondDimensionWatcherReDive.Models.UserAccount", "User") + .WithMany("Sessions") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ActiveProfile"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewMappingSnapshot", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.MetadataReviewOperation", "Operation") + .WithMany("MappingSnapshots") + .HasForeignKey("OperationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Operation"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewOperation", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.AnimationInfo", "AnimationInfo") + .WithMany() + .HasForeignKey("AnimationInfoId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AnimationInfo"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.PlaybackPreference", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.UserProfile", "Profile") + .WithOne() + .HasForeignKey("SecondDimensionWatcherReDive.Models.PlaybackPreference", "UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Profile"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.PlaybackProgress", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.AnimationInfo", "AnimationInfo") + .WithMany() + .HasForeignKey("AnimationInfoId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("SecondDimensionWatcherReDive.Models.UserProfile", "Profile") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AnimationInfo"); + + b.Navigation("Profile"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SubscriptionAutomationPolicy", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.Feed", "Feed") + .WithOne() + .HasForeignKey("SecondDimensionWatcherReDive.Models.SubscriptionAutomationPolicy", "FeedId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Feed"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.UserProfile", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.UserAccount", "User") + .WithMany("Profiles") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("User"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.WebDavToken", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.UserAccount", "User") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("User"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ChatConversation", b => + { + b.Navigation("Messages"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewOperation", b => + { + b.Navigation("MappingSnapshots"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SeasonBangumi", b => + { + b.Navigation("Subgroups"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.UserAccount", b => + { + b.Navigation("Profiles"); + + b.Navigation("Sessions"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/SecondDimensionWatcherReDive/Migrations/20260829155550_AddHouseholdIdentityAndAccessScopes.cs b/SecondDimensionWatcherReDive/Migrations/20260829155550_AddHouseholdIdentityAndAccessScopes.cs new file mode 100644 index 0000000..7c2317e --- /dev/null +++ b/SecondDimensionWatcherReDive/Migrations/20260829155550_AddHouseholdIdentityAndAccessScopes.cs @@ -0,0 +1,345 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace SecondDimensionWatcherReDive.Migrations +{ + /// + public partial class AddHouseholdIdentityAndAccessScopes : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "ExpiresAt", + table: "WebDavTokens", + type: "timestamp with time zone", + nullable: true); + + migrationBuilder.AddColumn( + name: "RevokedAt", + table: "WebDavTokens", + type: "timestamp with time zone", + nullable: true); + + migrationBuilder.AddColumn( + name: "Scope", + table: "WebDavTokens", + type: "character varying(32)", + maxLength: 32, + nullable: false, + defaultValue: "read"); + + migrationBuilder.AddColumn( + name: "UserId", + table: "WebDavTokens", + type: "uuid", + nullable: false, + defaultValue: new Guid("00000000-0000-0000-0000-000000000001")); + + migrationBuilder.AddColumn( + name: "VirtualRoot", + table: "WebDavTokens", + type: "character varying(2048)", + maxLength: 2048, + nullable: false, + defaultValue: "/"); + + migrationBuilder.AddColumn( + name: "ProfileId", + table: "ChatConversations", + type: "uuid", + nullable: false, + defaultValue: new Guid("00000000-0000-0000-0000-000000000000")); + + migrationBuilder.CreateTable( + name: "Users", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + Username = table.Column(type: "character varying(64)", maxLength: 64, nullable: false), + PasswordHash = table.Column(type: "character varying(128)", maxLength: 128, nullable: true), + Role = table.Column(type: "character varying(16)", maxLength: 16, nullable: false), + IsDisabled = table.Column(type: "boolean", nullable: false), + CreatedAt = table.Column(type: "timestamp with time zone", nullable: false), + UpdatedAt = table.Column(type: "timestamp with time zone", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_Users", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "Profiles", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + UserId = table.Column(type: "uuid", nullable: false), + Name = table.Column(type: "character varying(64)", maxLength: 64, nullable: false), + Avatar = table.Column(type: "character varying(512)", maxLength: 512, nullable: true), + PinHash = table.Column(type: "character varying(128)", maxLength: 128, nullable: true), + IsDefault = table.Column(type: "boolean", nullable: false), + CreatedAt = table.Column(type: "timestamp with time zone", nullable: false), + UpdatedAt = table.Column(type: "timestamp with time zone", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_Profiles", x => x.Id); + table.ForeignKey( + name: "FK_Profiles_Users_UserId", + column: x => x.UserId, + principalTable: "Users", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateTable( + name: "LoginSessions", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + UserId = table.Column(type: "uuid", nullable: false), + ActiveProfileId = table.Column(type: "uuid", nullable: false), + RefreshTokenHash = table.Column(type: "character varying(64)", maxLength: 64, nullable: false), + DeviceName = table.Column(type: "character varying(128)", maxLength: 128, nullable: true), + AuthenticatedAt = table.Column(type: "timestamp with time zone", nullable: false), + CreatedAt = table.Column(type: "timestamp with time zone", nullable: false), + LastSeenAt = table.Column(type: "timestamp with time zone", nullable: false), + ExpiresAt = table.Column(type: "timestamp with time zone", nullable: false), + RevokedAt = table.Column(type: "timestamp with time zone", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_LoginSessions", x => x.Id); + table.ForeignKey( + name: "FK_LoginSessions_Profiles_ActiveProfileId", + column: x => x.ActiveProfileId, + principalTable: "Profiles", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + table.ForeignKey( + name: "FK_LoginSessions_Users_UserId", + column: x => x.UserId, + principalTable: "Users", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + // Existing installations authenticated a single household and stored playback + // rows under Guid.Empty. Materialize that household only when legacy data exists; + // a truly fresh database must remain eligible for first-user registration. + migrationBuilder.Sql( + """ + INSERT INTO "Users" + ("Id", "Username", "PasswordHash", "Role", "IsDisabled", "CreatedAt", "UpdatedAt") + SELECT + '00000000-0000-0000-0000-000000000001', + 'admin', + NULL, + 'Admin', + FALSE, + CURRENT_TIMESTAMP, + CURRENT_TIMESTAMP + WHERE EXISTS (SELECT 1 FROM "PlaybackProgresses") + OR EXISTS (SELECT 1 FROM "PlaybackPreferences") + OR EXISTS (SELECT 1 FROM "ChatConversations") + OR EXISTS (SELECT 1 FROM "WebDavTokens"); + + INSERT INTO "Profiles" + ("Id", "UserId", "Name", "Avatar", "PinHash", "IsDefault", "CreatedAt", "UpdatedAt") + SELECT + '00000000-0000-0000-0000-000000000000', + '00000000-0000-0000-0000-000000000001', + 'Home', + NULL, + NULL, + TRUE, + CURRENT_TIMESTAMP, + CURRENT_TIMESTAMP + WHERE EXISTS ( + SELECT 1 FROM "Users" + WHERE "Id" = '00000000-0000-0000-0000-000000000001'); + + UPDATE "PlaybackProgresses" + SET "UserId" = '00000000-0000-0000-0000-000000000000'; + + UPDATE "PlaybackPreferences" + SET "UserId" = '00000000-0000-0000-0000-000000000000'; + """); + + migrationBuilder.CreateIndex( + name: "IX_WebDavTokens_UserId", + table: "WebDavTokens", + column: "UserId"); + + migrationBuilder.CreateIndex( + name: "IX_ChatConversations_ProfileId_UpdatedAt", + table: "ChatConversations", + columns: new[] { "ProfileId", "UpdatedAt" }); + + migrationBuilder.CreateIndex( + name: "IX_LoginSessions_ActiveProfileId", + table: "LoginSessions", + column: "ActiveProfileId"); + + migrationBuilder.CreateIndex( + name: "IX_LoginSessions_UserId_RevokedAt_ExpiresAt", + table: "LoginSessions", + columns: new[] { "UserId", "RevokedAt", "ExpiresAt" }); + + migrationBuilder.CreateIndex( + name: "IX_Profiles_UserId_Name", + table: "Profiles", + columns: new[] { "UserId", "Name" }, + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_Users_Username", + table: "Users", + column: "Username", + unique: true); + + migrationBuilder.AddForeignKey( + name: "FK_ChatConversations_Profiles_ProfileId", + table: "ChatConversations", + column: "ProfileId", + principalTable: "Profiles", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + + migrationBuilder.AddForeignKey( + name: "FK_PlaybackPreferences_Profiles_UserId", + table: "PlaybackPreferences", + column: "UserId", + principalTable: "Profiles", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + + migrationBuilder.AddForeignKey( + name: "FK_PlaybackProgresses_Profiles_UserId", + table: "PlaybackProgresses", + column: "UserId", + principalTable: "Profiles", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + + migrationBuilder.AddForeignKey( + name: "FK_WebDavTokens_Users_UserId", + table: "WebDavTokens", + column: "UserId", + principalTable: "Users", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + // The previous schema can represent only the untouched legacy household. Refuse + // downgrade before dropping any column when doing so would merge profile history, + // lose account credentials, widen a device root, revive a revoked token, or remove + // an expiry. PostgreSQL runs the migration transactionally, so this leaves the + // current schema and all data intact. + migrationBuilder.Sql( + """ + DO $$ + BEGIN + IF EXISTS ( + SELECT 1 FROM "Users" + WHERE "Id" <> '00000000-0000-0000-0000-000000000001' + OR "Username" <> 'admin' + OR "PasswordHash" IS NOT NULL + OR "Role" <> 'Admin' + OR "IsDisabled") + OR EXISTS ( + SELECT 1 FROM "Profiles" + WHERE "Id" <> '00000000-0000-0000-0000-000000000000' + OR "UserId" <> '00000000-0000-0000-0000-000000000001' + OR "Name" <> 'Home' + OR "Avatar" IS NOT NULL + OR "PinHash" IS NOT NULL + OR NOT "IsDefault") + OR EXISTS ( + SELECT 1 FROM "PlaybackProgresses" + WHERE "UserId" <> '00000000-0000-0000-0000-000000000000') + OR EXISTS ( + SELECT 1 FROM "PlaybackPreferences" + WHERE "UserId" <> '00000000-0000-0000-0000-000000000000') + OR EXISTS ( + SELECT 1 FROM "ChatConversations" + WHERE "ProfileId" <> '00000000-0000-0000-0000-000000000000') + OR EXISTS ( + SELECT 1 FROM "WebDavTokens" + WHERE "UserId" <> '00000000-0000-0000-0000-000000000001' + OR "Scope" <> 'read' + OR "VirtualRoot" <> '/' + OR "ExpiresAt" IS NOT NULL + OR "RevokedAt" IS NOT NULL) + THEN + RAISE EXCEPTION USING + ERRCODE = 'P0001', + MESSAGE = 'Cannot downgrade household identity safely: the old schema cannot represent current users, profiles, history, or device-token restrictions.'; + END IF; + END $$; + """); + + migrationBuilder.DropForeignKey( + name: "FK_ChatConversations_Profiles_ProfileId", + table: "ChatConversations"); + + migrationBuilder.DropForeignKey( + name: "FK_PlaybackPreferences_Profiles_UserId", + table: "PlaybackPreferences"); + + migrationBuilder.DropForeignKey( + name: "FK_PlaybackProgresses_Profiles_UserId", + table: "PlaybackProgresses"); + + migrationBuilder.DropForeignKey( + name: "FK_WebDavTokens_Users_UserId", + table: "WebDavTokens"); + + migrationBuilder.DropTable( + name: "LoginSessions"); + + migrationBuilder.DropTable( + name: "Profiles"); + + migrationBuilder.DropTable( + name: "Users"); + + migrationBuilder.DropIndex( + name: "IX_WebDavTokens_UserId", + table: "WebDavTokens"); + + migrationBuilder.DropIndex( + name: "IX_ChatConversations_ProfileId_UpdatedAt", + table: "ChatConversations"); + + migrationBuilder.DropColumn( + name: "ExpiresAt", + table: "WebDavTokens"); + + migrationBuilder.DropColumn( + name: "RevokedAt", + table: "WebDavTokens"); + + migrationBuilder.DropColumn( + name: "Scope", + table: "WebDavTokens"); + + migrationBuilder.DropColumn( + name: "UserId", + table: "WebDavTokens"); + + migrationBuilder.DropColumn( + name: "VirtualRoot", + table: "WebDavTokens"); + + migrationBuilder.DropColumn( + name: "ProfileId", + table: "ChatConversations"); + } + } +} diff --git a/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs b/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs index 8126b9e..756c4cc 100644 --- a/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs +++ b/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs @@ -25,7 +25,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) modelBuilder.Entity("SecondDimensionWatcherReDive.Models.Animation", b => { b.Property("Id") - .ValueGeneratedOnAdd() .HasColumnType("uuid"); b.Property("Name") @@ -273,6 +272,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("CreatedAt") .HasColumnType("timestamp with time zone"); + b.Property("ProfileId") + .HasColumnType("uuid"); + b.Property("Title") .HasColumnType("text"); @@ -281,6 +283,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasKey("Id"); + b.HasIndex("ProfileId", "UpdatedAt"); + b.ToTable("ChatConversations"); }); @@ -465,6 +469,51 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("Incidents"); }); + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.LoginSession", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("ActiveProfileId") + .HasColumnType("uuid"); + + b.Property("AuthenticatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("DeviceName") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("ExpiresAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastSeenAt") + .HasColumnType("timestamp with time zone"); + + b.Property("RefreshTokenHash") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("RevokedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("ActiveProfileId"); + + b.HasIndex("UserId", "RevokedAt", "ExpiresAt"); + + b.ToTable("LoginSessions"); + }); + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MediaLibrarySource", b => { b.Property("Id") @@ -674,7 +723,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) modelBuilder.Entity("SecondDimensionWatcherReDive.Models.PlaybackPreference", b => { b.Property("UserId") - .ValueGeneratedOnAdd() .HasColumnType("uuid"); b.Property("AudioLanguage") @@ -832,6 +880,81 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("SubscriptionAutomationPolicies"); }); + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.UserAccount", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("IsDisabled") + .HasColumnType("boolean"); + + b.Property("PasswordHash") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("Role") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Username") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.HasKey("Id"); + + b.HasIndex("Username") + .IsUnique(); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.UserProfile", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("Avatar") + .HasMaxLength(512) + .HasColumnType("character varying(512)"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("IsDefault") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("PinHash") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("UserId", "Name") + .IsUnique(); + + b.ToTable("Profiles"); + }); + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.WebDavToken", b => { b.Property("Id") @@ -844,16 +967,37 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Description") .HasColumnType("text"); + b.Property("ExpiresAt") + .HasColumnType("timestamp with time zone"); + + b.Property("RevokedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Scope") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + b.Property("TokenHash") .IsRequired() .HasColumnType("text"); + b.Property("UserId") + .HasColumnType("uuid"); + b.Property("Username") .IsRequired() .HasColumnType("text"); + b.Property("VirtualRoot") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + b.HasKey("Id"); + b.HasIndex("UserId"); + b.HasIndex("Username") .IsUnique(); @@ -896,6 +1040,17 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Navigation("SeasonBangumi"); }); + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ChatConversation", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.UserProfile", "Profile") + .WithMany() + .HasForeignKey("ProfileId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Profile"); + }); + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ChatMessage", b => { b.HasOne("SecondDimensionWatcherReDive.Models.ChatConversation", "Conversation") @@ -916,6 +1071,25 @@ protected override void BuildModel(ModelBuilder modelBuilder) .IsRequired(); }); + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.LoginSession", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.UserProfile", "ActiveProfile") + .WithMany() + .HasForeignKey("ActiveProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("SecondDimensionWatcherReDive.Models.UserAccount", "User") + .WithMany("Sessions") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ActiveProfile"); + + b.Navigation("User"); + }); + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewMappingSnapshot", b => { b.HasOne("SecondDimensionWatcherReDive.Models.MetadataReviewOperation", "Operation") @@ -938,6 +1112,17 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Navigation("AnimationInfo"); }); + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.PlaybackPreference", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.UserProfile", "Profile") + .WithOne() + .HasForeignKey("SecondDimensionWatcherReDive.Models.PlaybackPreference", "UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Profile"); + }); + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.PlaybackProgress", b => { b.HasOne("SecondDimensionWatcherReDive.Models.AnimationInfo", "AnimationInfo") @@ -946,7 +1131,15 @@ protected override void BuildModel(ModelBuilder modelBuilder) .OnDelete(DeleteBehavior.Cascade) .IsRequired(); + b.HasOne("SecondDimensionWatcherReDive.Models.UserProfile", "Profile") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + b.Navigation("AnimationInfo"); + + b.Navigation("Profile"); }); modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SubscriptionAutomationPolicy", b => @@ -960,6 +1153,28 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Navigation("Feed"); }); + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.UserProfile", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.UserAccount", "User") + .WithMany("Profiles") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("User"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.WebDavToken", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.UserAccount", "User") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("User"); + }); + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ChatConversation", b => { b.Navigation("Messages"); @@ -974,6 +1189,13 @@ protected override void BuildModel(ModelBuilder modelBuilder) { b.Navigation("Subgroups"); }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.UserAccount", b => + { + b.Navigation("Profiles"); + + b.Navigation("Sessions"); + }); #pragma warning restore 612, 618 } } diff --git a/SecondDimensionWatcherReDive/Models/ApplicationContext.cs b/SecondDimensionWatcherReDive/Models/ApplicationContext.cs index 59764ac..178422a 100644 --- a/SecondDimensionWatcherReDive/Models/ApplicationContext.cs +++ b/SecondDimensionWatcherReDive/Models/ApplicationContext.cs @@ -33,9 +33,78 @@ public ApplicationContext(DbContextOptions options) public DbSet PlaybackPreferences { get; set; } public DbSet MediaLibrarySources { get; set; } public DbSet ApplicationSettings { get; set; } + public DbSet Users { get; set; } + public DbSet Profiles { get; set; } + public DbSet LoginSessions { get; set; } protected override void OnModelCreating(ModelBuilder modelBuilder) { + modelBuilder.Entity() + .HasIndex(user => user.Username) + .IsUnique(); + + modelBuilder.Entity() + .Property(user => user.Username) + .HasMaxLength(64); + + modelBuilder.Entity() + .Property(user => user.PasswordHash) + .HasMaxLength(128); + + modelBuilder.Entity() + .Property(user => user.Role) + .HasConversion() + .HasMaxLength(16); + + modelBuilder.Entity() + .Property(profile => profile.Id) + .ValueGeneratedNever(); + + modelBuilder.Entity() + .Property(profile => profile.Name) + .HasMaxLength(64); + + modelBuilder.Entity() + .Property(profile => profile.Avatar) + .HasMaxLength(512); + + modelBuilder.Entity() + .Property(profile => profile.PinHash) + .HasMaxLength(128); + + modelBuilder.Entity() + .HasIndex(profile => new { profile.UserId, profile.Name }) + .IsUnique(); + + modelBuilder.Entity() + .HasOne(profile => profile.User) + .WithMany(user => user.Profiles) + .HasForeignKey(profile => profile.UserId) + .OnDelete(DeleteBehavior.Cascade); + + modelBuilder.Entity() + .Property(session => session.RefreshTokenHash) + .HasMaxLength(64); + + modelBuilder.Entity() + .Property(session => session.DeviceName) + .HasMaxLength(128); + + modelBuilder.Entity() + .HasIndex(session => new { session.UserId, session.RevokedAt, session.ExpiresAt }); + + modelBuilder.Entity() + .HasOne(session => session.User) + .WithMany(user => user.Sessions) + .HasForeignKey(session => session.UserId) + .OnDelete(DeleteBehavior.Cascade); + + modelBuilder.Entity() + .HasOne(session => session.ActiveProfile) + .WithMany() + .HasForeignKey(session => session.ActiveProfileId) + .OnDelete(DeleteBehavior.Restrict); + modelBuilder.Entity() .Property(settings => settings.Id) .ValueGeneratedNever(); @@ -171,6 +240,20 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) .HasIndex(t => t.Username) .IsUnique(); + modelBuilder.Entity() + .Property(token => token.Scope) + .HasMaxLength(32); + + modelBuilder.Entity() + .Property(token => token.VirtualRoot) + .HasMaxLength(2048); + + modelBuilder.Entity() + .HasOne(token => token.User) + .WithMany() + .HasForeignKey(token => token.UserId) + .OnDelete(DeleteBehavior.Cascade); + modelBuilder.Entity() .HasIndex(progress => new { @@ -193,6 +276,12 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) .HasForeignKey(progress => progress.AnimationInfoId) .OnDelete(DeleteBehavior.Cascade); + modelBuilder.Entity() + .HasOne(progress => progress.Profile) + .WithMany() + .HasForeignKey(progress => progress.UserId) + .OnDelete(DeleteBehavior.Cascade); + modelBuilder.Entity() .ToTable(table => { @@ -223,6 +312,12 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) .Property(preference => preference.AudioTrackLabel) .HasMaxLength(128); + modelBuilder.Entity() + .HasOne(preference => preference.Profile) + .WithOne() + .HasForeignKey(preference => preference.UserId) + .OnDelete(DeleteBehavior.Cascade); + modelBuilder.Entity() .HasIndex(b => b.MikanId) .IsUnique(); @@ -294,5 +389,14 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) .WithMany(c => c.Messages) .HasForeignKey(m => m.ConversationId) .OnDelete(DeleteBehavior.Cascade); + + modelBuilder.Entity() + .HasIndex(conversation => new { conversation.ProfileId, conversation.UpdatedAt }); + + modelBuilder.Entity() + .HasOne(conversation => conversation.Profile) + .WithMany() + .HasForeignKey(conversation => conversation.ProfileId) + .OnDelete(DeleteBehavior.Cascade); } } diff --git a/SecondDimensionWatcherReDive/Models/ChatConversation.cs b/SecondDimensionWatcherReDive/Models/ChatConversation.cs index 07cd5a9..f0e262f 100644 --- a/SecondDimensionWatcherReDive/Models/ChatConversation.cs +++ b/SecondDimensionWatcherReDive/Models/ChatConversation.cs @@ -3,6 +3,8 @@ namespace SecondDimensionWatcherReDive.Models; public class ChatConversation { public Guid Id { get; set; } + public Guid ProfileId { get; set; } + public UserProfile Profile { get; set; } = null!; public string? Title { get; set; } public DateTimeOffset CreatedAt { get; set; } public DateTimeOffset UpdatedAt { get; set; } diff --git a/SecondDimensionWatcherReDive/Models/LoginSession.cs b/SecondDimensionWatcherReDive/Models/LoginSession.cs new file mode 100644 index 0000000..c8aa129 --- /dev/null +++ b/SecondDimensionWatcherReDive/Models/LoginSession.cs @@ -0,0 +1,17 @@ +namespace SecondDimensionWatcherReDive.Models; + +public sealed class LoginSession +{ + public Guid Id { get; set; } + public Guid UserId { get; set; } + public UserAccount User { get; set; } = null!; + public Guid ActiveProfileId { get; set; } + public UserProfile ActiveProfile { get; set; } = null!; + public string RefreshTokenHash { get; set; } = string.Empty; + public string? DeviceName { get; set; } + public DateTimeOffset AuthenticatedAt { get; set; } + public DateTimeOffset CreatedAt { get; set; } + public DateTimeOffset LastSeenAt { get; set; } + public DateTimeOffset ExpiresAt { get; set; } + public DateTimeOffset? RevokedAt { get; set; } +} diff --git a/SecondDimensionWatcherReDive/Models/PlaybackPreference.cs b/SecondDimensionWatcherReDive/Models/PlaybackPreference.cs index 4381da2..9785b77 100644 --- a/SecondDimensionWatcherReDive/Models/PlaybackPreference.cs +++ b/SecondDimensionWatcherReDive/Models/PlaybackPreference.cs @@ -4,6 +4,8 @@ public class PlaybackPreference { public Guid UserId { get; set; } + public UserProfile? Profile { get; set; } + public string? SubtitleLanguage { get; set; } public string? SubtitleTrackLabel { get; set; } diff --git a/SecondDimensionWatcherReDive/Models/PlaybackProgress.cs b/SecondDimensionWatcherReDive/Models/PlaybackProgress.cs index 219ea48..ac573db 100644 --- a/SecondDimensionWatcherReDive/Models/PlaybackProgress.cs +++ b/SecondDimensionWatcherReDive/Models/PlaybackProgress.cs @@ -6,6 +6,8 @@ public class PlaybackProgress public Guid UserId { get; set; } + public UserProfile? Profile { get; set; } + public Guid AnimationInfoId { get; set; } public AnimationInfo? AnimationInfo { get; set; } diff --git a/SecondDimensionWatcherReDive/Models/UserAccount.cs b/SecondDimensionWatcherReDive/Models/UserAccount.cs new file mode 100644 index 0000000..4eed36a --- /dev/null +++ b/SecondDimensionWatcherReDive/Models/UserAccount.cs @@ -0,0 +1,16 @@ +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Models; + +public sealed class UserAccount +{ + public Guid Id { get; set; } + public string Username { get; set; } = string.Empty; + public string? PasswordHash { get; set; } + public UserRole Role { get; set; } + public bool IsDisabled { get; set; } + public DateTimeOffset CreatedAt { get; set; } + public DateTimeOffset UpdatedAt { get; set; } + public ICollection Profiles { get; set; } = []; + public ICollection Sessions { get; set; } = []; +} diff --git a/SecondDimensionWatcherReDive/Models/UserProfile.cs b/SecondDimensionWatcherReDive/Models/UserProfile.cs new file mode 100644 index 0000000..02674ad --- /dev/null +++ b/SecondDimensionWatcherReDive/Models/UserProfile.cs @@ -0,0 +1,14 @@ +namespace SecondDimensionWatcherReDive.Models; + +public sealed class UserProfile +{ + public Guid Id { get; set; } + public Guid UserId { get; set; } + public UserAccount User { get; set; } = null!; + public string Name { get; set; } = string.Empty; + public string? Avatar { get; set; } + public string? PinHash { get; set; } + public bool IsDefault { get; set; } + public DateTimeOffset CreatedAt { get; set; } + public DateTimeOffset UpdatedAt { get; set; } +} diff --git a/SecondDimensionWatcherReDive/Models/WebDavToken.cs b/SecondDimensionWatcherReDive/Models/WebDavToken.cs index bf88ac0..b284153 100644 --- a/SecondDimensionWatcherReDive/Models/WebDavToken.cs +++ b/SecondDimensionWatcherReDive/Models/WebDavToken.cs @@ -4,6 +4,10 @@ public class WebDavToken { public Guid Id { get; set; } + public Guid UserId { get; set; } + + public UserAccount User { get; set; } = null!; + public string Username { get; set; } = string.Empty; public string TokenHash { get; set; } = string.Empty; @@ -11,4 +15,12 @@ public class WebDavToken public string? Description { get; set; } public DateTimeOffset CreatedAt { get; set; } + + public string Scope { get; set; } = "read"; + + public string VirtualRoot { get; set; } = "/"; + + public DateTimeOffset? ExpiresAt { get; set; } + + public DateTimeOffset? RevokedAt { get; set; } } diff --git a/SecondDimensionWatcherReDive/Program.cs b/SecondDimensionWatcherReDive/Program.cs index 80f5f19..63e1e63 100644 --- a/SecondDimensionWatcherReDive/Program.cs +++ b/SecondDimensionWatcherReDive/Program.cs @@ -19,6 +19,7 @@ using SecondDimensionWatcherReDive.Framework.FileDownload; using SecondDimensionWatcherReDive.Framework.FileStore; using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Framework.Authorization; using SecondDimensionWatcherReDive.Framework.Tasks; using SecondDimensionWatcherReDive.Inference.AI; using SecondDimensionWatcherReDive.Models; @@ -120,7 +121,7 @@ ValidateIssuer = false, ValidateAudience = false, ValidateLifetime = true, - RequireExpirationTime = false + RequireExpirationTime = true }; builder.Services.AddSingleton(tokenValidationParams); @@ -134,9 +135,76 @@ { options.SaveToken = true; options.TokenValidationParameters = tokenValidationParams; + options.Events = new JwtBearerEvents + { + OnTokenValidated = async context => + { + var principal = context.Principal; + if (principal is null + || !principal.TryGetUserId(out var userId) + || !principal.TryGetProfileId(out var profileId) + || !principal.TryGetSessionId(out var sessionId)) + { + context.Fail("The access token has no valid session identity."); + return; + } + + var repository = context.HttpContext.RequestServices + .GetRequiredService(); + var authenticated = await repository.GetAuthenticatedSessionAsync( + sessionId, + DateTimeOffset.UtcNow, + context.HttpContext.RequestAborted); + var authenticatedAt = principal.FindFirst( + IdentityClaimTypes.AuthenticatedAt)?.Value; + if (authenticated is null + || authenticated.User.Id != userId + || authenticated.Profile.Id != profileId + || !string.Equals( + principal.FindFirst(System.Security.Claims.ClaimTypes.Role)?.Value, + authenticated.User.Role.ToString(), + StringComparison.Ordinal) + || authenticatedAt != authenticated.Session.AuthenticatedAt + .ToUnixTimeSeconds() + .ToString(System.Globalization.CultureInfo.InvariantCulture)) + { + context.Fail("The login session is no longer active."); + } + } + }; }).AddScheme( BasicAuthenticationHandler.SchemeName, _ => { }); +builder.Services.AddAuthorization(options => +{ + options.AddPolicy(AccessPolicies.ContentWrite, + policy => policy.RequireRole(nameof(UserRole.Admin), nameof(UserRole.Member))); + options.AddPolicy(AccessPolicies.PlaybackWrite, + policy => policy.RequireRole(nameof(UserRole.Admin), nameof(UserRole.Member))); + options.AddPolicy(AccessPolicies.ChatWrite, + policy => policy.RequireRole(nameof(UserRole.Admin), nameof(UserRole.Member))); + options.AddPolicy(AccessPolicies.Administrator, + policy => policy.RequireRole(nameof(UserRole.Admin))); + static bool HasRecentAuthentication(System.Security.Claims.ClaimsPrincipal principal) + { + if (!long.TryParse( + principal.FindFirst(IdentityClaimTypes.AuthenticatedAt)?.Value, + out var unixSeconds)) + return false; + var authenticatedAt = DateTimeOffset.FromUnixTimeSeconds(unixSeconds); + var age = DateTimeOffset.UtcNow - authenticatedAt; + return age >= TimeSpan.FromMinutes(-1) && age <= TimeSpan.FromMinutes(5); + } + + options.AddPolicy(AccessPolicies.RecentAuthentication, + policy => policy.RequireAssertion(context => HasRecentAuthentication(context.User))); + options.AddPolicy(AccessPolicies.RecentAdministrator, policy => + { + policy.RequireRole(nameof(UserRole.Admin)); + policy.RequireAssertion(context => HasRecentAuthentication(context.User)); + }); +}); + //Add distributed cache (Valkey / Redis or in-memory fallback) var valkeyConnection = builder.Configuration["Valkey:ConnectionString"]; if (!string.IsNullOrEmpty(valkeyConnection)) @@ -275,6 +343,8 @@ builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddSingleton(); @@ -351,6 +421,7 @@ app.MapFallbackToFile("index.html"); } +app.UseAuthentication(); app.UseAuthorization(); if (app.Configuration.GetValue("DisableCors") is true) app.UseCors("all"); diff --git a/SecondDimensionWatcherReDive/Repositories/ChatRepository.cs b/SecondDimensionWatcherReDive/Repositories/ChatRepository.cs index 337a5f7..4144ecd 100644 --- a/SecondDimensionWatcherReDive/Repositories/ChatRepository.cs +++ b/SecondDimensionWatcherReDive/Repositories/ChatRepository.cs @@ -7,21 +7,27 @@ namespace SecondDimensionWatcherReDive.Repositories; public class ChatRepository(ApplicationContext context) : IChatRepository { public async Task> GetConversationsAsync( + Guid profileId, CancellationToken cancellationToken) { return await context.ChatConversations .AsNoTracking() + .Where(c => c.ProfileId == profileId) .OrderByDescending(c => c.UpdatedAt) .Select(c => new ChatConversationSummary(c.Id, c.Title, c.CreatedAt, c.UpdatedAt)) .ToListAsync(cancellationToken); } public async Task GetConversationWithMessagesAsync( - Guid id, CancellationToken cancellationToken) + Guid id, + Guid profileId, + CancellationToken cancellationToken) { var conversation = await context.ChatConversations .AsNoTracking() - .FirstOrDefaultAsync(c => c.Id == id, cancellationToken); + .FirstOrDefaultAsync( + c => c.Id == id && c.ProfileId == profileId, + cancellationToken); if (conversation is null) return null; @@ -40,12 +46,15 @@ public async Task> GetConversationsAsync( } public async Task CreateConversationAsync( - string? title, CancellationToken cancellationToken) + Guid profileId, + string? title, + CancellationToken cancellationToken) { var now = DateTimeOffset.Now; var entity = new ChatConversation { Id = Guid.NewGuid(), + ProfileId = profileId, Title = title, CreatedAt = now, UpdatedAt = now @@ -57,9 +66,14 @@ public async Task CreateConversationAsync( return new ChatConversationSummary(entity.Id, entity.Title, entity.CreatedAt, entity.UpdatedAt); } - public async Task DeleteConversationAsync(Guid id, CancellationToken cancellationToken) + public async Task DeleteConversationAsync( + Guid id, + Guid profileId, + CancellationToken cancellationToken) { - var entity = await context.ChatConversations.FindAsync([id], cancellationToken); + var entity = await context.ChatConversations.FirstOrDefaultAsync( + conversation => conversation.Id == id && conversation.ProfileId == profileId, + cancellationToken); if (entity is null) return false; context.ChatConversations.Remove(entity); @@ -68,9 +82,14 @@ public async Task DeleteConversationAsync(Guid id, CancellationToken cance } public async Task UpdateConversationTitleAsync( - Guid id, string title, CancellationToken cancellationToken) + Guid id, + Guid profileId, + string title, + CancellationToken cancellationToken) { - var entity = await context.ChatConversations.FindAsync([id], cancellationToken); + var entity = await context.ChatConversations.FirstOrDefaultAsync( + conversation => conversation.Id == id && conversation.ProfileId == profileId, + cancellationToken); if (entity is null) return; entity.Title = title; @@ -79,8 +98,17 @@ public async Task UpdateConversationTitleAsync( } public async Task AddMessageAsync( - Guid conversationId, ChatMessageRecord message, CancellationToken cancellationToken) + Guid conversationId, + Guid profileId, + ChatMessageRecord message, + CancellationToken cancellationToken) { + var conversation = await context.ChatConversations.FirstOrDefaultAsync( + candidate => candidate.Id == conversationId + && candidate.ProfileId == profileId, + cancellationToken); + if (conversation is null) return; + var entity = new ChatMessage { Id = message.Id, @@ -97,16 +125,23 @@ public async Task AddMessageAsync( context.ChatMessages.Add(entity); // Update conversation timestamp - var conversation = await context.ChatConversations.FindAsync([conversationId], cancellationToken); - if (conversation is not null) - conversation.UpdatedAt = DateTimeOffset.Now; + conversation.UpdatedAt = DateTimeOffset.Now; await context.SaveChangesAsync(cancellationToken); } public async Task AddMessagesAsync( - Guid conversationId, IEnumerable messages, CancellationToken cancellationToken) + Guid conversationId, + Guid profileId, + IEnumerable messages, + CancellationToken cancellationToken) { + var conversation = await context.ChatConversations.FirstOrDefaultAsync( + candidate => candidate.Id == conversationId + && candidate.ProfileId == profileId, + cancellationToken); + if (conversation is null) return; + foreach (var message in messages) { context.ChatMessages.Add(new ChatMessage @@ -123,19 +158,20 @@ public async Task AddMessagesAsync( }); } - var conversation = await context.ChatConversations.FindAsync([conversationId], cancellationToken); - if (conversation is not null) - conversation.UpdatedAt = DateTimeOffset.Now; + conversation.UpdatedAt = DateTimeOffset.Now; await context.SaveChangesAsync(cancellationToken); } public async Task> GetMessagesAsync( - Guid conversationId, CancellationToken cancellationToken) + Guid conversationId, + Guid profileId, + CancellationToken cancellationToken) { return await context.ChatMessages .AsNoTracking() - .Where(m => m.ConversationId == conversationId) + .Where(m => m.ConversationId == conversationId + && m.Conversation.ProfileId == profileId) .OrderBy(m => m.Order) .Select(m => new ChatMessageRecord( m.Id, m.Role, m.Content, m.ToolCallsJson, @@ -144,9 +180,14 @@ public async Task> GetMessagesAsync( } public async Task GetMessageCountAsync( - Guid conversationId, CancellationToken cancellationToken) + Guid conversationId, + Guid profileId, + CancellationToken cancellationToken) { return await context.ChatMessages - .CountAsync(m => m.ConversationId == conversationId, cancellationToken); + .CountAsync( + m => m.ConversationId == conversationId + && m.Conversation.ProfileId == profileId, + cancellationToken); } } diff --git a/SecondDimensionWatcherReDive/Repositories/IdentityRepository.cs b/SecondDimensionWatcherReDive/Repositories/IdentityRepository.cs new file mode 100644 index 0000000..9b61063 --- /dev/null +++ b/SecondDimensionWatcherReDive/Repositories/IdentityRepository.cs @@ -0,0 +1,381 @@ +using System.Data; +using Microsoft.EntityFrameworkCore; +using Npgsql; +using SecondDimensionWatcherReDive.Framework.DataRepository; +using ProfileEntity = SecondDimensionWatcherReDive.Models.UserProfile; +using SessionEntity = SecondDimensionWatcherReDive.Models.LoginSession; +using UserEntity = SecondDimensionWatcherReDive.Models.UserAccount; + +namespace SecondDimensionWatcherReDive.Repositories; + +public sealed class IdentityRepository(Models.ApplicationContext context) : IIdentityRepository +{ + public Task AnyUsersAsync(CancellationToken cancellationToken) => + context.Users.AnyAsync(cancellationToken); + + public async Task FindUserByIdAsync( + Guid id, + CancellationToken cancellationToken) => + (await context.Users.AsNoTracking() + .FirstOrDefaultAsync(user => user.Id == id, cancellationToken))?.ToRecord(); + + public async Task FindUserByUsernameAsync( + string username, + CancellationToken cancellationToken) + { + var normalized = username.Trim().ToLowerInvariant(); + return (await context.Users.AsNoTracking() + .FirstOrDefaultAsync(user => user.Username == normalized, cancellationToken))?.ToRecord(); + } + + public async Task FindProfileAsync( + Guid id, + CancellationToken cancellationToken) => + (await context.Profiles.AsNoTracking() + .FirstOrDefaultAsync(profile => profile.Id == id, cancellationToken))?.ToRecord(); + + public async Task> GetProfilesAsync( + Guid userId, + CancellationToken cancellationToken) => + (await context.Profiles.AsNoTracking() + .Where(profile => profile.UserId == userId) + .OrderByDescending(profile => profile.IsDefault) + .ThenBy(profile => profile.Name) + .ToListAsync(cancellationToken)) + .Select(profile => profile.ToRecord()) + .ToList(); + + public async Task CreateUserWithProfileAsync( + UserAccount user, + UserProfile profile, + CancellationToken cancellationToken) + { + context.Users.Add(user.ToEntity()); + context.Profiles.Add(profile.ToEntity()); + // A single SaveChanges call is transactionally atomic and is executed by + // Npgsql's configured retry strategy. An explicit user transaction here + // would be rejected by EnableRetryOnFailure in production. + try + { + await context.SaveChangesAsync(cancellationToken); + } + catch (Exception exception) when (IsUniqueViolation(exception)) + { + context.ChangeTracker.Clear(); + throw new IdentityConflictException( + "A user or profile with the same identity already exists.", exception); + } + return new UserAccountWithProfiles(user, [profile]); + } + + public async Task SetPasswordHashAsync( + Guid userId, + string passwordHash, + DateTimeOffset now, + CancellationToken cancellationToken) + { + var affected = await context.Users + .Where(user => user.Id == userId && !user.IsDisabled) + .ExecuteUpdateAsync(setters => setters + .SetProperty(user => user.PasswordHash, passwordHash) + .SetProperty(user => user.UpdatedAt, now), cancellationToken); + return affected == 1; + } + + public async Task AddProfileAsync( + UserProfile profile, + CancellationToken cancellationToken) + { + context.Profiles.Add(profile.ToEntity()); + try + { + await context.SaveChangesAsync(cancellationToken); + } + catch (Exception exception) when (IsUniqueViolation(exception)) + { + context.ChangeTracker.Clear(); + throw new IdentityConflictException( + "A profile with the same name already exists for this user.", exception); + } + return profile; + } + + public async Task UpdateProfileAsync( + Guid profileId, + Guid userId, + string name, + string? avatar, + string? pinHash, + bool replacePin, + DateTimeOffset now, + CancellationToken cancellationToken) + { + int affected; + try + { + affected = await context.Profiles + .Where(profile => profile.Id == profileId && profile.UserId == userId) + .ExecuteUpdateAsync(setters => setters + .SetProperty(profile => profile.Name, name) + .SetProperty(profile => profile.Avatar, avatar) + .SetProperty(profile => profile.PinHash, + profile => replacePin ? pinHash : profile.PinHash) + .SetProperty(profile => profile.UpdatedAt, now), cancellationToken); + } + catch (Exception exception) when (IsUniqueViolation(exception)) + { + throw new IdentityConflictException( + "A profile with the same name already exists for this user.", exception); + } + return affected == 1; + } + + public async Task> GetUsersAsync( + CancellationToken cancellationToken) + { + var users = await context.Users.AsNoTracking() + .OrderBy(user => user.Username) + .ToListAsync(cancellationToken); + var profiles = await context.Profiles.AsNoTracking() + .OrderByDescending(profile => profile.IsDefault) + .ThenBy(profile => profile.Name) + .ToListAsync(cancellationToken); + var byUser = profiles.ToLookup(profile => profile.UserId); + return users.Select(user => new UserAccountWithProfiles( + user.ToRecord(), + byUser[user.Id].Select(profile => profile.ToRecord()).ToList())) + .ToList(); + } + + public async Task UpdateUserAccessAsync( + Guid userId, + UserRole role, + bool isDisabled, + DateTimeOffset now, + CancellationToken cancellationToken) + { + var strategy = context.Database.CreateExecutionStrategy(); + return await strategy.ExecuteAsync(async () => + { + await using var transaction = await context.Database.BeginTransactionAsync( + IsolationLevel.ReadCommitted, cancellationToken); + // Serialize the household-admin invariant across independent users. A row lock on + // only the target cannot prevent two administrators from demoting each other. + await context.Database.ExecuteSqlRawAsync( + "SELECT pg_advisory_xact_lock(600000000000000017)", + cancellationToken); + + var target = await context.Users.FirstOrDefaultAsync( + user => user.Id == userId, cancellationToken); + if (target is null) + { + await transaction.RollbackAsync(cancellationToken); + return UpdateUserAccessResult.NotFound; + } + + if (target.Role == UserRole.Admin + && !target.IsDisabled + && (role != UserRole.Admin || isDisabled) + && !await context.Users.AnyAsync( + user => user.Id != userId + && user.Role == UserRole.Admin + && !user.IsDisabled, + cancellationToken)) + { + await transaction.RollbackAsync(cancellationToken); + return UpdateUserAccessResult.LastAdministrator; + } + + var accessChanged = target.Role != role || target.IsDisabled != isDisabled; + target.Role = role; + target.IsDisabled = isDisabled; + target.UpdatedAt = now; + await context.SaveChangesAsync(cancellationToken); + if (accessChanged) + { + await context.LoginSessions + .Where(session => session.UserId == userId && session.RevokedAt == null) + .ExecuteUpdateAsync(setters => setters + .SetProperty(session => session.RevokedAt, now), cancellationToken); + } + + await transaction.CommitAsync(cancellationToken); + return UpdateUserAccessResult.Updated; + }); + } + + public async Task AddSessionAsync( + UserSession session, + CancellationToken cancellationToken) + { + context.LoginSessions.Add(session.ToEntity()); + await context.SaveChangesAsync(cancellationToken); + } + + public async Task GetAuthenticatedSessionAsync( + Guid sessionId, + DateTimeOffset now, + CancellationToken cancellationToken) + { + var entity = await context.LoginSessions.AsNoTracking() + .Include(session => session.User) + .Include(session => session.ActiveProfile) + .FirstOrDefaultAsync(session => session.Id == sessionId + && session.RevokedAt == null + && session.ExpiresAt > now + && !session.User.IsDisabled, + cancellationToken); + if (entity is null || entity.ActiveProfile.UserId != entity.UserId) + return null; + return new AuthenticatedSession( + entity.User.ToRecord(), + entity.ActiveProfile.ToRecord(), + entity.ToRecord()); + } + + public async Task TryRotateSessionAsync( + Guid sessionId, + string expectedRefreshTokenHash, + string newRefreshTokenHash, + Guid activeProfileId, + DateTimeOffset? authenticatedAt, + DateTimeOffset now, + DateTimeOffset expiresAt, + CancellationToken cancellationToken) + { + var affected = await context.LoginSessions + .Where(session => session.Id == sessionId + && session.RefreshTokenHash == expectedRefreshTokenHash + && session.RevokedAt == null + && session.ExpiresAt > now + && context.Profiles.Any(profile => + profile.Id == activeProfileId + && profile.UserId == session.UserId)) + .ExecuteUpdateAsync(setters => setters + .SetProperty(session => session.RefreshTokenHash, newRefreshTokenHash) + .SetProperty(session => session.ActiveProfileId, activeProfileId) + .SetProperty(session => session.AuthenticatedAt, + session => authenticatedAt ?? session.AuthenticatedAt) + .SetProperty(session => session.LastSeenAt, now) + .SetProperty(session => session.ExpiresAt, expiresAt), cancellationToken); + return affected == 1; + } + + public async Task> GetSessionsAsync( + Guid? userId, + CancellationToken cancellationToken) + { + var query = context.LoginSessions.AsNoTracking().AsQueryable(); + if (userId.HasValue) + query = query.Where(session => session.UserId == userId.Value); + return await query + .OrderByDescending(session => session.LastSeenAt) + .Select(session => new UserSessionSummary( + new UserSession( + session.Id, + session.UserId, + session.ActiveProfileId, + string.Empty, + session.DeviceName, + session.AuthenticatedAt, + session.CreatedAt, + session.LastSeenAt, + session.ExpiresAt, + session.RevokedAt), + session.User.Username, + session.ActiveProfile.Name)) + .ToListAsync(cancellationToken); + } + + public async Task RevokeSessionAsync( + Guid sessionId, + Guid? requiredUserId, + DateTimeOffset revokedAt, + CancellationToken cancellationToken) + { + var affected = await context.LoginSessions + .Where(session => session.Id == sessionId + && session.RevokedAt == null + && (!requiredUserId.HasValue + || session.UserId == requiredUserId.Value)) + .ExecuteUpdateAsync(setters => setters + .SetProperty(session => session.RevokedAt, revokedAt), cancellationToken); + return affected == 1; + } + + private static bool IsUniqueViolation(Exception exception) => + exception is PostgresException { SqlState: PostgresErrorCodes.UniqueViolation } + || exception.InnerException is not null && IsUniqueViolation(exception.InnerException); + +} + +internal static class IdentityRepositoryConverter +{ + internal static UserAccount ToRecord(this UserEntity entity) => new( + entity.Id, + entity.Username, + entity.PasswordHash, + entity.Role, + entity.IsDisabled, + entity.CreatedAt, + entity.UpdatedAt); + + internal static UserProfile ToRecord(this ProfileEntity entity) => new( + entity.Id, + entity.UserId, + entity.Name, + entity.Avatar, + entity.PinHash, + entity.IsDefault, + entity.CreatedAt, + entity.UpdatedAt); + + internal static UserSession ToRecord(this SessionEntity entity) => new( + entity.Id, + entity.UserId, + entity.ActiveProfileId, + entity.RefreshTokenHash, + entity.DeviceName, + entity.AuthenticatedAt, + entity.CreatedAt, + entity.LastSeenAt, + entity.ExpiresAt, + entity.RevokedAt); + + internal static UserEntity ToEntity(this UserAccount record) => new() + { + Id = record.Id, + Username = record.Username, + PasswordHash = record.PasswordHash, + Role = record.Role, + IsDisabled = record.IsDisabled, + CreatedAt = record.CreatedAt, + UpdatedAt = record.UpdatedAt + }; + + internal static ProfileEntity ToEntity(this UserProfile record) => new() + { + Id = record.Id, + UserId = record.UserId, + Name = record.Name, + Avatar = record.Avatar, + PinHash = record.PinHash, + IsDefault = record.IsDefault, + CreatedAt = record.CreatedAt, + UpdatedAt = record.UpdatedAt + }; + + internal static SessionEntity ToEntity(this UserSession record) => new() + { + Id = record.Id, + UserId = record.UserId, + ActiveProfileId = record.ActiveProfileId, + RefreshTokenHash = record.RefreshTokenHash, + DeviceName = record.DeviceName, + AuthenticatedAt = record.AuthenticatedAt, + CreatedAt = record.CreatedAt, + LastSeenAt = record.LastSeenAt, + ExpiresAt = record.ExpiresAt, + RevokedAt = record.RevokedAt + }; +} diff --git a/SecondDimensionWatcherReDive/Repositories/RepositoryConverter.cs b/SecondDimensionWatcherReDive/Repositories/RepositoryConverter.cs index 260e342..45417c2 100644 --- a/SecondDimensionWatcherReDive/Repositories/RepositoryConverter.cs +++ b/SecondDimensionWatcherReDive/Repositories/RepositoryConverter.cs @@ -107,10 +107,15 @@ public static DataRepo.SubscriptionAutomationPolicy ToRecord( public static DataRepo.WebDavToken ToRecord(this Models.WebDavToken entity) => new(entity.Id, + entity.UserId, entity.Username, entity.TokenHash, entity.Description, - entity.CreatedAt); + entity.CreatedAt, + entity.Scope, + entity.VirtualRoot, + entity.ExpiresAt, + entity.RevokedAt); public static DataRepo.PlaybackProgress ToRecord(this Models.PlaybackProgress entity) => new(entity.Id, @@ -259,10 +264,15 @@ public static Models.WebDavToken ToEntity(this DataRepo.WebDavToken record) => new() { Id = record.Id, + UserId = record.UserId, Username = record.Username, TokenHash = record.TokenHash, Description = record.Description, - CreatedAt = record.CreatedAt + CreatedAt = record.CreatedAt, + Scope = record.Scope, + VirtualRoot = record.VirtualRoot, + ExpiresAt = record.ExpiresAt, + RevokedAt = record.RevokedAt }; public static Models.PlaybackProgress ToEntity(this DataRepo.PlaybackProgress record) => diff --git a/SecondDimensionWatcherReDive/Repositories/WebDavTokenRepository.cs b/SecondDimensionWatcherReDive/Repositories/WebDavTokenRepository.cs index 39e06a4..8e23db6 100644 --- a/SecondDimensionWatcherReDive/Repositories/WebDavTokenRepository.cs +++ b/SecondDimensionWatcherReDive/Repositories/WebDavTokenRepository.cs @@ -33,12 +33,16 @@ public async Task AddAsync(WebDavToken token, CancellationToken cancellationToke await context.SaveChangesAsync(cancellationToken); } - public async Task RemoveByIdAsync(Guid id, CancellationToken cancellationToken) + public async Task RevokeByIdAsync( + Guid id, + DateTimeOffset revokedAt, + CancellationToken cancellationToken) { var entity = await context.WebDavTokens.FindAsync([id], cancellationToken); if (entity is null) return false; - context.WebDavTokens.Remove(entity); + if (entity.RevokedAt is null) + entity.RevokedAt = revokedAt; await context.SaveChangesAsync(cancellationToken); return true; } From 6366329ed7852ba20c5f94f3542e0020c55b5717 Mon Sep 17 00:00:00 2001 From: mahoshojoHCG Date: Sun, 30 Aug 2026 01:24:24 +0800 Subject: [PATCH 06/45] feat: add controlled plugin platform --- .../mock-server.mjs | 514 ++++-- .../settings/PluginSettingsSection.tsx | 403 +++++ .../settings/SettingsNavigation.tsx | 11 +- .../src/i18n/locales/en/settings.json | 49 +- .../src/i18n/locales/ja/settings.json | 49 +- .../src/i18n/locales/zh-CN/settings.json | 49 +- .../src/pages/SettingsPage.tsx | 3 + .../src/plugins/api.ts | 49 + .../src/plugins/hooks.ts | 5 + .../src/plugins/types.ts | 52 + .../IPluginCatalogRepository.cs | 31 + .../Plugin/IJavaScriptPluginLoader.cs | 18 +- .../Plugin/INotificationProvider.cs | 16 + .../Plugin/IPluginServices.cs | 4 - .../Plugin/PluginManifest.cs | 160 ++ .../Plugins/PluginApiTests.cs | 149 ++ .../WebDavWebApplicationFactory.cs | 3 + .../PluginControllerTests.cs | 33 + .../PluginDeploymentTests.cs | 17 + .../PluginEventTests.cs | 41 + .../PluginPlatformIntegrationTests.cs | 1382 +++++++++++++++++ .../SecondDimensionWatcherReDive.Test.csproj | 9 + .../Controllers/Converter.cs | 74 + .../External/AppJsonSerializerContext.cs | 8 + .../Controllers/External/PluginModels.cs | 78 + .../Controllers/PluginController.cs | 133 ++ .../Plugin/PluginEvent.cs | 47 +- .../Plugin/PluginHelper.cs | 4 +- .../Plugin/PluginServices.cs | 4 +- .../PluginPlatform/IPluginCapabilityBroker.cs | 13 + .../PluginPlatform/IPluginManager.cs | 26 + .../PluginPlatform/PluginCapabilityBroker.cs | 324 ++++ .../PluginLifecycleCoordinator.cs | 133 ++ .../PluginPlatform/PluginLifecycleJournal.cs | 32 + .../PluginPlatform/PluginManager.cs | 989 ++++++++++++ .../PluginPlatform/PluginManifestValidator.cs | 333 ++++ .../PluginNetworkConnectionFactory.cs | 119 ++ .../PluginPlatform/PluginPackageInspector.cs | 414 +++++ .../PluginPlatform/PluginPlatformOptions.cs | 27 + .../PluginPlatformServiceExtensions.cs | 71 + .../PluginPlatform/PluginProcessExecutor.cs | 291 ++++ .../PluginPlatform/PluginProviderRegistry.cs | 116 ++ .../PluginPlatform/PluginSafeFileAccess.cs | 438 ++++++ .../PluginPlatform/PluginWorkerHost.cs | 154 ++ .../PluginPlatform/PluginWorkerProtocol.cs | 34 + SecondDimensionWatcherReDive/Program.cs | 10 + .../Repositories/PluginCatalogRepository.cs | 191 +++ .../Utils/FileStore/FileStoreProvider.cs | 22 +- .../appsettings.example.json | 24 + deployments/podman-compose.yml | 1 + docs/container-deployment.md | 5 +- docs/plugin-platform.md | 40 + examples/plugins/scoped-storage/index.js | 35 + examples/plugins/scoped-storage/manifest.json | 36 + examples/plugins/webhook/index.js | 16 + examples/plugins/webhook/manifest.json | 31 + packaging/appsettings.yml | 23 + sdk/javascript/README.md | 13 + sdk/javascript/plugin-api.d.ts | 39 + 59 files changed, 7282 insertions(+), 113 deletions(-) create mode 100644 SecondDimensionWatcherReDive.Client/src/components/settings/PluginSettingsSection.tsx create mode 100644 SecondDimensionWatcherReDive.Client/src/plugins/api.ts create mode 100644 SecondDimensionWatcherReDive.Client/src/plugins/hooks.ts create mode 100644 SecondDimensionWatcherReDive.Client/src/plugins/types.ts create mode 100644 SecondDimensionWatcherReDive.Framework/DataRepository/IPluginCatalogRepository.cs create mode 100644 SecondDimensionWatcherReDive.Framework/Plugin/INotificationProvider.cs create mode 100644 SecondDimensionWatcherReDive.Framework/Plugin/PluginManifest.cs create mode 100644 SecondDimensionWatcherReDive.IntegrationTest/Plugins/PluginApiTests.cs create mode 100644 SecondDimensionWatcherReDive.Test/PluginControllerTests.cs create mode 100644 SecondDimensionWatcherReDive.Test/PluginDeploymentTests.cs create mode 100644 SecondDimensionWatcherReDive.Test/PluginPlatformIntegrationTests.cs create mode 100644 SecondDimensionWatcherReDive/Controllers/External/PluginModels.cs create mode 100644 SecondDimensionWatcherReDive/Controllers/PluginController.cs create mode 100644 SecondDimensionWatcherReDive/PluginPlatform/IPluginCapabilityBroker.cs create mode 100644 SecondDimensionWatcherReDive/PluginPlatform/IPluginManager.cs create mode 100644 SecondDimensionWatcherReDive/PluginPlatform/PluginCapabilityBroker.cs create mode 100644 SecondDimensionWatcherReDive/PluginPlatform/PluginLifecycleCoordinator.cs create mode 100644 SecondDimensionWatcherReDive/PluginPlatform/PluginLifecycleJournal.cs create mode 100644 SecondDimensionWatcherReDive/PluginPlatform/PluginManager.cs create mode 100644 SecondDimensionWatcherReDive/PluginPlatform/PluginManifestValidator.cs create mode 100644 SecondDimensionWatcherReDive/PluginPlatform/PluginNetworkConnectionFactory.cs create mode 100644 SecondDimensionWatcherReDive/PluginPlatform/PluginPackageInspector.cs create mode 100644 SecondDimensionWatcherReDive/PluginPlatform/PluginPlatformOptions.cs create mode 100644 SecondDimensionWatcherReDive/PluginPlatform/PluginPlatformServiceExtensions.cs create mode 100644 SecondDimensionWatcherReDive/PluginPlatform/PluginProcessExecutor.cs create mode 100644 SecondDimensionWatcherReDive/PluginPlatform/PluginProviderRegistry.cs create mode 100644 SecondDimensionWatcherReDive/PluginPlatform/PluginSafeFileAccess.cs create mode 100644 SecondDimensionWatcherReDive/PluginPlatform/PluginWorkerHost.cs create mode 100644 SecondDimensionWatcherReDive/PluginPlatform/PluginWorkerProtocol.cs create mode 100644 SecondDimensionWatcherReDive/Repositories/PluginCatalogRepository.cs create mode 100644 docs/plugin-platform.md create mode 100644 examples/plugins/scoped-storage/index.js create mode 100644 examples/plugins/scoped-storage/manifest.json create mode 100644 examples/plugins/webhook/index.js create mode 100644 examples/plugins/webhook/manifest.json create mode 100644 sdk/javascript/README.md create mode 100644 sdk/javascript/plugin-api.d.ts diff --git a/SecondDimensionWatcherReDive.Client/mock-server.mjs b/SecondDimensionWatcherReDive.Client/mock-server.mjs index f5ee4d4..b3d583a 100644 --- a/SecondDimensionWatcherReDive.Client/mock-server.mjs +++ b/SecondDimensionWatcherReDive.Client/mock-server.mjs @@ -692,50 +692,147 @@ let feeds = [ // Per-feed subscription automation policies and historical releases. const POLICY_CREATED_AT = new Date(Date.now() - 86400_000).toISOString(); const subscriptionPolicies = new Map([ - [feeds[0].id, { - feedId: feeds[0].id, - subtitleGroups: ["LoliHouse", "喵萌奶茶屋"], - resolutions: ["1080p"], - codecs: ["HEVC"], - languages: ["简中", "繁中"], - minSizeBytes: 300 * 1024 * 1024, - maxSizeBytes: 1600 * 1024 * 1024, - excludedKeywords: ["合集", "NCOP"], - mode: "ManualConfirm", - createdAt: POLICY_CREATED_AT, - updatedAt: new Date(Date.now() - 3600_000 * 8).toISOString(), - }], - [feeds[1].id, { - feedId: feeds[1].id, - subtitleGroups: ["ANi", "SubsPlease"], - resolutions: ["1080p"], - codecs: [], - languages: ["繁中"], - minSizeBytes: null, - maxSizeBytes: 1400 * 1024 * 1024, - excludedKeywords: ["预告"], - mode: "AutoDownload", - createdAt: POLICY_CREATED_AT, - updatedAt: new Date(Date.now() - 3600_000 * 3).toISOString(), - }], + [ + feeds[0].id, + { + feedId: feeds[0].id, + subtitleGroups: ["LoliHouse", "喵萌奶茶屋"], + resolutions: ["1080p"], + codecs: ["HEVC"], + languages: ["简中", "繁中"], + minSizeBytes: 300 * 1024 * 1024, + maxSizeBytes: 1600 * 1024 * 1024, + excludedKeywords: ["合集", "NCOP"], + mode: "ManualConfirm", + createdAt: POLICY_CREATED_AT, + updatedAt: new Date(Date.now() - 3600_000 * 8).toISOString(), + }, + ], + [ + feeds[1].id, + { + feedId: feeds[1].id, + subtitleGroups: ["ANi", "SubsPlease"], + resolutions: ["1080p"], + codecs: [], + languages: ["繁中"], + minSizeBytes: null, + maxSizeBytes: 1400 * 1024 * 1024, + excludedKeywords: ["预告"], + mode: "AutoDownload", + createdAt: POLICY_CREATED_AT, + updatedAt: new Date(Date.now() - 3600_000 * 3).toISOString(), + }, + ], ]); const RELEASE_HISTORY_BY_FEED = new Map([ - [feeds[0].id, [ - { id: randomUUID(), title: "[LoliHouse] 葬送的芙莉莲 - 28 [WebRip 1080p HEVC-10bit AAC][简繁内封]", publishedAt: new Date(Date.now() - 3600_000 * 5).toISOString(), sizeBytes: 824 * 1024 * 1024, subtitleGroup: "LoliHouse", resolution: "1080p", codec: "HEVC", languages: ["简中", "繁中"] }, - { id: randomUUID(), title: "[ANi] 葬送的芙莉莲 - 28 [1080P][繁日双语]", publishedAt: new Date(Date.now() - 3600_000 * 12).toISOString(), sizeBytes: 516 * 1024 * 1024, subtitleGroup: "ANi", resolution: "1080p", codec: "AVC", languages: ["繁中", "日语"] }, - { id: randomUUID(), title: "[喵萌奶茶屋] 葬送的芙莉莲 01-28 合集 [1080p HEVC][简繁]", publishedAt: new Date(Date.now() - 86400_000).toISOString(), sizeBytes: 18.4 * 1024 * 1024 * 1024, subtitleGroup: "喵萌奶茶屋", resolution: "1080p", codec: "HEVC", languages: ["简中", "繁中"] }, - { id: randomUUID(), title: "[LoliHouse] 葬送的芙莉莲 - 27 [2160p HEVC][简繁]", publishedAt: new Date(Date.now() - 86400_000 * 2).toISOString(), sizeBytes: 2250 * 1024 * 1024, subtitleGroup: "LoliHouse", resolution: "2160p", codec: "HEVC", languages: ["简中", "繁中"] }, - ]], - [feeds[1].id, [ - { id: randomUUID(), title: "[ANi] 迷宫饭 - 24 [1080P][繁日双语]", publishedAt: new Date(Date.now() - 3600_000 * 7).toISOString(), sizeBytes: 612 * 1024 * 1024, subtitleGroup: "ANi", resolution: "1080p", codec: "AVC", languages: ["繁中", "日语"] }, - { id: randomUUID(), title: "[SubsPlease] Dungeon Meshi - 24 (1080p) [English]", publishedAt: new Date(Date.now() - 3600_000 * 18).toISOString(), sizeBytes: 1380 * 1024 * 1024, subtitleGroup: "SubsPlease", resolution: "1080p", codec: "AVC", languages: ["English"] }, - { id: randomUUID(), title: "[ANi] 迷宫饭 完结纪念预告 [1080P][繁中]", publishedAt: new Date(Date.now() - 86400_000 * 2).toISOString(), sizeBytes: 92 * 1024 * 1024, subtitleGroup: "ANi", resolution: "1080p", codec: "AVC", languages: ["繁中"] }, - ]], - [feeds[2].id, [ - { id: randomUUID(), title: "[LoliHouse] 药屋少女的呢喃 - 24 [WebRip 1080p HEVC][简繁]", publishedAt: new Date(Date.now() - 3600_000 * 10).toISOString(), sizeBytes: 745 * 1024 * 1024, subtitleGroup: "LoliHouse", resolution: "1080p", codec: "HEVC", languages: ["简中", "繁中"] }, - { id: randomUUID(), title: "[ANi] 药屋少女的呢喃 - 24 [720P][繁中]", publishedAt: new Date(Date.now() - 86400_000).toISOString(), sizeBytes: 324 * 1024 * 1024, subtitleGroup: "ANi", resolution: "720p", codec: "AVC", languages: ["繁中"] }, - ]], + [ + feeds[0].id, + [ + { + id: randomUUID(), + title: + "[LoliHouse] 葬送的芙莉莲 - 28 [WebRip 1080p HEVC-10bit AAC][简繁内封]", + publishedAt: new Date(Date.now() - 3600_000 * 5).toISOString(), + sizeBytes: 824 * 1024 * 1024, + subtitleGroup: "LoliHouse", + resolution: "1080p", + codec: "HEVC", + languages: ["简中", "繁中"], + }, + { + id: randomUUID(), + title: "[ANi] 葬送的芙莉莲 - 28 [1080P][繁日双语]", + publishedAt: new Date(Date.now() - 3600_000 * 12).toISOString(), + sizeBytes: 516 * 1024 * 1024, + subtitleGroup: "ANi", + resolution: "1080p", + codec: "AVC", + languages: ["繁中", "日语"], + }, + { + id: randomUUID(), + title: "[喵萌奶茶屋] 葬送的芙莉莲 01-28 合集 [1080p HEVC][简繁]", + publishedAt: new Date(Date.now() - 86400_000).toISOString(), + sizeBytes: 18.4 * 1024 * 1024 * 1024, + subtitleGroup: "喵萌奶茶屋", + resolution: "1080p", + codec: "HEVC", + languages: ["简中", "繁中"], + }, + { + id: randomUUID(), + title: "[LoliHouse] 葬送的芙莉莲 - 27 [2160p HEVC][简繁]", + publishedAt: new Date(Date.now() - 86400_000 * 2).toISOString(), + sizeBytes: 2250 * 1024 * 1024, + subtitleGroup: "LoliHouse", + resolution: "2160p", + codec: "HEVC", + languages: ["简中", "繁中"], + }, + ], + ], + [ + feeds[1].id, + [ + { + id: randomUUID(), + title: "[ANi] 迷宫饭 - 24 [1080P][繁日双语]", + publishedAt: new Date(Date.now() - 3600_000 * 7).toISOString(), + sizeBytes: 612 * 1024 * 1024, + subtitleGroup: "ANi", + resolution: "1080p", + codec: "AVC", + languages: ["繁中", "日语"], + }, + { + id: randomUUID(), + title: "[SubsPlease] Dungeon Meshi - 24 (1080p) [English]", + publishedAt: new Date(Date.now() - 3600_000 * 18).toISOString(), + sizeBytes: 1380 * 1024 * 1024, + subtitleGroup: "SubsPlease", + resolution: "1080p", + codec: "AVC", + languages: ["English"], + }, + { + id: randomUUID(), + title: "[ANi] 迷宫饭 完结纪念预告 [1080P][繁中]", + publishedAt: new Date(Date.now() - 86400_000 * 2).toISOString(), + sizeBytes: 92 * 1024 * 1024, + subtitleGroup: "ANi", + resolution: "1080p", + codec: "AVC", + languages: ["繁中"], + }, + ], + ], + [ + feeds[2].id, + [ + { + id: randomUUID(), + title: "[LoliHouse] 药屋少女的呢喃 - 24 [WebRip 1080p HEVC][简繁]", + publishedAt: new Date(Date.now() - 3600_000 * 10).toISOString(), + sizeBytes: 745 * 1024 * 1024, + subtitleGroup: "LoliHouse", + resolution: "1080p", + codec: "HEVC", + languages: ["简中", "繁中"], + }, + { + id: randomUUID(), + title: "[ANi] 药屋少女的呢喃 - 24 [720P][繁中]", + publishedAt: new Date(Date.now() - 86400_000).toISOString(), + sizeBytes: 324 * 1024 * 1024, + subtitleGroup: "ANi", + resolution: "720p", + codec: "AVC", + languages: ["繁中"], + }, + ], + ], ]); function simulatePolicy(feedId, policy) { @@ -755,28 +852,59 @@ function simulatePolicy(feedId, policy) { if (field === "resolution") { normalized = normalized.replace(/\s/g, ""); const aliases = { - "4K": "2160P", UHD: "2160P", "2160": "2160P", - "1440": "1440P", FHD: "1080P", "1080": "1080P", - HD: "720P", "720": "720P", "576": "576P", "480": "480P", + "4K": "2160P", + UHD: "2160P", + 2160: "2160P", + 1440: "1440P", + FHD: "1080P", + 1080: "1080P", + HD: "720P", + 720: "720P", + 576: "576P", + 480: "480P", }; return aliases[normalized] ?? normalized; } if (field === "codec") { normalized = normalized.replace(/[.\-\s]/g, ""); const aliases = { - H265: "HEVC", X265: "HEVC", H264: "AVC", X264: "AVC", + H265: "HEVC", + X265: "HEVC", + H264: "AVC", + X264: "AVC", }; return aliases[normalized] ?? normalized; } if (field === "languages") { normalized = normalized.replace(/[_\-\s]/g, ""); const aliases = { - CHS: "ZHHANS", SC: "ZHHANS", GB: "ZHHANS", ZHCN: "ZHHANS", - "简体": "ZHHANS", "简中": "ZHHANS", "簡中": "ZHHANS", "简体中文": "ZHHANS", - CHT: "ZHHANT", TC: "ZHHANT", BIG5: "ZHHANT", ZHTW: "ZHHANT", ZHHK: "ZHHANT", - "繁体": "ZHHANT", "繁體": "ZHHANT", "繁中": "ZHHANT", "繁體中文": "ZHHANT", - JPN: "JA", JAP: "JA", "日语": "JA", "日語": "JA", "日本語": "JA", JAPANESE: "JA", - ENG: "EN", "英语": "EN", "英語": "EN", ENGLISH: "EN", + CHS: "ZHHANS", + SC: "ZHHANS", + GB: "ZHHANS", + ZHCN: "ZHHANS", + 简体: "ZHHANS", + 简中: "ZHHANS", + 簡中: "ZHHANS", + 简体中文: "ZHHANS", + CHT: "ZHHANT", + TC: "ZHHANT", + BIG5: "ZHHANT", + ZHTW: "ZHHANT", + ZHHK: "ZHHANT", + 繁体: "ZHHANT", + 繁體: "ZHHANT", + 繁中: "ZHHANT", + 繁體中文: "ZHHANT", + JPN: "JA", + JAP: "JA", + 日语: "JA", + 日語: "JA", + 日本語: "JA", + JAPANESE: "JA", + ENG: "EN", + 英语: "EN", + 英語: "EN", + ENGLISH: "EN", }; return aliases[normalized] ?? normalized; } @@ -786,7 +914,13 @@ function simulatePolicy(feedId, policy) { const actual = actualValues.filter(Boolean); const expected = (expectedValues ?? []).filter(Boolean); if (expected.length === 0) { - return { field, passed: true, actual: actual.join(", ") || null, expected: null, message: "anyValueAllowed" }; + return { + field, + passed: true, + actual: actual.join(", ") || null, + expected: null, + message: "anyValueAllowed", + }; } const normalizedExpected = new Set( expected.map((value) => normalizeAllowedValue(field, value)), @@ -794,28 +928,47 @@ function simulatePolicy(feedId, policy) { const passed = actual.some((value) => normalizedExpected.has(normalizeAllowedValue(field, value)), ); - return { field, passed, actual: actual.join(", ") || null, expected: expected.join(", "), message: passed ? "allowedValueMatched" : "allowedValueMissed" }; + return { + field, + passed, + actual: actual.join(", ") || null, + expected: expected.join(", "), + message: passed ? "allowedValueMatched" : "allowedValueMissed", + }; }; const entries = history.map((item) => { const explanations = [ - checkAllowed("subtitleGroup", [item.subtitleGroup], policy.subtitleGroups), + checkAllowed( + "subtitleGroup", + [item.subtitleGroup], + policy.subtitleGroups, + ), checkAllowed("resolution", [item.resolution], policy.resolutions), checkAllowed("codec", [item.codec], policy.codecs), checkAllowed("languages", item.languages, policy.languages), ]; - const min = typeof policy.minSizeBytes === "number" ? policy.minSizeBytes : null; - const max = typeof policy.maxSizeBytes === "number" ? policy.maxSizeBytes : null; - const sizePassed = (min == null || item.sizeBytes >= min) && (max == null || item.sizeBytes <= max); + const min = + typeof policy.minSizeBytes === "number" ? policy.minSizeBytes : null; + const max = + typeof policy.maxSizeBytes === "number" ? policy.maxSizeBytes : null; + const sizePassed = + (min == null || item.sizeBytes >= min) && + (max == null || item.sizeBytes <= max); explanations.push({ field: "size", passed: sizePassed, actual: formatBytes(item.sizeBytes), - expected: min == null && max == null ? null : `${min == null ? "0 B" : formatBytes(min)} – ${max == null ? "∞" : formatBytes(max)}`, + expected: + min == null && max == null + ? null + : `${min == null ? "0 B" : formatBytes(min)} – ${max == null ? "∞" : formatBytes(max)}`, message: sizePassed ? "withinSizeRange" : "outsideSizeRange", }); const excluded = (policy.excludedKeywords ?? []).filter(Boolean); - const found = excluded.find((keyword) => item.title.toLowerCase().includes(keyword.toLowerCase())); + const found = excluded.find((keyword) => + item.title.toLowerCase().includes(keyword.toLowerCase()), + ); explanations.push({ field: "excludedKeywords", passed: !found, @@ -823,10 +976,21 @@ function simulatePolicy(feedId, policy) { expected: excluded.length > 0 ? excluded.join(", ") : null, message: found ? "excludedKeywordFound" : "noExcludedKeyword", }); - return { id: item.id, title: item.title, publishedAt: item.publishedAt, sizeBytes: item.sizeBytes, matched: explanations.every((reason) => reason.passed), explanations }; + return { + id: item.id, + title: item.title, + publishedAt: item.publishedAt, + sizeBytes: item.sizeBytes, + matched: explanations.every((reason) => reason.passed), + explanations, + }; }); - return { total: entries.length, matched: entries.filter((entry) => entry.matched).length, entries }; + return { + total: entries.length, + matched: entries.filter((entry) => entry.matched).length, + entries, + }; } // WebDAV access tokens @@ -903,6 +1067,60 @@ let systemSettings = { }, }; +const mockPluginManifest = { + id: "example.webhook", + name: "Webhook notifications", + version: "1.0.0", + apiVersion: "1.0", + entryPoint: "index.js", + description: "Mock notification provider for the controlled plugin UI.", + dependencies: [], + capabilities: { + networkDomains: ["hooks.example.com"], + fileRoots: [], + notifications: true, + downloadControl: false, + storageAccess: false, + backgroundTasks: false, + }, + platforms: ["any"], + fileSha256: { + "index.js": + "cb04e27dacbadf8de122b491b2c2d32cb553564fcc9c390a3ab4d922a2cd0e1b", + }, + signaturePublisher: null, + signatureAlgorithm: null, + providers: [ + { + kind: "notification", + name: "webhook", + handlers: { send: "sendNotification" }, + }, + ], + dataVersion: 1, + dataMigration: null, +}; + +let mockPlugins = []; + +function mockInstalledPlugin(manifest = mockPluginManifest) { + return { + manifest, + isEnabled: false, + approvedCapabilities: manifest.capabilities, + compatibilityErrors: [], + health: { + status: "healthy", + consecutiveFailures: 0, + lastSuccessAt: null, + lastFailureAt: null, + lastError: null, + circuitOpenUntil: null, + }, + hasConfiguration: false, + }; +} + const deploymentSecrets = { openAi: { isConfigured: true, source: "deployment" }, anthropic: { isConfigured: false, source: "none" }, @@ -1052,7 +1270,9 @@ function playablePaths() { !entry.isDirectory && /\.(mkv|mp4|webm|avi|flv|wmv|mov|m4v|ts|m2ts)$/i.test(entry.fileName) ) { - paths.push(directory ? `${directory}/${entry.fileName}` : entry.fileName); + paths.push( + directory ? `${directory}/${entry.fileName}` : entry.fileName, + ); } } } @@ -1122,7 +1342,9 @@ function associatedSubtitles(animation, videoPath) { entry.fileName.toLowerCase().startsWith(stem.toLowerCase()), ) .map((entry) => { - const path = directory ? `${directory}/${entry.fileName}` : entry.fileName; + const path = directory + ? `${directory}/${entry.fileName}` + : entry.fileName; const language = entry.fileName.includes("zh-Hans") ? "zh-Hans" : entry.fileName.includes(".en.") @@ -1153,7 +1375,8 @@ if (finishedForPlayback[0]) { }); } const previousEpisode = finishedForPlayback.find( - (animation) => animation.animation?.tmdbId === "209867" && animation.episode === 27, + (animation) => + animation.animation?.tmdbId === "209867" && animation.episode === 27, ); if (previousEpisode) { const path = "Season 1/EP01.mp4"; @@ -1172,7 +1395,8 @@ let mockIncidents = [ type: "feedFailure", severity: "error", title: "Mikan RSS returned HTTP 503", - detail: "The feed could not be refreshed during the last three sync attempts.", + detail: + "The feed could not be refreshed during the last three sync attempts.", sourceId: feeds[0]?.id ?? null, detectedAt: new Date(Date.now() - 42 * 60_000).toISOString(), updatedAt: new Date(Date.now() - 12 * 60_000).toISOString(), @@ -1187,7 +1411,8 @@ let mockIncidents = [ type: "downloadStalled", severity: "warning", title: "Download has not progressed for 20 minutes", - detail: "No peers are currently available. Retry will reannounce the torrent.", + detail: + "No peers are currently available. Retry will reannounce the torrent.", sourceId: finishedForPlayback[1]?.id ?? null, detectedAt: new Date(Date.now() - 25 * 60_000).toISOString(), updatedAt: new Date(Date.now() - 5 * 60_000).toISOString(), @@ -1217,7 +1442,8 @@ let mockIncidents = [ type: "fileMappingFailure", severity: "error", title: "Downloaded files could not be mapped", - detail: "The download completed, but no playable video mapping was produced.", + detail: + "The download completed, but no playable video mapping was produced.", sourceId: finishedForPlayback[2]?.id ?? null, detectedAt: new Date(Date.now() - 2 * 3600_000).toISOString(), updatedAt: new Date(Date.now() - 2 * 3600_000).toISOString(), @@ -1509,6 +1735,79 @@ async function route(method, pathname, searchParams, req, res) { } } + // --- Controlled plugins --- + + if (method === "GET" && pathname === "/api/plugins") { + return json(res, mockPlugins); + } + + if (method === "POST" && pathname === "/api/plugins/preview") { + await readBody(req); + return json(res, { + token: randomBytes(24).toString("hex"), + packageSha256: + "7d9fc7ef8ef86ffa9c4965f05b3e715b5f4ef872af326ed8bc4761a0003f71d3", + manifest: mockPluginManifest, + compatibilityErrors: [], + isSignatureTrusted: false, + signatureStatus: "Package is unsigned (mock development mode).", + expiresAt: new Date(Date.now() + 30 * 60_000).toISOString(), + }); + } + + if (method === "POST" && pathname === "/api/plugins/preview-remote") { + return json( + res, + { + code: "remote_install_disabled", + message: "Remote JavaScript installation is disabled.", + }, + 403, + ); + } + + if ( + method === "POST" && + (pathname === "/api/plugins/install" || + /^\/api\/plugins\/[^/]+\/upgrade$/.test(pathname)) + ) { + await readBody(req); + mockPlugins = [ + ...mockPlugins.filter( + (plugin) => plugin.manifest.id !== mockPluginManifest.id, + ), + mockInstalledPlugin(), + ]; + return json(res, { + id: mockPluginManifest.id, + version: mockPluginManifest.version, + isUpgrade: pathname.endsWith("/upgrade"), + compatibilityErrors: [], + }); + } + + { + const match = pathname.match(/^\/api\/plugins\/([^/]+)\/(enable|disable)$/); + if (match && method === "POST") { + const plugin = mockPlugins.find( + (candidate) => candidate.manifest.id === decodeURIComponent(match[1]), + ); + if (!plugin) return json(res, { code: "plugin_not_found" }, 404); + plugin.isEnabled = match[2] === "enable"; + return empty(res); + } + } + + { + const match = pathname.match(/^\/api\/plugins\/([^/]+)$/); + if (match && method === "DELETE") { + mockPlugins = mockPlugins.filter( + (plugin) => plugin.manifest.id !== decodeURIComponent(match[1]), + ); + return empty(res); + } + } + // --- Playback continuity --- if (method === "GET" && pathname === "/api/playback/continue") { @@ -1555,7 +1854,11 @@ async function route(method, pathname, searchParams, req, res) { if (method === "GET" && pathname === "/api/playback/context") { const animation = animations.get(searchParams.get("animationInfoId")); const path = searchParams.get("path"); - if (!animation || !animation.isDownloadFinished || !playablePaths().includes(path)) { + if ( + !animation || + !animation.isDownloadFinished || + !playablePaths().includes(path) + ) { return empty(res, 404); } return json(res, { @@ -1576,7 +1879,8 @@ async function route(method, pathname, searchParams, req, res) { if (method === "PUT" && pathname === "/api/playback/progress") { const body = await readBody(req); const animation = animations.get(body.animationInfoId); - if (!animation || !playablePaths().includes(body.path)) return empty(res, 404); + if (!animation || !playablePaths().includes(body.path)) + return empty(res, 404); const positionSeconds = Math.max(0, Number(body.positionSeconds) || 0); const durationSeconds = Math.max(0, Number(body.durationSeconds) || 0); const key = playbackKey(animation.id, body.path); @@ -1586,7 +1890,10 @@ async function route(method, pathname, searchParams, req, res) { (durationSeconds > 0 && positionSeconds / durationSeconds >= 0.9); const updatedAt = new Date().toISOString(); const stored = { - positionSeconds: Math.min(positionSeconds, durationSeconds || positionSeconds), + positionSeconds: Math.min( + positionSeconds, + durationSeconds || positionSeconds, + ), durationSeconds, isWatched, updatedAt, @@ -1599,7 +1906,8 @@ async function route(method, pathname, searchParams, req, res) { if (method === "PUT" && pathname === "/api/playback/watched") { const body = await readBody(req); const animation = animations.get(body.animationInfoId); - if (!animation || !playablePaths().includes(body.path)) return empty(res, 404); + if (!animation || !playablePaths().includes(body.path)) + return empty(res, 404); const key = playbackKey(animation.id, body.path); const previous = playbackProgress.get(key) ?? { positionSeconds: 0, @@ -1638,7 +1946,10 @@ async function route(method, pathname, searchParams, req, res) { if (method === "GET" && pathname === "/api/incidents") { const type = searchParams.get("type"); const includeResolved = searchParams.get("includeResolved") === "true"; - const skip = Math.max(0, parseInt(searchParams.get("skip") ?? "0", 10) || 0); + const skip = Math.max( + 0, + parseInt(searchParams.get("skip") ?? "0", 10) || 0, + ); const take = Math.min( 200, Math.max(1, parseInt(searchParams.get("take") ?? "50", 10) || 50), @@ -1680,7 +1991,8 @@ async function route(method, pathname, searchParams, req, res) { incident.lastRetryError = null; incident.canRetry = false; } else { - incident.lastRetryError = "Free space is still below the configured threshold"; + incident.lastRetryError = + "Free space is still below the configured threshold"; } results.push({ incidentId: incident.id, @@ -1701,15 +2013,21 @@ async function route(method, pathname, searchParams, req, res) { if (method === "POST" && match) { const incident = mockIncidents.find((item) => item.id === match[1]); if (!incident) return empty(res, 404); - if (incident.resolvedAt) return json(res, { error: "Already resolved" }, 409); + if (incident.resolvedAt) + return json(res, { error: "Already resolved" }, 409); incident.retryCount += 1; incident.lastRetryAt = new Date().toISOString(); incident.updatedAt = incident.lastRetryAt; if (incident.type === "diskSpaceLow") { - incident.lastRetryError = "Free space is still below the configured threshold"; + incident.lastRetryError = + "Free space is still below the configured threshold"; return json( res, - { incidentId: incident.id, success: false, error: incident.lastRetryError }, + { + incidentId: incident.id, + success: false, + error: incident.lastRetryError, + }, 422, ); } @@ -2026,8 +2344,7 @@ async function route(method, pathname, searchParams, req, res) { g.episodes.sort( (a, b) => new Date(b.publishTime).getTime() - - new Date(a.publishTime).getTime() || - b.id.localeCompare(a.id), + new Date(a.publishTime).getTime() || b.id.localeCompare(a.id), ); g.episodeCount = g.episodes.length; return g; @@ -2324,11 +2641,15 @@ async function route(method, pathname, searchParams, req, res) { // POST /api/subscription-policies/:feedId/simulate { - const m = pathname.match(/^\/api\/subscription-policies\/([^/]+)\/simulate$/); + const m = pathname.match( + /^\/api\/subscription-policies\/([^/]+)\/simulate$/, + ); if (method === "POST" && m) { const feedId = decodeURIComponent(m[1]); if (!feeds.some((feed) => feed.id === feedId)) return empty(res, 404); - return readBody(req).then((body) => json(res, simulatePolicy(feedId, body))); + return readBody(req).then((body) => + json(res, simulatePolicy(feedId, body)), + ); } } @@ -2349,18 +2670,34 @@ async function route(method, pathname, searchParams, req, res) { const existing = subscriptionPolicies.get(feedId); const policy = { feedId, - subtitleGroups: Array.isArray(body.subtitleGroups) ? body.subtitleGroups : [], - resolutions: Array.isArray(body.resolutions) ? body.resolutions : [], + subtitleGroups: Array.isArray(body.subtitleGroups) + ? body.subtitleGroups + : [], + resolutions: Array.isArray(body.resolutions) + ? body.resolutions + : [], codecs: Array.isArray(body.codecs) ? body.codecs : [], languages: Array.isArray(body.languages) ? body.languages : [], - minSizeBytes: typeof body.minSizeBytes === "number" ? body.minSizeBytes : null, - maxSizeBytes: typeof body.maxSizeBytes === "number" ? body.maxSizeBytes : null, - excludedKeywords: Array.isArray(body.excludedKeywords) ? body.excludedKeywords : [], - mode: ["NotifyOnly", "ManualConfirm", "AutoDownload"].includes(body.mode) ? body.mode : "ManualConfirm", + minSizeBytes: + typeof body.minSizeBytes === "number" ? body.minSizeBytes : null, + maxSizeBytes: + typeof body.maxSizeBytes === "number" ? body.maxSizeBytes : null, + excludedKeywords: Array.isArray(body.excludedKeywords) + ? body.excludedKeywords + : [], + mode: ["NotifyOnly", "ManualConfirm", "AutoDownload"].includes( + body.mode, + ) + ? body.mode + : "ManualConfirm", createdAt: existing?.createdAt ?? new Date().toISOString(), updatedAt: new Date().toISOString(), }; - if (policy.minSizeBytes != null && policy.maxSizeBytes != null && policy.minSizeBytes > policy.maxSizeBytes) { + if ( + policy.minSizeBytes != null && + policy.maxSizeBytes != null && + policy.minSizeBytes > policy.maxSizeBytes + ) { return json(res, { error: "Invalid size range" }, 400); } subscriptionPolicies.set(feedId, policy); @@ -2903,8 +3240,7 @@ server.listen(PORT, () => { (animation) => animation.isDownloadFinished, ).length; const downloadingCount = [...animations.values()].filter( - (animation) => - animation.isDownloadTracked && !animation.isDownloadFinished, + (animation) => animation.isDownloadTracked && !animation.isDownloadFinished, ).length; console.log( ` ${animations.size} anime entries (${finishedCount} finished, ${downloadingCount} active downloads, rest untracked)`, diff --git a/SecondDimensionWatcherReDive.Client/src/components/settings/PluginSettingsSection.tsx b/SecondDimensionWatcherReDive.Client/src/components/settings/PluginSettingsSection.tsx new file mode 100644 index 0000000..d72a105 --- /dev/null +++ b/SecondDimensionWatcherReDive.Client/src/components/settings/PluginSettingsSection.tsx @@ -0,0 +1,403 @@ +import React from "react"; +import { useTranslation } from "react-i18next"; + +import { + AlertTriangle, + CheckCircle2, + PackageCheck, + Plug, + ShieldCheck, + ShieldX, + Trash2, + Upload, +} from "lucide-react"; + +import { + installPlugin, + previewPlugin, + setPluginEnabled, + uninstallPlugin, +} from "../../plugins/api"; +import { usePlugins } from "../../plugins/hooks"; +import { PluginCapabilities, PluginPackagePreview } from "../../plugins/types"; +import { useToast } from "../ToastProvider"; +import { Button } from "../ui/Button"; +import { Card } from "../ui/Card"; +import { Spinner } from "../ui/Spinner"; + +export const PluginSettingsSection: React.FC = () => { + const { t } = useTranslation("settings"); + const { addToast } = useToast(); + const { data: plugins, error, mutate } = usePlugins(); + const [file, setFile] = React.useState(null); + const [preview, setPreview] = React.useState( + null, + ); + const [approved, setApproved] = React.useState(false); + const [busy, setBusy] = React.useState(false); + + const run = React.useCallback( + async (operation: () => Promise, success: string) => { + setBusy(true); + try { + await operation(); + await mutate(); + addToast({ title: success, color: "success" }); + } catch (operationError) { + addToast({ + title: t("system.plugins.operationFailed"), + text: + operationError instanceof Error + ? operationError.message + : String(operationError), + color: "danger", + }); + } finally { + setBusy(false); + } + }, + [addToast, mutate, t], + ); + + const inspect = async () => { + if (!file) return; + setBusy(true); + try { + setPreview(await previewPlugin(file)); + setApproved(false); + } catch (previewError) { + addToast({ + title: t("system.plugins.previewFailed"), + text: + previewError instanceof Error + ? previewError.message + : String(previewError), + color: "danger", + }); + } finally { + setBusy(false); + } + }; + + const isUpgrade = Boolean( + preview && + plugins?.some((plugin) => plugin.manifest.id === preview.manifest.id), + ); + + return ( +
+
+

+ {t("system.plugins.eyebrow")} +

+

+ {t("system.plugins.title")} +

+

+ {t("system.plugins.description")} +

+
+ + } + title={t("system.plugins.install.title")} + description={t("system.plugins.install.description")} + > +
+ { + setFile(event.target.files?.[0] ?? null); + setPreview(null); + setApproved(false); + }} + className="min-w-0 flex-1 text-sm text-muted file:mr-3 file:rounded-md file:border-0 file:bg-canvas file:px-3 file:py-2 file:text-sm file:text-foreground" + /> + +
+ + {preview ? ( +
+
+
+

+ {preview.manifest.name} {preview.manifest.version} +

+

+ {preview.manifest.id} · API {preview.manifest.apiVersion} · + SHA-256 {preview.packageSha256} +

+
+ + {preview.isSignatureTrusted ? ( + + ) : ( + + )} + {preview.signatureStatus} + +
+ + + + {preview.compatibilityErrors.length ? ( +
+ {preview.compatibilityErrors.map((message) => ( +

+ + {message} +

+ ))} +
+ ) : null} + + + +
+ ) : null} + + +
+

+ {t("system.plugins.installed.title")} +

+ {error ? ( +

{t("system.plugins.loadFailed")}

+ ) : !plugins ? ( +
+ +
+ ) : plugins.length === 0 ? ( +

+ {t("system.plugins.installed.empty")} +

+ ) : ( + plugins.map((plugin) => ( + + ) : ( + + ) + } + title={`${plugin.manifest.name} ${plugin.manifest.version}`} + description={plugin.manifest.description ?? plugin.manifest.id} + footer={ +
+

+ {t("system.plugins.installed.dataPolicy")} +

+
+ + + +
+
+ } + > +
+ + {t("system.plugins.installed.api", { + version: plugin.manifest.apiVersion, + })} + + + {t("system.plugins.installed.health", { + status: plugin.health.status, + })} + + + {t("system.plugins.installed.failures", { + count: plugin.health.consecutiveFailures, + })} + +
+ + {plugin.compatibilityErrors.map((message) => ( +

+ + {message} +

+ ))} + {plugin.health.lastError ? ( +

+ {plugin.health.lastError} +

+ ) : null} + + )) + )} +
+
+ ); +}; + +const CapabilityList: React.FC<{ + capabilities: PluginCapabilities; + compact?: boolean; +}> = ({ capabilities, compact = false }) => { + const { t } = useTranslation("settings"); + const values = [ + ...capabilities.networkDomains.map((domain) => + t("system.plugins.capabilities.network", { value: domain }), + ), + ...capabilities.fileRoots.map((root) => + t("system.plugins.capabilities.files", { value: root }), + ), + ...(capabilities.notifications + ? [t("system.plugins.capabilities.notifications")] + : []), + ...(capabilities.downloadControl + ? [t("system.plugins.capabilities.downloads")] + : []), + ...(capabilities.storageAccess + ? [t("system.plugins.capabilities.storage")] + : []), + ...(capabilities.backgroundTasks + ? [t("system.plugins.capabilities.background")] + : []), + ]; + return ( +
+ {!compact ? ( +

+ {t("system.plugins.capabilities.title")} +

+ ) : null} +
+ {(values.length ? values : [t("system.plugins.capabilities.none")]).map( + (value) => ( + + {value} + + ), + )} +
+
+ ); +}; diff --git a/SecondDimensionWatcherReDive.Client/src/components/settings/SettingsNavigation.tsx b/SecondDimensionWatcherReDive.Client/src/components/settings/SettingsNavigation.tsx index 741573f..7d523e5 100644 --- a/SecondDimensionWatcherReDive.Client/src/components/settings/SettingsNavigation.tsx +++ b/SecondDimensionWatcherReDive.Client/src/components/settings/SettingsNavigation.tsx @@ -1,7 +1,14 @@ import React from "react"; import { useTranslation } from "react-i18next"; -import { Activity, Bot, Database, Download, Network } from "lucide-react"; +import { + Activity, + Bot, + Database, + Download, + Network, + Puzzle, +} from "lucide-react"; import { cn } from "../../lib/cn"; import { Select } from "./SettingsControls"; @@ -12,6 +19,7 @@ export const settingsSectionIds = [ "media", "health", "access", + "plugins", ] as const; export type SettingsSectionId = (typeof settingsSectionIds)[number]; @@ -22,6 +30,7 @@ const sectionIcons: Record = { media: , health: , access: , + plugins: , }; export interface SettingsNavigationProps { diff --git a/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/settings.json b/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/settings.json index 9c91e39..e0171a5 100644 --- a/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/settings.json +++ b/SecondDimensionWatcherReDive.Client/src/i18n/locales/en/settings.json @@ -10,7 +10,8 @@ "downloads": "Downloads & storage", "media": "Media & metadata", "health": "Health monitoring", - "access": "Access protocols" + "access": "Access protocols", + "plugins": "Plugins" }, "pendingRestart": { "title": "Some settings are waiting for a restart", @@ -182,6 +183,52 @@ "maxConnections": "Maximum connections", "restartHelp": "NFS binds its listening port when the application starts. Changes are saved now and used after a restart." } + }, + "plugins": { + "eyebrow": "Controlled extensions", + "title": "Plugin platform", + "description": "Install local, versioned packages through checksum, signature, compatibility, and explicit capability review. JavaScript runs in a short-lived isolated worker and never receives .NET services.", + "loadFailed": "Could not load installed plugins.", + "previewFailed": "Package inspection failed", + "operationFailed": "Plugin operation failed", + "enabled": "Plugin enabled", + "disabled": "Plugin disabled", + "uninstalled": "Plugin uninstalled; configuration and data were retained", + "deleted": "Plugin, configuration, and data permanently deleted", + "install": { + "title": "Inspect a local package", + "description": "Remote URL installation is disabled. Upload a .sdwpkg or .zip obtained through a trusted administrative channel; no code runs during inspection or installation.", + "inspect": "Inspect package", + "approve": "I reviewed and approve every capability shown above. Any changed package or manifest requires a new approval.", + "install": "Install disabled", + "upgrade": "Upgrade disabled", + "installed": "Plugin installed disabled; enable it after compatibility review", + "upgraded": "Plugin upgraded disabled; configuration was preserved and the declared data strategy was applied" + }, + "installed": { + "title": "Installed plugins", + "empty": "No plugin packages are installed.", + "enable": "Enable", + "disable": "Disable", + "uninstall": "Uninstall", + "uninstallConfirm": "Uninstall {{name}}? Its configuration and plugin data will be retained for a future reinstall.", + "deleteData": "Delete all", + "deleteConfirm": "Permanently delete {{name}}, its configuration, and all plugin data? This cannot be undone.", + "dataPolicy": "Uninstall preserves configuration and data by default. Delete all is irreversible. Data-version changes require an explicit reset strategy.", + "api": "API {{version}}", + "health": "Health: {{status}}", + "failures": "Consecutive failures: {{count}}" + }, + "capabilities": { + "title": "Requested capabilities", + "none": "No privileged capabilities", + "network": "Network: {{value}}", + "files": "Files: {{value}}", + "notifications": "Publish notifications", + "downloads": "Control downloads", + "storage": "Plugin-scoped storage", + "background": "Background tasks" + } } }, "mediaLibrary": { diff --git a/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/settings.json b/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/settings.json index 40eebfe..c75398f 100644 --- a/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/settings.json +++ b/SecondDimensionWatcherReDive.Client/src/i18n/locales/ja/settings.json @@ -10,7 +10,8 @@ "downloads": "ダウンロードと保存先", "media": "メディアとメタデータ", "health": "ヘルス監視", - "access": "アクセスプロトコル" + "access": "アクセスプロトコル", + "plugins": "プラグイン" }, "pendingRestart": { "title": "再起動待ちの設定があります", @@ -182,6 +183,52 @@ "maxConnections": "最大接続数", "restartHelp": "NFS はアプリケーション起動時に待受ポートをバインドします。変更は保存され、再起動後に使用されます。" } + }, + "plugins": { + "eyebrow": "制御された拡張", + "title": "プラグインプラットフォーム", + "description": "ローカルのバージョン付きパッケージは、チェックサム、署名、互換性、明示的な権限確認を経てインストールされます。JavaScript は短命な分離ワーカーで実行され、.NET サービスにはアクセスできません。", + "loadFailed": "インストール済みプラグインを読み込めませんでした。", + "previewFailed": "パッケージ検査に失敗しました", + "operationFailed": "プラグイン操作に失敗しました", + "enabled": "プラグインを有効にしました", + "disabled": "プラグインを無効にしました", + "uninstalled": "プラグインを削除し、設定とデータを保持しました", + "deleted": "プラグイン、設定、データを完全に削除しました", + "install": { + "title": "ローカルパッケージを検査", + "description": "リモート URL からのインストールは無効です。信頼できる管理経路で取得した .sdwpkg または .zip をアップロードしてください。検査・インストール中にコードは実行されません。", + "inspect": "パッケージを検査", + "approve": "上記のすべての権限を確認し、承認します。パッケージまたは manifest が変わった場合は再承認が必要です。", + "install": "無効状態でインストール", + "upgrade": "無効状態でアップグレード", + "installed": "プラグインを無効状態でインストールしました", + "upgraded": "プラグインを無効状態で更新し、設定と宣言済みデータ戦略を適用しました" + }, + "installed": { + "title": "インストール済みプラグイン", + "empty": "プラグインはまだインストールされていません。", + "enable": "有効化", + "disable": "無効化", + "uninstall": "アンインストール", + "uninstallConfirm": "{{name}} を削除しますか？設定とデータは再インストール用に保持されます。", + "deleteData": "すべて削除", + "deleteConfirm": "{{name}}、その設定、すべてのプラグインデータを完全に削除しますか？この操作は元に戻せません。", + "dataPolicy": "アンインストール時は設定とデータを既定で保持します。「すべて削除」は元に戻せません。データバージョン変更には明示的な reset 戦略が必要です。", + "api": "API {{version}}", + "health": "状態: {{status}}", + "failures": "連続失敗: {{count}}" + }, + "capabilities": { + "title": "要求された権限", + "none": "特権なし", + "network": "ネットワーク: {{value}}", + "files": "ファイル: {{value}}", + "notifications": "通知を発行", + "downloads": "ダウンロード制御", + "storage": "プラグイン専用ストレージ", + "background": "バックグラウンドタスク" + } } }, "mediaLibrary": { diff --git a/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/settings.json b/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/settings.json index 1d101be..3dcdee7 100644 --- a/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/settings.json +++ b/SecondDimensionWatcherReDive.Client/src/i18n/locales/zh-CN/settings.json @@ -10,7 +10,8 @@ "downloads": "下载与存储", "media": "媒体与元数据", "health": "健康监控", - "access": "访问协议" + "access": "访问协议", + "plugins": "插件" }, "pendingRestart": { "title": "存在等待重启生效的设置", @@ -182,6 +183,52 @@ "maxConnections": "最大连接数", "restartHelp": "NFS 在应用启动时绑定监听端口。本区更改会被保存，但需要重启应用后才会使用。" } + }, + "plugins": { + "eyebrow": "受控扩展", + "title": "插件平台", + "description": "本地版本化插件包必须经过校验和、签名、兼容性和显式权限审查。JavaScript 在短生命周期隔离进程中执行，无法获取 .NET 服务。", + "loadFailed": "无法加载已安装插件。", + "previewFailed": "插件包检查失败", + "operationFailed": "插件操作失败", + "enabled": "插件已启用", + "disabled": "插件已停用", + "uninstalled": "插件已卸载，配置和数据已保留", + "deleted": "插件、配置和数据已永久删除", + "install": { + "title": "检查本地插件包", + "description": "远程 URL 安装已禁用。请从可信管理渠道取得 .sdwpkg 或 .zip 后上传；检查和安装阶段不会执行代码。", + "inspect": "检查插件包", + "approve": "我已审阅并批准上方列出的全部权限。插件包或 manifest 发生变化后必须重新批准。", + "install": "安装（默认停用）", + "upgrade": "升级（默认停用）", + "installed": "插件已安装但尚未启用，请完成兼容性复核后启用", + "upgraded": "插件已升级但尚未启用；配置已保留，并已应用声明的数据策略" + }, + "installed": { + "title": "已安装插件", + "empty": "尚未安装插件包。", + "enable": "启用", + "disable": "停用", + "uninstall": "卸载", + "uninstallConfirm": "卸载 {{name}}？配置和插件数据会保留，供以后重新安装。", + "deleteData": "全部删除", + "deleteConfirm": "永久删除 {{name}}、其配置和所有插件数据？此操作无法撤销。", + "dataPolicy": "卸载默认保留配置和数据；“全部删除”无法撤销。数据版本变化必须显式声明 reset 策略。", + "api": "API {{version}}", + "health": "健康：{{status}}", + "failures": "连续失败：{{count}}" + }, + "capabilities": { + "title": "申请的权限", + "none": "无特权能力", + "network": "网络：{{value}}", + "files": "文件：{{value}}", + "notifications": "发布通知", + "downloads": "控制下载", + "storage": "插件专属存储", + "background": "后台任务" + } } }, "mediaLibrary": { diff --git a/SecondDimensionWatcherReDive.Client/src/pages/SettingsPage.tsx b/SecondDimensionWatcherReDive.Client/src/pages/SettingsPage.tsx index 55dae21..3cf31a9 100644 --- a/SecondDimensionWatcherReDive.Client/src/pages/SettingsPage.tsx +++ b/SecondDimensionWatcherReDive.Client/src/pages/SettingsPage.tsx @@ -9,6 +9,7 @@ import { AiSettingsSection } from "../components/settings/AiSettingsSection"; import { DownloadSettingsSection } from "../components/settings/DownloadSettingsSection"; import { HealthSettingsSection } from "../components/settings/HealthSettingsSection"; import { MediaSettingsSection } from "../components/settings/MediaSettingsSection"; +import { PluginSettingsSection } from "../components/settings/PluginSettingsSection"; import { SettingsNavigation, SettingsSectionId, @@ -160,6 +161,8 @@ const ActiveSection: React.FC = ({ ); case "access": return ; + case "plugins": + return ; case "ai": default: return ; diff --git a/SecondDimensionWatcherReDive.Client/src/plugins/api.ts b/SecondDimensionWatcherReDive.Client/src/plugins/api.ts new file mode 100644 index 0000000..ca45af7 --- /dev/null +++ b/SecondDimensionWatcherReDive.Client/src/plugins/api.ts @@ -0,0 +1,49 @@ +import fetcher from "../auth/httpClient"; +import { + InstalledPlugin, + PluginCapabilities, + PluginPackagePreview, +} from "./types"; + +export const getPlugins = () => fetcher("/api/plugins"); + +export const previewPlugin = (packageFile: File) => { + const body = new FormData(); + body.append("package", packageFile); + return fetcher("/api/plugins/preview", { + method: "POST", + body, + }); +}; + +export const installPlugin = ( + preview: PluginPackagePreview, + upgrade: boolean, +) => + fetcher( + `/api/plugins${upgrade ? `/${preview.manifest.id}/upgrade` : "/install"}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + previewToken: preview.token, + expectedSha256: preview.packageSha256, + approvedCapabilities: preview.manifest + .capabilities satisfies PluginCapabilities, + }), + }, + ); + +export const setPluginEnabled = (id: string, enabled: boolean) => + fetcher( + `/api/plugins/${encodeURIComponent(id)}/${enabled ? "enable" : "disable"}`, + { + method: "POST", + }, + ); + +export const uninstallPlugin = (id: string, deleteData = false) => + fetcher( + `/api/plugins/${encodeURIComponent(id)}?deleteData=${String(deleteData)}`, + { method: "DELETE" }, + ); diff --git a/SecondDimensionWatcherReDive.Client/src/plugins/hooks.ts b/SecondDimensionWatcherReDive.Client/src/plugins/hooks.ts new file mode 100644 index 0000000..4087ef2 --- /dev/null +++ b/SecondDimensionWatcherReDive.Client/src/plugins/hooks.ts @@ -0,0 +1,5 @@ +import useSWR from "swr"; + +import { getPlugins } from "./api"; + +export const usePlugins = () => useSWR("/api/plugins", getPlugins); diff --git a/SecondDimensionWatcherReDive.Client/src/plugins/types.ts b/SecondDimensionWatcherReDive.Client/src/plugins/types.ts new file mode 100644 index 0000000..16e462a --- /dev/null +++ b/SecondDimensionWatcherReDive.Client/src/plugins/types.ts @@ -0,0 +1,52 @@ +export interface PluginCapabilities { + networkDomains: string[]; + fileRoots: string[]; + notifications: boolean; + downloadControl: boolean; + storageAccess: boolean; + backgroundTasks: boolean; +} + +export interface PluginManifest { + id: string; + name: string; + version: string; + apiVersion: string; + entryPoint: string; + description?: string; + dependencies: { id: string; minimumVersion: string }[]; + capabilities: PluginCapabilities; + platforms: string[]; + fileSha256: Record; + signaturePublisher?: string; + signatureAlgorithm?: string; + providers: { kind: string; name: string; handlers: Record }[]; + dataVersion: number; + dataMigration?: { strategy: string; description?: string }; +} + +export interface PluginPackagePreview { + token: string; + packageSha256: string; + manifest: PluginManifest; + compatibilityErrors: string[]; + isSignatureTrusted: boolean; + signatureStatus: string; + expiresAt: string; +} + +export interface InstalledPlugin { + manifest: PluginManifest; + isEnabled: boolean; + approvedCapabilities: PluginCapabilities; + compatibilityErrors: string[]; + health: { + status: string; + consecutiveFailures: number; + lastSuccessAt?: string; + lastFailureAt?: string; + lastError?: string; + circuitOpenUntil?: string; + }; + hasConfiguration: boolean; +} diff --git a/SecondDimensionWatcherReDive.Framework/DataRepository/IPluginCatalogRepository.cs b/SecondDimensionWatcherReDive.Framework/DataRepository/IPluginCatalogRepository.cs new file mode 100644 index 0000000..13418d6 --- /dev/null +++ b/SecondDimensionWatcherReDive.Framework/DataRepository/IPluginCatalogRepository.cs @@ -0,0 +1,31 @@ +using SecondDimensionWatcherReDive.Framework.Plugin; + +namespace SecondDimensionWatcherReDive.Framework.DataRepository; + +public sealed record PluginCatalogEntry( + PluginManifest Manifest, + bool IsEnabled, + PluginCapabilities ApprovedCapabilities, + PluginHealth Health, + string PackageDirectory, + string ConfigurationJson, + int DataVersion, + string? PublisherFingerprint); + +public sealed record RetainedPluginData( + string Id, + string ConfigurationJson, + int DataVersion, + DateTimeOffset RetainedAt, + string? PublisherFingerprint); + +public interface IPluginCatalogRepository +{ + Task> GetAllAsync(CancellationToken cancellationToken); + Task FindAsync(string id, CancellationToken cancellationToken); + Task SaveAsync(PluginCatalogEntry entry, CancellationToken cancellationToken); + Task RemoveAsync(string id, CancellationToken cancellationToken); + Task FindRetainedAsync(string id, CancellationToken cancellationToken); + Task SaveRetainedAsync(RetainedPluginData retained, CancellationToken cancellationToken); + Task RemoveRetainedAsync(string id, CancellationToken cancellationToken); +} diff --git a/SecondDimensionWatcherReDive.Framework/Plugin/IJavaScriptPluginLoader.cs b/SecondDimensionWatcherReDive.Framework/Plugin/IJavaScriptPluginLoader.cs index a51f69d..ecc50ea 100644 --- a/SecondDimensionWatcherReDive.Framework/Plugin/IJavaScriptPluginLoader.cs +++ b/SecondDimensionWatcherReDive.Framework/Plugin/IJavaScriptPluginLoader.cs @@ -1,6 +1,20 @@ namespace SecondDimensionWatcherReDive.Framework.Plugin; +/// +/// Provides the two-phase, local-package installation boundary for JavaScript plugins. +/// A package is never evaluated by either operation; execution is only possible after +/// an explicit capability approval and a separate enable operation. +/// public interface IJavaScriptPluginLoader { - public IPlugin LoadJavaScriptPlugin(string script); -} \ No newline at end of file + Task PreviewPackageAsync( + Stream package, + string fileName, + CancellationToken cancellationToken); + + Task InstallPackageAsync( + string previewToken, + string expectedSha256, + PluginCapabilities approvedCapabilities, + CancellationToken cancellationToken); +} diff --git a/SecondDimensionWatcherReDive.Framework/Plugin/INotificationProvider.cs b/SecondDimensionWatcherReDive.Framework/Plugin/INotificationProvider.cs new file mode 100644 index 0000000..9083a0d --- /dev/null +++ b/SecondDimensionWatcherReDive.Framework/Plugin/INotificationProvider.cs @@ -0,0 +1,16 @@ +namespace SecondDimensionWatcherReDive.Framework.Plugin; + +public sealed record PluginNotification( + string Title, + string Message, + string Severity = "info", + IReadOnlyDictionary? Metadata = null); + +public interface INotificationProvider +{ + string Name { get; } + + Task SendAsync( + PluginNotification notification, + CancellationToken cancellationToken); +} diff --git a/SecondDimensionWatcherReDive.Framework/Plugin/IPluginServices.cs b/SecondDimensionWatcherReDive.Framework/Plugin/IPluginServices.cs index eedee23..021971a 100644 --- a/SecondDimensionWatcherReDive.Framework/Plugin/IPluginServices.cs +++ b/SecondDimensionWatcherReDive.Framework/Plugin/IPluginServices.cs @@ -17,8 +17,4 @@ public interface IPluginServices /// Thrown when TParams type is incorrect for the specified event name. IPluginEventRegister GetRegister(string eventName); - /// - /// Represents a service provider for plugin-related operations. - /// - IServiceProvider ServiceProvider { get; } } diff --git a/SecondDimensionWatcherReDive.Framework/Plugin/PluginManifest.cs b/SecondDimensionWatcherReDive.Framework/Plugin/PluginManifest.cs new file mode 100644 index 0000000..580166e --- /dev/null +++ b/SecondDimensionWatcherReDive.Framework/Plugin/PluginManifest.cs @@ -0,0 +1,160 @@ +using System.Text; +using System.Text.Json; + +namespace SecondDimensionWatcherReDive.Framework.Plugin; + +public static class PluginApi +{ + public const string CurrentVersion = "1.0"; +} + +public sealed record PluginManifest +{ + public required string Id { get; init; } + public required string Name { get; init; } + public required string Version { get; init; } + public required string ApiVersion { get; init; } + public required string EntryPoint { get; init; } + public string? Description { get; init; } + public IReadOnlyList Dependencies { get; init; } = []; + public PluginCapabilities Capabilities { get; init; } = new(); + public IReadOnlyList Platforms { get; init; } = ["any"]; + public PluginIntegrity? Integrity { get; init; } + public PluginSignature? Signature { get; init; } + public IReadOnlyList Providers { get; init; } = []; + public int DataVersion { get; init; } = 1; + public PluginDataMigration? DataMigration { get; init; } +} + +public sealed record PluginDependency(string Id, string MinimumVersion); + +public sealed record PluginCapabilities +{ + public IReadOnlyList NetworkDomains { get; init; } = []; + public IReadOnlyList FileRoots { get; init; } = []; + public bool Notifications { get; init; } + public bool DownloadControl { get; init; } + public bool StorageAccess { get; init; } + public bool BackgroundTasks { get; init; } +} + +public sealed record PluginIntegrity +{ + /// + /// SHA-256 digests for every regular package file except manifest.json. Paths use '/' separators. + /// The exact path set and every digest are covered by the publisher signature. + /// + public IReadOnlyDictionary Files { get; init; } = + new Dictionary(StringComparer.Ordinal); +} + +public sealed record PluginSignature( + string Publisher, + string Algorithm, + string Value); + +public sealed record PluginProviderDeclaration +{ + public required string Kind { get; init; } + public required string Name { get; init; } + public required IReadOnlyDictionary Handlers { get; init; } +} + +public sealed record PluginDataMigration +{ + /// Preserve or Reset. A data version change requires an explicit Reset. + public required string Strategy { get; init; } + public string? Description { get; init; } +} + +public sealed record PluginPackagePreview( + string Token, + string PackageSha256, + PluginManifest Manifest, + IReadOnlyList CompatibilityErrors, + bool IsSignatureTrusted, + string SignatureStatus, + DateTimeOffset ExpiresAt); + +public sealed record PluginInstallResult( + string Id, + string Version, + bool IsUpgrade, + IReadOnlyList CompatibilityErrors); + +public sealed record PluginHealth( + string Status, + int ConsecutiveFailures, + DateTimeOffset? LastSuccessAt, + DateTimeOffset? LastFailureAt, + string? LastError, + DateTimeOffset? CircuitOpenUntil); + +public sealed record InstalledPlugin( + PluginManifest Manifest, + bool IsEnabled, + PluginCapabilities ApprovedCapabilities, + IReadOnlyList CompatibilityErrors, + PluginHealth Health, + JsonElement Configuration, + bool DataRetainedFromUninstall = false); + +/// +/// Produces the unambiguous payload covered by an RSA-SHA256 publisher signature. +/// Every execution-relevant manifest field is included; the signature value itself is excluded. +/// +public static class PluginSignaturePayload +{ + public static byte[] Create(PluginManifest manifest) + { + var lines = new List { "sdw-plugin-signature-v2" }; + Add(lines, "id", manifest.Id); + Add(lines, "name", manifest.Name); + Add(lines, "description", manifest.Description ?? string.Empty); + Add(lines, "version", manifest.Version); + Add(lines, "api", manifest.ApiVersion); + Add(lines, "entry", manifest.EntryPoint); + foreach (var file in (manifest.Integrity?.Files ?? new Dictionary()) + .OrderBy(value => value.Key, StringComparer.Ordinal)) + { + Add(lines, "file-path", file.Key); + Add(lines, "file-sha256", file.Value.ToLowerInvariant()); + } + Add(lines, "data-version", manifest.DataVersion.ToString(System.Globalization.CultureInfo.InvariantCulture)); + Add(lines, "migration-strategy", manifest.DataMigration?.Strategy ?? string.Empty); + Add(lines, "migration-description", manifest.DataMigration?.Description ?? string.Empty); + Add(lines, "notifications", manifest.Capabilities.Notifications ? "1" : "0"); + Add(lines, "download-control", manifest.Capabilities.DownloadControl ? "1" : "0"); + Add(lines, "storage-access", manifest.Capabilities.StorageAccess ? "1" : "0"); + Add(lines, "background-tasks", manifest.Capabilities.BackgroundTasks ? "1" : "0"); + foreach (var value in manifest.Capabilities.NetworkDomains.Order(StringComparer.OrdinalIgnoreCase)) + Add(lines, "network-domain", value.ToLowerInvariant()); + foreach (var value in manifest.Capabilities.FileRoots.Order(StringComparer.Ordinal)) + Add(lines, "file-root", value); + foreach (var value in manifest.Platforms.Order(StringComparer.OrdinalIgnoreCase)) + Add(lines, "platform", value.ToLowerInvariant()); + foreach (var dependency in manifest.Dependencies + .OrderBy(value => value.Id, StringComparer.Ordinal) + .ThenBy(value => value.MinimumVersion, StringComparer.Ordinal)) + { + Add(lines, "dependency-id", dependency.Id); + Add(lines, "dependency-version", dependency.MinimumVersion); + } + foreach (var provider in manifest.Providers + .OrderBy(value => value.Kind, StringComparer.Ordinal) + .ThenBy(value => value.Name, StringComparer.Ordinal)) + { + Add(lines, "provider-kind", provider.Kind); + Add(lines, "provider-name", provider.Name); + foreach (var handler in provider.Handlers.OrderBy(value => value.Key, StringComparer.Ordinal)) + { + Add(lines, "handler-operation", handler.Key); + Add(lines, "handler-name", handler.Value); + } + } + return Encoding.UTF8.GetBytes(string.Join('\n', lines)); + } + + private static void Add(ICollection lines, string name, string value) + => lines.Add($"{name}:{Convert.ToBase64String(Encoding.UTF8.GetBytes(value))}"); +} diff --git a/SecondDimensionWatcherReDive.IntegrationTest/Plugins/PluginApiTests.cs b/SecondDimensionWatcherReDive.IntegrationTest/Plugins/PluginApiTests.cs new file mode 100644 index 0000000..04449a9 --- /dev/null +++ b/SecondDimensionWatcherReDive.IntegrationTest/Plugins/PluginApiTests.cs @@ -0,0 +1,149 @@ +using System.IO.Compression; +using System.Net; +using System.Net.Http.Json; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; + +namespace SecondDimensionWatcherReDive.IntegrationTest.Plugins; + +[TestClass] +public sealed class PluginApiTests +{ + [TestMethod] + public async Task ManagementApi_RequiresPreviewApproval_AndSupportsLifecycle() + { + await using var factory = new WebDavWebApplicationFactory(); + using var client = factory.CreateJwtClient(); + var id = $"test.api-{Guid.NewGuid():N}"; + await using var package = CreatePackage(id, "1.0"); + using var form = new MultipartFormDataContent(); + form.Add(new StreamContent(package), "package", $"{id}.sdwpkg"); + + using var previewResponse = await client.PostAsync("/api/plugins/preview", form); + Assert.AreEqual(HttpStatusCode.OK, previewResponse.StatusCode, + await previewResponse.Content.ReadAsStringAsync()); + using var preview = JsonDocument.Parse(await previewResponse.Content.ReadAsStringAsync()); + var previewRoot = preview.RootElement; + Assert.AreEqual(id, previewRoot.GetProperty("manifest").GetProperty("id").GetString()); + Assert.IsFalse(previewRoot.GetProperty("isSignatureTrusted").GetBoolean()); + var capabilities = previewRoot.GetProperty("manifest").GetProperty("capabilities").Clone(); + + using var installResponse = await client.PostAsJsonAsync("/api/plugins/install", new + { + previewToken = previewRoot.GetProperty("token").GetString(), + expectedSha256 = previewRoot.GetProperty("packageSha256").GetString(), + approvedCapabilities = capabilities + }); + Assert.AreEqual(HttpStatusCode.OK, installResponse.StatusCode, + await installResponse.Content.ReadAsStringAsync()); + + using var enableResponse = await client.PostAsync($"/api/plugins/{id}/enable", null); + Assert.AreEqual(HttpStatusCode.OK, enableResponse.StatusCode, + await enableResponse.Content.ReadAsStringAsync()); + using var listResponse = await client.GetAsync("/api/plugins"); + Assert.AreEqual(HttpStatusCode.OK, listResponse.StatusCode); + using var list = JsonDocument.Parse(await listResponse.Content.ReadAsStringAsync()); + var installed = list.RootElement.EnumerateArray().Single(item => + item.GetProperty("manifest").GetProperty("id").GetString() == id); + Assert.IsTrue(installed.GetProperty("isEnabled").GetBoolean()); + Assert.IsFalse(installed.TryGetProperty("configuration", out _), + "Plugin configuration must not leak from the management response."); + + using var disableResponse = await client.PostAsync($"/api/plugins/{id}/disable", null); + Assert.AreEqual(HttpStatusCode.OK, disableResponse.StatusCode); + using var deleteResponse = await client.DeleteAsync($"/api/plugins/{id}"); + Assert.AreEqual(HttpStatusCode.OK, deleteResponse.StatusCode); + } + + [TestMethod] + public async Task ManagementApi_DisablesRemoteInstall_AndExplainsApiIncompatibility() + { + await using var factory = new WebDavWebApplicationFactory(); + using var client = factory.CreateJwtClient(); + using var remote = await client.PostAsJsonAsync("/api/plugins/preview-remote", new + { + url = "https://untrusted.example/plugin.js", + expectedSha256 = (string?)null + }); + Assert.AreEqual(HttpStatusCode.Forbidden, remote.StatusCode); + StringAssert.Contains(await remote.Content.ReadAsStringAsync(), "remote_install_disabled"); + + var id = $"test.future-{Guid.NewGuid():N}"; + await using var package = CreatePackage(id, "9.0"); + using var form = new MultipartFormDataContent(); + form.Add(new StreamContent(package), "package", $"{id}.sdwpkg"); + using var previewResponse = await client.PostAsync("/api/plugins/preview", form); + using var preview = JsonDocument.Parse(await previewResponse.Content.ReadAsStringAsync()); + var root = preview.RootElement; + using var install = await client.PostAsJsonAsync("/api/plugins/install", new + { + previewToken = root.GetProperty("token").GetString(), + expectedSha256 = root.GetProperty("packageSha256").GetString(), + approvedCapabilities = root.GetProperty("manifest").GetProperty("capabilities").Clone() + }); + Assert.AreEqual(HttpStatusCode.OK, install.StatusCode, await install.Content.ReadAsStringAsync()); + + using var enable = await client.PostAsync($"/api/plugins/{id}/enable", null); + Assert.AreEqual(HttpStatusCode.Conflict, enable.StatusCode); + var error = await enable.Content.ReadAsStringAsync(); + StringAssert.Contains(error, "incompatible"); + StringAssert.Contains(error, "9.0"); + await client.DeleteAsync($"/api/plugins/{id}?deleteData=true"); + } + + [TestMethod] + public async Task ManagementApi_ReturnsBadRequestForMalformedPluginId() + { + await using var factory = new WebDavWebApplicationFactory(); + using var client = factory.CreateJwtClient(); + + using var response = await client.PostAsync("/api/plugins/bad!id/enable", null); + + Assert.AreEqual(HttpStatusCode.BadRequest, response.StatusCode); + StringAssert.Contains(await response.Content.ReadAsStringAsync(), "invalid_plugin_request"); + } + + private static MemoryStream CreatePackage(string id, string apiVersion) + { + const string script = "globalThis.sdwPlugin={handlers:{ping:()=>({ok:true})}};"; + var digest = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(script))).ToLowerInvariant(); + var manifest = JsonSerializer.Serialize(new + { + id, + name = id, + version = "1.0.0", + apiVersion, + entryPoint = "index.js", + dependencies = Array.Empty(), + capabilities = new + { + networkDomains = Array.Empty(), + fileRoots = Array.Empty(), + notifications = false, + downloadControl = false, + storageAccess = false, + backgroundTasks = false + }, + platforms = new[] { "any" }, + integrity = new { files = new Dictionary { ["index.js"] = digest } }, + providers = Array.Empty(), + dataVersion = 1 + }); + var stream = new MemoryStream(); + using (var archive = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: true)) + { + WriteEntry(archive, "manifest.json", manifest); + WriteEntry(archive, "index.js", script); + } + stream.Position = 0; + return stream; + } + + private static void WriteEntry(ZipArchive archive, string path, string content) + { + var entry = archive.CreateEntry(path); + using var writer = new StreamWriter(entry.Open(), new UTF8Encoding(false)); + writer.Write(content); + } +} diff --git a/SecondDimensionWatcherReDive.IntegrationTest/WebDavWebApplicationFactory.cs b/SecondDimensionWatcherReDive.IntegrationTest/WebDavWebApplicationFactory.cs index ae5cf79..fffafb2 100644 --- a/SecondDimensionWatcherReDive.IntegrationTest/WebDavWebApplicationFactory.cs +++ b/SecondDimensionWatcherReDive.IntegrationTest/WebDavWebApplicationFactory.cs @@ -46,6 +46,9 @@ static WebDavWebApplicationFactory() Environment.SetEnvironmentVariable("AI__Anthropic__ApiKey", string.Empty); Environment.SetEnvironmentVariable("TmdbApiKey", string.Empty); Environment.SetEnvironmentVariable("Valkey__ConnectionString", string.Empty); + Environment.SetEnvironmentVariable("PluginPlatform__RootPath", + Path.Combine(Path.GetTempPath(), $"sdw-plugin-api-tests-{Environment.ProcessId}")); + Environment.SetEnvironmentVariable("PluginPlatform__AllowUnsignedLocalPackages", "true"); } public List Mappings { get; } = new(); diff --git a/SecondDimensionWatcherReDive.Test/PluginControllerTests.cs b/SecondDimensionWatcherReDive.Test/PluginControllerTests.cs new file mode 100644 index 0000000..eaa3eab --- /dev/null +++ b/SecondDimensionWatcherReDive.Test/PluginControllerTests.cs @@ -0,0 +1,33 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Moq; +using SecondDimensionWatcherReDive.Controllers; +using SecondDimensionWatcherReDive.Framework.Plugin; +using SecondDimensionWatcherReDive.PluginPlatform; + +namespace SecondDimensionWatcherReDive.Test; + +[TestClass] +public sealed class PluginControllerTests +{ + [TestMethod] + public async Task Preview_WhenStagingCapacityIsReached_ReturnsConflict() + { + var loader = new Mock(); + loader.Setup(value => value.PreviewPackageAsync( + It.IsAny(), + It.IsAny(), + It.IsAny())) + .ThrowsAsync(new InvalidOperationException("The plugin preview staging limit has been reached.")); + var controller = new PluginController(null!, loader.Object); + await using var content = new MemoryStream([1]); + var package = new FormFile(content, 0, content.Length, "package", "test.sdwpkg"); + + var result = await controller.Preview(package, CancellationToken.None); + + var conflict = Assert.IsInstanceOfType(result.Result); + Assert.AreEqual(StatusCodes.Status409Conflict, conflict.StatusCode); + var error = Assert.IsInstanceOfType(conflict.Value); + Assert.AreEqual("plugin_preview_capacity_reached", error.Code); + } +} diff --git a/SecondDimensionWatcherReDive.Test/PluginDeploymentTests.cs b/SecondDimensionWatcherReDive.Test/PluginDeploymentTests.cs new file mode 100644 index 0000000..dc2ac20 --- /dev/null +++ b/SecondDimensionWatcherReDive.Test/PluginDeploymentTests.cs @@ -0,0 +1,17 @@ +namespace SecondDimensionWatcherReDive.Test; + +[TestClass] +public sealed class PluginDeploymentTests +{ + [TestMethod] + public void PodmanCompose_PersistsPluginPlatformUnderApplicationDataVolume() + { + var compose = File.ReadAllText(Path.Combine( + AppContext.BaseDirectory, + "Deployment", + "podman-compose.yml")); + + StringAssert.Contains(compose, "PluginPlatform__RootPath: \"/app/data/plugins\""); + StringAssert.Contains(compose, "- appdata:/app/data"); + } +} diff --git a/SecondDimensionWatcherReDive.Test/PluginEventTests.cs b/SecondDimensionWatcherReDive.Test/PluginEventTests.cs index 94690b8..026ea5e 100644 --- a/SecondDimensionWatcherReDive.Test/PluginEventTests.cs +++ b/SecondDimensionWatcherReDive.Test/PluginEventTests.cs @@ -62,4 +62,45 @@ public async Task Invoke_WithNoHandlers_DoesNotThrow() // If we get here without exception, the test passes } + + [TestMethod] + public async Task Invoke_WhenHandlerFailsOrTimesOut_ContinuesWithRemainingHandlers() + { + var errors = new List(); + var pluginEvent = new PluginEvent( + TimeSpan.FromMilliseconds(40), + errors.Add); + var calls = new List(); + pluginEvent.Register((_, _) => throw new InvalidOperationException("broken")); + pluginEvent.Register(async (_, _) => await Task.Delay(TimeSpan.FromSeconds(5))); + pluginEvent.Register((_, _) => + { + calls.Add(3); + return Task.CompletedTask; + }); + + await pluginEvent.InvokeAsync( + new FileDownloadCompleteParam(Guid.NewGuid(), "/path", "local"), + CancellationToken.None); + + CollectionAssert.AreEqual(new[] { 3 }, calls); + Assert.HasCount(2, errors); + } + + [TestMethod] + public async Task Invoke_WithSingleHandler_PropagatesCallerCancellation() + { + var errors = new List(); + var pluginEvent = new PluginEvent( + TimeSpan.FromSeconds(5), + errors.Add); + pluginEvent.Register(async (_, cancellationToken) => + await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken)); + using var cancellation = new CancellationTokenSource(TimeSpan.FromMilliseconds(50)); + + await Assert.ThrowsExactlyAsync(() => pluginEvent.InvokeAsync( + new FileDownloadCompleteParam(Guid.NewGuid(), "/path", "local"), cancellation.Token)); + + Assert.IsEmpty(errors); + } } diff --git a/SecondDimensionWatcherReDive.Test/PluginPlatformIntegrationTests.cs b/SecondDimensionWatcherReDive.Test/PluginPlatformIntegrationTests.cs new file mode 100644 index 0000000..029b682 --- /dev/null +++ b/SecondDimensionWatcherReDive.Test/PluginPlatformIntegrationTests.cs @@ -0,0 +1,1382 @@ +using System.Diagnostics; +using System.IO.Compression; +using System.Net; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Options; +using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Framework.Plugin; +using SecondDimensionWatcherReDive.PluginPlatform; +using SecondDimensionWatcherReDive.Repositories; + +namespace SecondDimensionWatcherReDive.Test; + +[TestClass] +public sealed class PluginPlatformIntegrationTests +{ + private const string PingScript = """ + 'use strict'; + globalThis.sdwPlugin = { handlers: { + ping(input, configuration) { return { value: input.value, marker: configuration.marker || null }; }, + inspectHost() { return { requireType: typeof require, fetchType: typeof fetch, getTypeType: typeof __sdwHost.GetType }; } + }}; + """; + + [TestMethod] + public async Task Enable_RejectsMissingDependenciesAndIncompatibleApi_WithClearReasons() + { + await using var fixture = new PluginPlatformFixture(); + var dependent = Manifest("test.dependent", capabilities: new PluginCapabilities()) with + { + Dependencies = [new PluginDependency("test.dependency", "1.2.0")] + }; + await fixture.InstallAsync(dependent, PingScript); + + var missing = await Assert.ThrowsExactlyAsync(() => + fixture.Manager.EnableAsync(dependent.Id, CancellationToken.None)); + StringAssert.Contains(missing.Message, "test.dependency"); + StringAssert.Contains(missing.Message, "not installed"); + + await fixture.InstallAndEnableAsync(Manifest("test.dependency", version: "1.2.0"), PingScript); + await fixture.Manager.EnableAsync(dependent.Id, CancellationToken.None); + + var incompatible = Manifest("test.future-api", apiVersion: "2.0"); + await fixture.InstallAsync(incompatible, PingScript); + var apiError = await Assert.ThrowsExactlyAsync(() => + fixture.Manager.EnableAsync(incompatible.Id, CancellationToken.None)); + StringAssert.Contains(apiError.Message, "API 2.0"); + StringAssert.Contains(apiError.Message, PluginApi.CurrentVersion); + } + + [TestMethod] + public async Task Install_RequiresExactApproval_AndTrustedSignatureByDefault() + { + using var signingKey = RSA.Create(2048); + await using var fixture = new PluginPlatformFixture(options => + { + options.AllowUnsignedLocalPackages = false; + options.TrustedPublisherPublicKeys["test-publisher"] = signingKey.ExportSubjectPublicKeyInfoPem(); + }); + + var unsigned = await fixture.PreviewAsync(Manifest("test.unsigned"), PingScript); + await Assert.ThrowsExactlyAsync(() => fixture.Manager.InstallPackageAsync( + unsigned.Token, + unsigned.PackageSha256, + unsigned.Manifest.Capabilities, + CancellationToken.None)); + + var signedManifest = Sign(Manifest("test.signed"), PingScript, "test-publisher", signingKey); + var signed = await fixture.PreviewAsync(signedManifest, PingScript); + Assert.IsTrue(signed.IsSignatureTrusted, signed.SignatureStatus); + + var tampered = signedManifest with + { + Capabilities = signedManifest.Capabilities with { StorageAccess = true } + }; + var tamperedPreview = await fixture.PreviewAsync(tampered, PingScript); + Assert.IsFalse(tamperedPreview.IsSignatureTrusted, + "Changing an approved capability must invalidate the publisher signature."); + + await Assert.ThrowsExactlyAsync(() => fixture.Manager.InstallPackageAsync( + signed.Token, + signed.PackageSha256, + signed.Manifest.Capabilities with { StorageAccess = true }, + CancellationToken.None)); + + // A failed approval does not consume the staged package; exact approval succeeds. + await fixture.Manager.InstallPackageAsync( + signed.Token, + signed.PackageSha256, + signed.Manifest.Capabilities, + CancellationToken.None); + + var signedWithAsset = Sign( + Manifest("test.signed-asset"), PingScript, "test-publisher", signingKey, + ("assets/prompt.txt", "publisher content")); + await using var tamperedAssetPackage = BuildPackage( + signedWithAsset, PingScript, ("assets/prompt.txt", "attacker content")); + var assetError = await Assert.ThrowsExactlyAsync(() => + fixture.Manager.PreviewPackageAsync( + tamperedAssetPackage, "tampered-asset.sdwpkg", CancellationToken.None)); + StringAssert.Contains(assetError.Message, "assets/prompt.txt"); + } + + [TestMethod] + public async Task PackageInspection_RejectsArchiveTraversalBeforeExecution() + { + await using var fixture = new PluginPlatformFixture(); + var manifest = WithIntegrity(Manifest("test.traversal"), PingScript); + await using var package = BuildPackage(manifest, PingScript, ("../escape.js", "malicious")); + await Assert.ThrowsExactlyAsync(() => fixture.Manager.PreviewPackageAsync( + package, + "traversal.sdwpkg", + CancellationToken.None)); + Assert.IsFalse(File.Exists(Path.Combine(fixture.RootPath, "escape.js"))); + } + + [TestMethod] + public async Task PackageInspection_RejectsVersionTraversalBeforeExtraction() + { + await using var fixture = new PluginPlatformFixture(); + const string maliciousVersion = "1.0.0+/../../outside"; + var manifest = Manifest("test.version-traversal", version: maliciousVersion); + var escapedPath = Path.GetFullPath(Path.Combine( + fixture.RootPath, "packages", manifest.Id, maliciousVersion)); + + var error = await Assert.ThrowsExactlyAsync(() => + fixture.PreviewAsync(manifest, PingScript)); + + StringAssert.Contains(error.Message, "valid semantic version"); + Assert.IsFalse(Directory.Exists(escapedPath), + "An invalid manifest version must be rejected before any extraction path is created."); + } + + [TestMethod] + public async Task Manifest_RejectsAmbiguousDependencyVersionsAndUnsafeDisplayIdentifiers() + { + await using var fixture = new PluginPlatformFixture(); + var invalidDependency = Manifest("test.invalid-dependency") with + { + Dependencies = [new PluginDependency("test.dependency", "1.0.0+/../../outside")] + }; + var dependencyError = await Assert.ThrowsExactlyAsync(() => + fixture.PreviewAsync(invalidDependency, PingScript)); + StringAssert.Contains(dependencyError.Message, "invalid minimum version"); + + var invalidProvider = Manifest("test.invalid-provider", capabilities: new PluginCapabilities + { + Notifications = true + }) with + { + Name = "Unsafe\u0001name", + Providers = [new PluginProviderDeclaration + { + Kind = "notification", + Name = "../../local", + Handlers = new Dictionary { ["send"] = "sendNotification" } + }] + }; + var providerError = await Assert.ThrowsExactlyAsync(() => + fixture.PreviewAsync(invalidProvider, PingScript)); + StringAssert.Contains(providerError.Message, "ASCII identifier"); + StringAssert.Contains(providerError.Message, "control characters"); + } + + [TestMethod] + public async Task Manifest_RejectsAmbiguousIdsAndUnknownMigrationStrategy() + { + await using var fixture = new PluginPlatformFixture(); + foreach (var invalidId in new[] { "foo.", "foo..bar" }) + { + var idError = await Assert.ThrowsExactlyAsync(() => + fixture.PreviewAsync(Manifest(invalidId), PingScript)); + StringAssert.Contains(idError.Message, "Plugin id"); + } + + var invalidMigration = Manifest("test.invalid-migration") with + { + DataMigration = new PluginDataMigration { Strategy = "garbage" } + }; + var migrationError = await Assert.ThrowsExactlyAsync(() => + fixture.PreviewAsync(invalidMigration, PingScript)); + StringAssert.Contains(migrationError.Message, "preserve"); + StringAssert.Contains(migrationError.Message, "reset"); + } + + [TestMethod] + public async Task PackageStaging_EnforcesCountByteAndApprovalInputBounds() + { + var firstManifest = Manifest("test.stage-one"); + await using var probe = BuildPackage(firstManifest, PingScript); + var packageBytes = probe.Length; + + await using (var countFixture = new PluginPlatformFixture(options => + { + options.MaximumStagedPackages = 1; + })) + { + var preview = await countFixture.PreviewAsync(firstManifest, PingScript); + var countError = await Assert.ThrowsExactlyAsync(() => + countFixture.PreviewAsync(Manifest("test.stage-two"), PingScript)); + StringAssert.Contains(countError.Message, "staging limit"); + + await Assert.ThrowsExactlyAsync(() => + countFixture.Manager.InstallPackageAsync(null!, preview.PackageSha256, + preview.Manifest.Capabilities, CancellationToken.None)); + await Assert.ThrowsExactlyAsync(() => + countFixture.Manager.InstallPackageAsync(preview.Token, "not-a-checksum", + preview.Manifest.Capabilities, CancellationToken.None)); + } + + await using var byteFixture = new PluginPlatformFixture(options => + { + options.MaximumStagedPackages = 4; + options.MaximumPackageBytes = packageBytes + 1_024; + options.MaximumStagedPackageBytes = packageBytes + 32; + }); + await byteFixture.PreviewAsync(firstManifest, PingScript); + var byteError = await Assert.ThrowsExactlyAsync(() => + byteFixture.PreviewAsync(Manifest("test.stage-two"), PingScript)); + StringAssert.Contains(byteError.Message, "staging byte limit"); + Assert.HasCount(1, Directory.EnumerateFiles( + Path.Combine(byteFixture.RootPath, "staging"), "*.sdwpkg").ToArray()); + } + + [TestMethod] + public async Task Manifest_RejectsProviderWithoutCapabilityAndInvalidHandlerBeforeInstall() + { + await using var fixture = new PluginPlatformFixture(); + var missingCapability = Manifest("test.notification-capability") with + { + Providers = [new PluginProviderDeclaration + { + Kind = "notification", + Name = "notifier", + Handlers = new Dictionary { ["send"] = "sendNotification" } + }] + }; + var capabilityError = await Assert.ThrowsExactlyAsync(() => + fixture.PreviewAsync(missingCapability, PingScript)); + StringAssert.Contains(capabilityError.Message, "notifications capability"); + + var invalidHandler = missingCapability with + { + Capabilities = new PluginCapabilities { Notifications = true }, + Providers = [missingCapability.Providers[0] with + { + Handlers = new Dictionary { ["send"] = "../escape" } + }] + }; + var handlerError = await Assert.ThrowsExactlyAsync(() => + fixture.PreviewAsync(invalidHandler, PingScript)); + StringAssert.Contains(handlerError.Message, "invalid handler name"); + } + + [TestMethod] + public async Task Worker_DeniesUnapprovedNetworkAndFiles_AndDoesNotExposeClrOrWebGlobals() + { + const string script = """ + 'use strict'; + globalThis.sdwPlugin = { handlers: { + network() { return sdw.request('network.request', { method: 'GET', url: 'https://example.com/' }); }, + file() { return sdw.request('file.read', { path: '/etc/passwd' }); }, + inspect() { return { + requireType: typeof require, + fetchType: typeof fetch, + exposedHostObjects: Object.getOwnPropertyNames(globalThis).filter(name => { + try { return globalThis[name] && typeof globalThis[name].GetType === 'function'; } catch { return false; } + }) + }; }, + catchDenied() { + try { sdw.request('network.request', { method: 'GET', url: 'https://example.com/' }); } + catch (error) { return { + hostExceptionType: typeof error.hostException, + getTypeType: typeof error.GetType, + constructorName: error.constructor && error.constructor.name + }; } + } + }}; + """; + await using var fixture = new PluginPlatformFixture(); + var manifest = Manifest("test.denied"); + await fixture.InstallAndEnableAsync(manifest, script); + + var network = await Assert.ThrowsExactlyAsync(() => + fixture.InvokeAsync(manifest.Id, "network")); + StringAssert.Contains(network.Message, "not approved"); + var file = await Assert.ThrowsExactlyAsync(() => + fixture.InvokeAsync(manifest.Id, "file")); + StringAssert.Contains(file.Message, "No file roots"); + + var inspected = await fixture.InvokeAsync(manifest.Id, "inspect"); + Assert.AreEqual("undefined", inspected.GetProperty("requireType").GetString()); + Assert.AreEqual("undefined", inspected.GetProperty("fetchType").GetString()); + Assert.AreEqual(0, inspected.GetProperty("exposedHostObjects").GetArrayLength()); + var caught = await fixture.InvokeAsync(manifest.Id, "catchDenied"); + Assert.AreEqual("undefined", caught.GetProperty("hostExceptionType").GetString()); + Assert.AreEqual("undefined", caught.GetProperty("getTypeType").GetString()); + Assert.AreEqual("Error", caught.GetProperty("constructorName").GetString()); + } + + [TestMethod] + [DataRow("127.0.0.1")] + [DataRow("169.254.169.254")] + [DataRow("192.168.1.10")] + [DataRow("192.88.99.2")] + [DataRow("64:ff9b::a9fe:a9fe")] + [DataRow("::192.168.1.10")] + [DataRow("::ffff:0:127.0.0.1")] + [DataRow("::ffff:0:169.254.169.254")] + [DataRow("100:0:0:1::1")] + [DataRow("2001:5::1")] + [DataRow("2001:10::1")] + [DataRow("3fff::1")] + [DataRow("5f00::1")] + [DataRow("fec0::1")] + [DataRow("fe00::1")] + [DataRow("4000::1")] + public async Task NetworkCapability_RejectsApprovedHostResolvingToNonPublicAddress(string address) + { + using var networkHandler = PluginNetworkConnectionFactory.Create( + new FixedDnsResolver(IPAddress.Parse(address))); + await using var fixture = new PluginPlatformFixture(httpHandler: networkHandler); + const string script = """ + globalThis.sdwPlugin={handlers:{request:()=>sdw.request('network.request', + {method:'GET',url:'http://approved.example/resource'})}}; + """; + var manifest = Manifest("test.ssrf", capabilities: new PluginCapabilities + { + NetworkDomains = ["approved.example"] + }); + await fixture.InstallAndEnableAsync(manifest, script); + + var error = await Assert.ThrowsExactlyAsync(() => + fixture.InvokeAsync(manifest.Id, "request")); + StringAssert.Contains(error.Message, "non-public address"); + } + + [TestMethod] + public async Task NetworkCapability_RejectsMixedPublicAndPrivateDnsAnswers() + { + using var networkHandler = PluginNetworkConnectionFactory.Create( + new FixedDnsResolver(IPAddress.Parse("2606:4700:4700::1111"), IPAddress.Loopback)); + await using var fixture = new PluginPlatformFixture(httpHandler: networkHandler); + const string script = """ + globalThis.sdwPlugin={handlers:{request:()=>sdw.request('network.request', + {method:'GET',url:'http://approved.example/resource'})}}; + """; + var manifest = Manifest("test.mixed-dns", capabilities: new PluginCapabilities + { + NetworkDomains = ["approved.example"] + }); + await fixture.InstallAndEnableAsync(manifest, script); + + var error = await Assert.ThrowsExactlyAsync(() => + fixture.InvokeAsync(manifest.Id, "request")); + StringAssert.Contains(error.Message, "non-public address"); + } + + [TestMethod] + [DataRow("2606:4700:4700::1111")] + [DataRow("2001:4860:4860::8888")] + public void NetworkCapability_AcceptsOrdinaryGlobalUnicastAddress(string address) + => Assert.IsTrue(PluginNetworkConnectionFactory.IsPublicAddress(IPAddress.Parse(address))); + + [TestMethod] + public async Task SafeFileAccess_RejectsDirectoryToSymlinkSwapBetweenApprovalAndOpen() + { + if (!OperatingSystem.IsLinux() && !OperatingSystem.IsMacOS()) + { + Assert.Inconclusive("The deterministic openat race test requires a POSIX host."); + return; + } + var sandbox = Path.Combine(Path.GetTempPath(), $"sdw-plugin-race-{Guid.NewGuid():N}"); + var approved = Directory.CreateDirectory(Path.Combine(sandbox, "approved")).FullName; + var live = Directory.CreateDirectory(Path.Combine(approved, "live")).FullName; + var parked = Path.Combine(approved, "parked"); + var outside = Directory.CreateDirectory(Path.Combine(sandbox, "outside")).FullName; + var target = Path.Combine(live, "secret.txt"); + var outsideSecret = Path.Combine(outside, "secret.txt"); + await File.WriteAllTextAsync(target, "approved"); + await File.WriteAllTextAsync(outsideSecret, "outside secret"); + var access = new PluginSafeFileAccess(); + var swapped = 0; + access.BeforeOpenForTesting = () => + { + if (Interlocked.Exchange(ref swapped, 1) != 0) return; + Directory.Move(live, parked); + Directory.CreateSymbolicLink(live, outside); + }; + try + { + await Assert.ThrowsExactlyAsync(() => + access.ReadAsync(approved, target, 1_024, CancellationToken.None)); + Assert.AreEqual("outside secret", await File.ReadAllTextAsync(outsideSecret)); + } + finally + { + if (Directory.Exists(live) || File.Exists(live)) Directory.Delete(live); + if (Directory.Exists(sandbox)) Directory.Delete(sandbox, recursive: true); + } + } + + [TestMethod] + public async Task PluginDataQuota_RejectsGrowthAtomicallyAndPreservesExistingFile() + { + var (_, storageScript) = LoadExample("scoped-storage"); + await using var fixture = new PluginPlatformFixture(options => + { + options.MaximumPluginDataBytes = 1_024; + options.MaximumPluginDataFiles = 1; + options.MaximumPluginDataPathDepth = 2; + options.CircuitBreakerFailures = 20; + }); + var manifest = StorageManifest("test.quota"); + await fixture.InstallAndEnableAsync(manifest, storageScript); + var original = Enumerable.Repeat((byte)0x41, 700).ToArray(); + await fixture.InvokeAsync(manifest.Id, "seed", new + { + path = "state.bin", + base64 = Convert.ToBase64String(original) + }); + + var extraError = await Assert.ThrowsExactlyAsync(() => + fixture.InvokeAsync(manifest.Id, "seed", new + { + path = "extra.bin", + base64 = Convert.ToBase64String(new byte[400]) + })); + StringAssert.Contains(extraError.Message, "quota"); + var overwriteError = await Assert.ThrowsExactlyAsync(() => + fixture.InvokeAsync(manifest.Id, "seed", new + { + path = "state.bin", + base64 = Convert.ToBase64String(new byte[1_100]) + })); + StringAssert.Contains(overwriteError.Message, "quota"); + await Assert.ThrowsExactlyAsync(() => + fixture.InvokeAsync(manifest.Id, "seed", new + { + path = "one/two/three.bin", + base64 = Convert.ToBase64String(new byte[1]) + })); + + var dataRoot = Path.Combine(fixture.RootPath, "data", manifest.Id); + CollectionAssert.AreEqual(original, await File.ReadAllBytesAsync(Path.Combine(dataRoot, "state.bin"))); + Assert.IsFalse(File.Exists(Path.Combine(dataRoot, "extra.bin"))); + } + + [TestMethod] + public async Task Worker_ContainsTimeoutCrashAndResourceExhaustion_ThenOpensCircuit() + { + const string hostileScript = """ + 'use strict'; + globalThis.sdwPlugin = { handlers: { + timeout() { while (true) {} }, + crash() { throw new Error('intentional crash'); }, + memory() { const values = []; while (true) values.push(new ArrayBuffer(1048576)); } + }}; + """; + await using var fixture = new PluginPlatformFixture(options => + { + options.InvocationTimeoutMilliseconds = 400; + options.MaximumWorkerCpuMilliseconds = 300; + options.MaximumWorkerMemoryMegabytes = 64; + options.MaximumConcurrentWorkers = 1; + options.MaximumConcurrentWorkersPerPlugin = 1; + options.CircuitBreakerFailures = 3; + }); + var hostile = Manifest("test.hostile"); + await fixture.InstallAndEnableAsync(hostile, hostileScript); + var healthy = Manifest("test.healthy"); + await fixture.InstallAndEnableAsync(healthy, PingScript); + var stopwatch = Stopwatch.StartNew(); + + var firstTimeout = fixture.InvokeAsync(hostile.Id, "timeout"); + await Task.Delay(100); + var rejected = await Task.WhenAll(Enumerable.Range(0, 20).Select(async _ => + { + try + { + await fixture.InvokeAsync(hostile.Id, "timeout"); + return null; + } + catch (Exception exception) + { + return exception; + } + })); + Assert.IsTrue(rejected.All(exception => exception is PluginCapacityExceededException)); + await Assert.ThrowsExactlyAsync(() => + fixture.InvokeAsync(healthy.Id, "ping", new { value = 1 })); + await Assert.ThrowsExactlyAsync(() => firstTimeout); + var healthAfterCapacity = (await fixture.Manager.GetAllAsync(CancellationToken.None)) + .Single(plugin => plugin.Manifest.Id == hostile.Id).Health; + Assert.AreEqual(1, healthAfterCapacity.ConsecutiveFailures, + "Capacity rejections must not degrade plugin health."); + var healthyHealth = (await fixture.Manager.GetAllAsync(CancellationToken.None)) + .Single(plugin => plugin.Manifest.Id == healthy.Id).Health; + Assert.AreEqual(0, healthyHealth.ConsecutiveFailures, + "A global capacity rejection must not be attributed to another plugin."); + await Assert.ThrowsExactlyAsync(() => fixture.InvokeAsync(hostile.Id, "crash")); + await Assert.ThrowsAsync(() => fixture.InvokeAsync(hostile.Id, "memory")); + Assert.IsLessThan(TimeSpan.FromSeconds(8), stopwatch.Elapsed); + + var circuit = await Assert.ThrowsExactlyAsync(() => + fixture.InvokeAsync(hostile.Id, "crash")); + StringAssert.Contains(circuit.Message, "circuit is open"); + + var result = await fixture.InvokeAsync(healthy.Id, "ping", new { value = 7 }); + Assert.AreEqual(7, result.GetProperty("value").GetInt32()); + } + + [TestMethod] + public async Task NotificationAndStorageExamples_PassProviderIntegrationFlow() + { + var http = new RecordingHttpMessageHandler(); + await using var fixture = new PluginPlatformFixture(httpHandler: http); + + var (webhook, webhookScript) = LoadExample("webhook"); + await fixture.InstallAndEnableAsync(webhook, webhookScript); + await fixture.Manager.UpdateConfigurationAsync( + webhook.Id, + JsonSerializer.SerializeToElement(new { url = "https://hooks.example.com/sdw" }), + CancellationToken.None); + + var (storage, storageScript) = LoadExample("scoped-storage"); + await fixture.InstallAndEnableAsync(storage, storageScript); + + var registry = new PluginProviderRegistry(fixture.Manager); + var notificationProvider = registry.GetNotificationProviders().Single(); + Assert.AreEqual("plugin:example.webhook:webhook", notificationProvider.Name); + await notificationProvider.SendAsync(new PluginNotification("Ready", "Library scan completed"), + CancellationToken.None); + Assert.AreEqual("https://hooks.example.com/sdw", http.LastRequestUri?.ToString()); + StringAssert.Contains(http.LastBody ?? string.Empty, "Library scan completed"); + + var bytes = Encoding.UTF8.GetBytes("isolated storage"); + await fixture.Manager.InvokeAsync(storage.Id, "seed", JsonSerializer.SerializeToElement(new + { + path = "folder/item.txt", + base64 = Convert.ToBase64String(bytes) + }), CancellationToken.None); + var fileStore = registry.GetFileStores().Single(); + Assert.AreEqual("plugin:example.scoped-storage:example-scoped", fileStore.Name); + Assert.IsTrue(await fileStore.ExistAsync("folder/item.txt", CancellationToken.None)); + var info = await fileStore.FileInfoAsync("folder/item.txt", CancellationToken.None); + Assert.AreEqual(bytes.Length, info.Length); + await using var stream = await fileStore.OpenReadStreamAsync("folder/item.txt", CancellationToken.None); + using var reader = new StreamReader(stream); + Assert.AreEqual("isolated storage", await reader.ReadToEndAsync()); + var listed = await fileStore.EnumerateDirectory("folder").ToListAsync(); + Assert.HasCount(1, listed); + Assert.AreEqual("item.txt", listed[0].FileName); + } + + [TestMethod] + public async Task ProviderIdentity_IsQualifiedAndStableAcrossDisableUninstallAndReinstall() + { + var (_, storageScript) = LoadExample("scoped-storage"); + await using var fixture = new PluginPlatformFixture(); + var first = StorageManifest("test.provider-one", "shared"); + var second = StorageManifest("test.provider-two", "shared"); + await fixture.InstallAndEnableAsync(first, storageScript); + await fixture.InstallAndEnableAsync(second, storageScript); + var registry = new PluginProviderRegistry(fixture.Manager); + + CollectionAssert.AreEquivalent( + new[] { "plugin:test.provider-one:shared", "plugin:test.provider-two:shared" }, + registry.GetFileStores().Select(store => store.Name).ToArray()); + Assert.IsFalse(registry.GetFileStores().Any(store => store.Name == "local")); + + await fixture.Manager.DisableAsync(first.Id, CancellationToken.None); + CollectionAssert.AreEqual( + new[] { "plugin:test.provider-two:shared" }, + registry.GetFileStores().Select(store => store.Name).ToArray()); + + await fixture.Manager.UninstallAsync(first.Id, deleteData: false, CancellationToken.None); + await fixture.InstallAndEnableAsync(first, storageScript); + Assert.IsTrue(registry.GetFileStores().Any(store => + store.Name == "plugin:test.provider-one:shared")); + } + + [TestMethod] + public async Task DeleteDataUninstall_DoesNotDeleteAnotherPluginWithMigrationLikeId() + { + var (_, storageScript) = LoadExample("scoped-storage"); + await using var fixture = new PluginPlatformFixture(); + var removed = StorageManifest("test.a"); + var neighbor = StorageManifest("test.a.migration-b"); + await fixture.InstallAndEnableAsync(removed, storageScript); + await fixture.InstallAndEnableAsync(neighbor, storageScript); + await fixture.InvokeAsync(neighbor.Id, "seed", new + { + path = "state.txt", + base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes("neighbor")) + }); + + await fixture.Manager.UninstallAsync(removed.Id, deleteData: true, CancellationToken.None); + + var exists = await fixture.InvokeAsync(neighbor.Id, "exists", new { path = "state.txt" }); + Assert.IsTrue(exists.GetProperty("exists").GetBoolean()); + Assert.IsTrue(Directory.Exists(Path.Combine(fixture.RootPath, "data", neighbor.Id))); + } + + [TestMethod] + public async Task UpgradeAndUninstall_ApplyExplicitDataAndConfigurationStrategy() + { + const string storageScript = """ + 'use strict'; + globalThis.sdwPlugin = { handlers: { + config(input, configuration) { return configuration; }, + seed(input) { return sdw.request('data.write', input); }, + exists(input) { return sdw.request('data.exists', input); } + }}; + """; + await using var fixture = new PluginPlatformFixture(); + var versionOne = StorageManifest("test.lifecycle"); + await fixture.InstallAndEnableAsync(versionOne, storageScript); + await fixture.Manager.UpdateConfigurationAsync(versionOne.Id, + JsonSerializer.SerializeToElement(new { marker = "preserved" }), CancellationToken.None); + await fixture.Manager.InvokeAsync(versionOne.Id, "seed", JsonSerializer.SerializeToElement(new + { + path = "state.txt", + base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes("old")) + }), CancellationToken.None); + + await fixture.Manager.UninstallAsync(versionOne.Id, deleteData: false, CancellationToken.None); + await fixture.InstallAndEnableAsync(versionOne, storageScript); + var restoredConfig = await fixture.InvokeAsync(versionOne.Id, "config"); + Assert.AreEqual("preserved", restoredConfig.GetProperty("marker").GetString()); + var retained = await fixture.InvokeAsync(versionOne.Id, "exists", new { path = "state.txt" }); + Assert.IsTrue(retained.GetProperty("exists").GetBoolean()); + + var unsafeUpgrade = versionOne with { Version = "2.0.0", DataVersion = 2 }; + var unsafePreview = await fixture.PreviewAsync(unsafeUpgrade, storageScript); + var migrationError = await Assert.ThrowsExactlyAsync(() => + fixture.Manager.UpgradeAsync(versionOne.Id, unsafePreview.Token, unsafePreview.PackageSha256, + unsafePreview.Manifest.Capabilities, CancellationToken.None)); + StringAssert.Contains(migrationError.Message, "dataMigration.strategy = 'reset'"); + + var resetUpgrade = unsafeUpgrade with + { + DataMigration = new PluginDataMigration + { + Strategy = "reset", + Description = "Version 2 intentionally starts with an empty cache." + } + }; + var resetPreview = await fixture.PreviewAsync(resetUpgrade, storageScript); + await fixture.Manager.UpgradeAsync(versionOne.Id, resetPreview.Token, resetPreview.PackageSha256, + resetPreview.Manifest.Capabilities, CancellationToken.None); + await fixture.Manager.EnableAsync(versionOne.Id, CancellationToken.None); + var reset = await fixture.InvokeAsync(versionOne.Id, "exists", new { path = "state.txt" }); + Assert.IsFalse(reset.GetProperty("exists").GetBoolean()); + var preservedConfig = await fixture.InvokeAsync(versionOne.Id, "config"); + Assert.AreEqual("preserved", preservedConfig.GetProperty("marker").GetString()); + } + + [TestMethod] + public async Task UpgradeAndUninstall_CancelAndDrainOldWorkersBeforeMovingDataOrPackage() + { + var (_, baseStorageScript) = LoadExample("scoped-storage"); + var slowStorageScript = baseStorageScript + """ + + globalThis.sdwPlugin.handlers.slowSeed = function(input) { + const deadline = Date.now() + 3000; + while (Date.now() < deadline) {} + return sdw.request('data.write', input); + }; + """; + await using var fixture = new PluginPlatformFixture(options => + { + options.InvocationTimeoutMilliseconds = 8_000; + options.MaximumWorkerCpuMilliseconds = 7_000; + }); + var versionOne = StorageManifest("test.lifecycle-drain"); + await fixture.InstallAndEnableAsync(versionOne, slowStorageScript); + await fixture.InvokeAsync(versionOne.Id, "seed", new + { + path = "old.txt", + base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes("old")) + }); + + var oldInvocation = fixture.InvokeAsync(versionOne.Id, "slowSeed", new + { + path = "stale-after-upgrade.txt", + base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes("stale")) + }); + await Task.Delay(150); + var versionTwo = versionOne with + { + Version = "2.0.0", + DataVersion = 2, + DataMigration = new PluginDataMigration { Strategy = "reset" } + }; + var preview = await fixture.PreviewAsync(versionTwo, slowStorageScript); + await fixture.Manager.UpgradeAsync(versionOne.Id, preview.Token, preview.PackageSha256, + preview.Manifest.Capabilities, CancellationToken.None); + await Assert.ThrowsExactlyAsync(() => oldInvocation); + await fixture.Manager.EnableAsync(versionOne.Id, CancellationToken.None); + Assert.IsFalse(File.Exists(Path.Combine( + fixture.RootPath, "data", versionOne.Id, "stale-after-upgrade.txt"))); + + var uninstallInvocation = fixture.InvokeAsync(versionOne.Id, "slowSeed", new + { + path = "stale-after-uninstall.txt", + base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes("stale")) + }); + await Task.Delay(150); + await fixture.Manager.UninstallAsync(versionOne.Id, deleteData: true, CancellationToken.None); + await Assert.ThrowsExactlyAsync(() => uninstallInvocation); + Assert.IsFalse(Directory.Exists(Path.Combine(fixture.RootPath, "data", versionOne.Id))); + Assert.IsFalse(Directory.Exists(Path.Combine( + fixture.RootPath, "packages", versionOne.Id, versionTwo.Version))); + } + + [TestMethod] + public async Task CascadingDisableAndUninstall_CancelDependentWorkersBeforeReturning() + { + var (_, baseStorageScript) = LoadExample("scoped-storage"); + var slowStorageScript = baseStorageScript + """ + + globalThis.sdwPlugin.handlers.slowSeed = function(input) { + const deadline = Date.now() + 3000; + while (Date.now() < deadline) {} + return sdw.request('data.write', input); + }; + """; + await using var fixture = new PluginPlatformFixture(options => + { + options.InvocationTimeoutMilliseconds = 8_000; + options.MaximumWorkerCpuMilliseconds = 7_000; + }); + var dependency = Manifest("test.cascade-root"); + var dependent = StorageManifest("test.cascade-dependent") with + { + Dependencies = [new PluginDependency(dependency.Id, dependency.Version)] + }; + await fixture.InstallAndEnableAsync(dependency, PingScript); + await fixture.InstallAndEnableAsync(dependent, slowStorageScript); + + var disableInvocation = fixture.InvokeAsync(dependent.Id, "slowSeed", new + { + path = "stale-after-disable.txt", + base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes("stale")) + }); + await Task.Delay(150); + await fixture.Manager.DisableAsync(dependency.Id, CancellationToken.None); + await Assert.ThrowsExactlyAsync(() => disableInvocation); + Assert.IsFalse((await fixture.Manager.GetAllAsync(CancellationToken.None)) + .Single(plugin => plugin.Manifest.Id == dependent.Id).IsEnabled); + Assert.IsFalse(File.Exists(Path.Combine( + fixture.RootPath, "data", dependent.Id, "stale-after-disable.txt"))); + + await fixture.Manager.EnableAsync(dependency.Id, CancellationToken.None); + await fixture.Manager.EnableAsync(dependent.Id, CancellationToken.None); + var uninstallInvocation = fixture.InvokeAsync(dependent.Id, "slowSeed", new + { + path = "stale-after-uninstall.txt", + base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes("stale")) + }); + await Task.Delay(150); + await fixture.Manager.UninstallAsync(dependency.Id, deleteData: true, CancellationToken.None); + await Assert.ThrowsExactlyAsync(() => uninstallInvocation); + Assert.IsFalse((await fixture.Manager.GetAllAsync(CancellationToken.None)) + .Single(plugin => plugin.Manifest.Id == dependent.Id).IsEnabled); + Assert.IsFalse(File.Exists(Path.Combine( + fixture.RootPath, "data", dependent.Id, "stale-after-uninstall.txt"))); + } + + [TestMethod] + public async Task StartupRecovery_RollsBackPreparedUninstall() + { + var (_, storageScript) = LoadExample("scoped-storage"); + await using var fixture = new PluginPlatformFixture(); + var manifest = StorageManifest("test.recover-uninstall-prepared"); + var unaffected = Manifest("test.recover-unaffected"); + await fixture.InstallAndEnableAsync(manifest, storageScript); + await fixture.InstallAndEnableAsync(unaffected, PingScript); + await fixture.InvokeAsync(manifest.Id, "seed", new + { + path = "state.txt", + base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes("preserve")) + }); + fixture.Manager.LifecycleCheckpointForTesting = checkpoint => + { + if (checkpoint == PluginLifecycleCheckpoint.AfterMove) + throw new PluginProcessCrashSimulationException(); + }; + + await Assert.ThrowsExactlyAsync(() => + fixture.Manager.UninstallAsync(manifest.Id, deleteData: true, CancellationToken.None)); + GetSingleLifecycleTransaction(fixture.RootPath, "uninstall", manifest.Id); + Assert.IsFalse(Directory.Exists(Path.Combine( + fixture.RootPath, "packages", manifest.Id, manifest.Version))); + Assert.IsFalse(Directory.Exists(Path.Combine(fixture.RootPath, "data", manifest.Id))); + var overlapError = await Assert.ThrowsExactlyAsync(() => + fixture.Manager.UninstallAsync(unaffected.Id, deleteData: true, CancellationToken.None)); + StringAssert.Contains(overlapError.Message, "pending lifecycle recovery"); + var invokeError = await Assert.ThrowsExactlyAsync(() => + fixture.InvokeAsync(unaffected.Id, "ping", new { value = 1 })); + StringAssert.Contains(invokeError.Message, "pending lifecycle recovery"); + + var restarted = fixture.CreateRestartedManager(); + var restartedPlugins = await restarted.GetAllAsync(CancellationToken.None); + var restored = restartedPlugins.Single(plugin => + plugin.Manifest.Id == manifest.Id); + Assert.AreEqual(manifest.Version, restored.Manifest.Version); + Assert.IsTrue(restored.IsEnabled); + Assert.IsTrue(restartedPlugins.Single(plugin => plugin.Manifest.Id == unaffected.Id).IsEnabled); + var exists = await restarted.InvokeAsync(manifest.Id, "exists", + JsonSerializer.SerializeToElement(new { path = "state.txt" }), CancellationToken.None); + Assert.IsTrue(exists.GetProperty("exists").GetBoolean()); + AssertNoLifecycleTransactions(fixture.RootPath); + } + + [TestMethod] + public async Task StartupRecovery_FinalizesCommittedUninstallWithPartiallyMissingPayload() + { + var (_, storageScript) = LoadExample("scoped-storage"); + await using var fixture = new PluginPlatformFixture(); + var manifest = StorageManifest("test.recover-uninstall-committed"); + await fixture.InstallAndEnableAsync(manifest, storageScript); + await fixture.InvokeAsync(manifest.Id, "seed", new + { + path = "state.txt", + base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes("delete")) + }); + fixture.Manager.LifecycleCheckpointForTesting = checkpoint => + { + if (checkpoint != PluginLifecycleCheckpoint.AfterCommit) return; + var transactionPath = GetSingleLifecycleTransaction( + fixture.RootPath, "uninstall", manifest.Id); + Directory.Delete(Path.Combine(transactionPath, "package"), recursive: true); + File.Delete(Path.Combine(transactionPath, "data", "state.txt")); + throw new PluginProcessCrashSimulationException(); + }; + + await Assert.ThrowsExactlyAsync(() => + fixture.Manager.UninstallAsync(manifest.Id, deleteData: true, CancellationToken.None)); + var pendingPreview = await fixture.PreviewAsync(manifest, storageScript); + var pendingError = await Assert.ThrowsExactlyAsync(() => + fixture.Manager.InstallPackageAsync(pendingPreview.Token, pendingPreview.PackageSha256, + pendingPreview.Manifest.Capabilities, CancellationToken.None)); + StringAssert.Contains(pendingError.Message, "pending lifecycle recovery"); + + var restarted = fixture.CreateRestartedManager(); + Assert.IsEmpty(await restarted.GetAllAsync(CancellationToken.None)); + Assert.IsFalse(Directory.Exists(Path.Combine( + fixture.RootPath, "packages", manifest.Id, manifest.Version))); + Assert.IsFalse(Directory.Exists(Path.Combine(fixture.RootPath, "data", manifest.Id))); + AssertNoLifecycleTransactions(fixture.RootPath); + } + + [TestMethod] + public async Task StartupRecovery_RollsBackPreparedResetUpgrade() + { + var (_, storageScript) = LoadExample("scoped-storage"); + await using var fixture = new PluginPlatformFixture(); + var versionOne = StorageManifest("test.recover-upgrade-prepared"); + await fixture.InstallAndEnableAsync(versionOne, storageScript); + await fixture.InvokeAsync(versionOne.Id, "seed", new + { + path = "state.txt", + base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes("preserve")) + }); + var versionTwo = versionOne with + { + Version = "2.0.0", + DataVersion = 2, + DataMigration = new PluginDataMigration { Strategy = "reset" } + }; + var preview = await fixture.PreviewAsync(versionTwo, storageScript); + fixture.Manager.LifecycleCheckpointForTesting = checkpoint => + { + if (checkpoint == PluginLifecycleCheckpoint.AfterMove) + throw new PluginProcessCrashSimulationException(); + }; + + await Assert.ThrowsExactlyAsync(() => + fixture.Manager.UpgradeAsync(versionOne.Id, preview.Token, preview.PackageSha256, + preview.Manifest.Capabilities, CancellationToken.None)); + GetSingleLifecycleTransaction(fixture.RootPath, "upgrade", versionOne.Id); + Assert.IsFalse(Directory.Exists(Path.Combine(fixture.RootPath, "data", versionOne.Id))); + + var restarted = fixture.CreateRestartedManager(); + var restored = (await restarted.GetAllAsync(CancellationToken.None)).Single(plugin => + plugin.Manifest.Id == versionOne.Id); + Assert.AreEqual(versionOne.Version, restored.Manifest.Version); + Assert.IsTrue(restored.IsEnabled); + var exists = await restarted.InvokeAsync(versionOne.Id, "exists", + JsonSerializer.SerializeToElement(new { path = "state.txt" }), CancellationToken.None); + Assert.IsTrue(exists.GetProperty("exists").GetBoolean()); + Assert.IsFalse(Directory.Exists(Path.Combine( + fixture.RootPath, "packages", versionOne.Id, versionTwo.Version))); + AssertNoLifecycleTransactions(fixture.RootPath); + } + + [TestMethod] + public async Task StartupRecovery_FinalizesCommittedResetUpgradeAndBlocksOverlappingLifecycle() + { + var (_, storageScript) = LoadExample("scoped-storage"); + await using var fixture = new PluginPlatformFixture(); + var versionOne = StorageManifest("test.recover-upgrade-committed"); + await fixture.InstallAndEnableAsync(versionOne, storageScript); + await fixture.InvokeAsync(versionOne.Id, "seed", new + { + path = "state.txt", + base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes("delete")) + }); + var versionTwo = versionOne with + { + Version = "2.0.0", + DataVersion = 2, + DataMigration = new PluginDataMigration { Strategy = "reset" } + }; + var preview = await fixture.PreviewAsync(versionTwo, storageScript); + fixture.Manager.LifecycleCheckpointForTesting = checkpoint => + { + if (checkpoint != PluginLifecycleCheckpoint.AfterCommit) return; + var transactionPath = GetSingleLifecycleTransaction( + fixture.RootPath, "upgrade", versionOne.Id); + File.Delete(Path.Combine(transactionPath, "data", "state.txt")); + throw new IOException("Simulated committed-cleanup interruption."); + }; + + await Assert.ThrowsExactlyAsync(() => fixture.Manager.UpgradeAsync( + versionOne.Id, preview.Token, preview.PackageSha256, + preview.Manifest.Capabilities, CancellationToken.None)); + var pendingInvocation = await Assert.ThrowsExactlyAsync(() => + fixture.InvokeAsync(versionOne.Id, "exists", new { path = "state.txt" })); + StringAssert.Contains(pendingInvocation.Message, "pending lifecycle recovery"); + var pendingUninstall = await Assert.ThrowsExactlyAsync(() => + fixture.Manager.UninstallAsync(versionOne.Id, deleteData: true, CancellationToken.None)); + StringAssert.Contains(pendingUninstall.Message, "pending lifecycle recovery"); + var versionThree = versionTwo with { Version = "3.0.0" }; + var versionThreePreview = await fixture.PreviewAsync(versionThree, storageScript); + var pendingUpgrade = await Assert.ThrowsExactlyAsync(() => + fixture.Manager.UpgradeAsync(versionOne.Id, versionThreePreview.Token, + versionThreePreview.PackageSha256, versionThreePreview.Manifest.Capabilities, + CancellationToken.None)); + StringAssert.Contains(pendingUpgrade.Message, "pending lifecycle recovery"); + GetSingleLifecycleTransaction(fixture.RootPath, "upgrade", versionOne.Id); + + var restarted = fixture.CreateRestartedManager(); + var installed = (await restarted.GetAllAsync(CancellationToken.None)).Single(plugin => + plugin.Manifest.Id == versionOne.Id); + Assert.AreEqual(versionTwo.Version, installed.Manifest.Version); + Assert.IsFalse(installed.IsEnabled); + Assert.IsFalse(Directory.Exists(Path.Combine(fixture.RootPath, "data", versionOne.Id))); + Assert.IsFalse(Directory.Exists(Path.Combine( + fixture.RootPath, "packages", versionOne.Id, versionOne.Version))); + Assert.IsTrue(Directory.Exists(Path.Combine( + fixture.RootPath, "packages", versionOne.Id, versionTwo.Version))); + AssertNoLifecycleTransactions(fixture.RootPath); + + await restarted.UninstallAsync(versionOne.Id, deleteData: true, CancellationToken.None); + var restartedAgain = fixture.CreateRestartedManager(); + Assert.IsEmpty(await restartedAgain.GetAllAsync(CancellationToken.None)); + Assert.IsFalse(Directory.Exists(Path.Combine( + fixture.RootPath, "packages", versionOne.Id, versionTwo.Version))); + AssertNoLifecycleTransactions(fixture.RootPath); + } + + [TestMethod] + public async Task Invoke_AcquiresLifecycleLeaseBeforeManagementGateCanUninstallStaleEntry() + { + const string slowScript = """ + globalThis.sdwPlugin={handlers:{slow:()=>{const end=Date.now()+3000;while(Date.now() + { + options.InvocationTimeoutMilliseconds = 8_000; + options.MaximumWorkerCpuMilliseconds = 7_000; + }); + var manifest = Manifest("test.invoke-ordering"); + await fixture.InstallAndEnableAsync(manifest, slowScript); + using var reachedLeasePoint = new ManualResetEventSlim(); + using var releaseLeasePoint = new ManualResetEventSlim(); + fixture.Manager.BeforeInvocationLeaseForTesting = () => + { + fixture.Manager.BeforeInvocationLeaseForTesting = null; + reachedLeasePoint.Set(); + releaseLeasePoint.Wait(TimeSpan.FromSeconds(5)); + }; + + var invocation = Task.Run(() => fixture.InvokeAsync(manifest.Id, "slow")); + Assert.IsTrue(reachedLeasePoint.Wait(TimeSpan.FromSeconds(2))); + var uninstall = fixture.Manager.UninstallAsync(manifest.Id, deleteData: true, CancellationToken.None); + await Task.Delay(100); + Assert.IsFalse(uninstall.IsCompleted, + "Uninstall must not pass the management gate before the invocation owns its lifecycle lease."); + releaseLeasePoint.Set(); + await uninstall; + await Assert.ThrowsExactlyAsync(() => invocation); + Assert.IsFalse(Directory.Exists(Path.Combine( + fixture.RootPath, "packages", manifest.Id, manifest.Version))); + } + + [TestMethod] + public async Task Upgrade_WhenPostCatalogStepFails_RestoresOldCatalogPackageAndSnapshot() + { + const string versionOneScript = "globalThis.sdwPlugin={handlers:{version:()=>({value:1})}};"; + const string versionTwoScript = "globalThis.sdwPlugin={handlers:{version:()=>({value:2})}};"; + await using var fixture = new PluginPlatformFixture(enableFailureInjection: true); + var versionOne = Manifest("test.rollback", version: "1.0.0"); + await fixture.InstallAndEnableAsync(versionOne, versionOneScript); + fixture.FailingRepository!.FailNextRetainedRemoval = true; + var versionTwo = versionOne with { Version = "2.0.0" }; + var preview = await fixture.PreviewAsync(versionTwo, versionTwoScript); + + await Assert.ThrowsExactlyAsync(() => fixture.Manager.UpgradeAsync( + versionOne.Id, + preview.Token, + preview.PackageSha256, + preview.Manifest.Capabilities, + CancellationToken.None)); + + var installed = (await fixture.Manager.GetAllAsync(CancellationToken.None)).Single(plugin => + plugin.Manifest.Id == versionOne.Id); + Assert.AreEqual("1.0.0", installed.Manifest.Version); + Assert.IsTrue(installed.IsEnabled); + var invoked = await fixture.InvokeAsync(versionOne.Id, "version"); + Assert.AreEqual(1, invoked.GetProperty("value").GetInt32()); + Assert.IsTrue(Directory.Exists(Path.Combine( + fixture.RootPath, "packages", versionOne.Id, "1.0.0"))); + Assert.IsFalse(Directory.Exists(Path.Combine( + fixture.RootPath, "packages", versionOne.Id, "2.0.0"))); + } + + [TestMethod] + public async Task Uninstall_WhenCatalogRemovalFails_RestoresPackageDataCatalogAndSnapshot() + { + var (_, storageScript) = LoadExample("scoped-storage"); + await using var fixture = new PluginPlatformFixture(enableFailureInjection: true); + var manifest = StorageManifest("test.uninstall-rollback"); + await fixture.InstallAndEnableAsync(manifest, storageScript); + await fixture.InvokeAsync(manifest.Id, "seed", new + { + path = "state.txt", + base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes("preserve me")) + }); + fixture.FailingRepository!.FailNextCatalogRemoval = true; + + await Assert.ThrowsExactlyAsync(() => + fixture.Manager.UninstallAsync(manifest.Id, deleteData: true, CancellationToken.None)); + + var restored = (await fixture.Manager.GetAllAsync(CancellationToken.None)).Single(plugin => + plugin.Manifest.Id == manifest.Id); + Assert.IsTrue(restored.IsEnabled); + Assert.IsTrue(Directory.Exists(Path.Combine( + fixture.RootPath, "packages", manifest.Id, manifest.Version))); + var exists = await fixture.InvokeAsync(manifest.Id, "exists", new { path = "state.txt" }); + Assert.IsTrue(exists.GetProperty("exists").GetBoolean()); + } + + [TestMethod] + public async Task UpgradeAndRetainedReinstall_RejectPublisherKeySubstitution() + { + using var originalKey = RSA.Create(2048); + using var substituteKey = RSA.Create(2048); + await using var fixture = new PluginPlatformFixture(options => + { + options.AllowUnsignedLocalPackages = false; + options.TrustedPublisherPublicKeys["original"] = originalKey.ExportSubjectPublicKeyInfoPem(); + options.TrustedPublisherPublicKeys["substitute"] = substituteKey.ExportSubjectPublicKeyInfoPem(); + }); + var versionOne = Sign(Manifest("test.publisher-owner"), PingScript, "original", originalKey); + await fixture.InstallAndEnableAsync(versionOne, PingScript); + var versionTwo = Sign(versionOne with { Version = "2.0.0", Signature = null }, PingScript, + "substitute", substituteKey); + var substituteUpgrade = await fixture.PreviewAsync(versionTwo, PingScript); + + var upgradeError = await Assert.ThrowsExactlyAsync(() => + fixture.Manager.UpgradeAsync(versionOne.Id, substituteUpgrade.Token, + substituteUpgrade.PackageSha256, substituteUpgrade.Manifest.Capabilities, + CancellationToken.None)); + StringAssert.Contains(upgradeError.Message, "publisher identity"); + + await fixture.Manager.UninstallAsync(versionOne.Id, deleteData: false, CancellationToken.None); + var reinstallManifest = Sign( + Manifest(versionOne.Id), PingScript, "substitute", substituteKey); + var substituteReinstall = await fixture.PreviewAsync(reinstallManifest, PingScript); + var reinstallError = await Assert.ThrowsExactlyAsync(() => + fixture.Manager.InstallPackageAsync(substituteReinstall.Token, + substituteReinstall.PackageSha256, substituteReinstall.Manifest.Capabilities, + CancellationToken.None)); + StringAssert.Contains(reinstallError.Message, "retained owner"); + } + + private static PluginManifest Manifest( + string id, + string version = "1.0.0", + string apiVersion = PluginApi.CurrentVersion, + PluginCapabilities? capabilities = null) + => new() + { + Id = id, + Name = id, + Version = version, + ApiVersion = apiVersion, + EntryPoint = "index.js", + Capabilities = capabilities ?? new PluginCapabilities(), + Integrity = new PluginIntegrity + { + Files = new Dictionary + { + ["index.js"] = new string('0', 64) + } + } + }; + + private static string GetSingleLifecycleTransaction(string rootPath, string operation, string pluginId) + { + var transactionRoot = Path.Combine(rootPath, "transactions"); + var transactions = Directory.Exists(transactionRoot) + ? Directory.EnumerateDirectories(transactionRoot, $"{operation}-{pluginId}-*").ToArray() + : []; + Assert.HasCount(1, transactions); + Assert.IsTrue(File.Exists($"{transactions[0]}.journal.json")); + return transactions[0]; + } + + private static void AssertNoLifecycleTransactions(string rootPath) + { + var transactionRoot = Path.Combine(rootPath, "transactions"); + if (!Directory.Exists(transactionRoot)) return; + Assert.IsEmpty(Directory.EnumerateDirectories(transactionRoot).ToArray()); + Assert.IsEmpty(Directory.EnumerateFiles( + transactionRoot, "*.journal.json", SearchOption.TopDirectoryOnly).ToArray()); + } + + private static PluginManifest StorageManifest(string id, string? providerName = null) + => Manifest(id, capabilities: new PluginCapabilities { StorageAccess = true }) with + { + Providers = [new PluginProviderDeclaration + { + Kind = "storage", + Name = providerName ?? $"{id}-store", + Handlers = new Dictionary + { + ["exists"] = "exists", + ["info"] = "info", + ["read"] = "read", + ["list"] = "list" + } + }] + }; + + private static (PluginManifest Manifest, string Script) LoadExample(string name) + { + var directory = Path.Combine(AppContext.BaseDirectory, "Examples", name); + var manifest = JsonSerializer.Deserialize( + File.ReadAllText(Path.Combine(directory, "manifest.json")), + new JsonSerializerOptions(JsonSerializerDefaults.Web)) + ?? throw new InvalidDataException($"Example manifest '{name}' is invalid."); + return (manifest, File.ReadAllText(Path.Combine(directory, manifest.EntryPoint))); + } + + private static PluginManifest WithIntegrity( + PluginManifest manifest, + string script, + params (string Path, string Content)[] extraEntries) + => manifest with + { + Integrity = new PluginIntegrity + { + Files = new[] { (manifest.EntryPoint, script) } + .Concat(extraEntries) + .ToDictionary( + entry => entry.Item1, + entry => Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(entry.Item2))) + .ToLowerInvariant(), + StringComparer.Ordinal) + } + }; + + private static PluginManifest Sign( + PluginManifest manifest, + string script, + string publisher, + RSA key, + params (string Path, string Content)[] extraEntries) + { + manifest = WithIntegrity(manifest, script, extraEntries); + var payload = PluginSignaturePayload.Create(manifest); + return manifest with + { + Signature = new PluginSignature( + publisher, + "RSA-SHA256", + Convert.ToBase64String(key.SignData(payload, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1))) + }; + } + + private static MemoryStream BuildPackage( + PluginManifest manifest, + string script, + params (string Path, string Content)[] extraEntries) + { + manifest = manifest.Integrity?.Files.TryGetValue(manifest.EntryPoint, out var entryDigest) != true || + entryDigest == new string('0', 64) + ? WithIntegrity(manifest, script, extraEntries) + : manifest; + var stream = new MemoryStream(); + using (var archive = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: true)) + { + WriteEntry(archive, "manifest.json", JsonSerializer.Serialize(manifest, + new JsonSerializerOptions(JsonSerializerDefaults.Web) { WriteIndented = true })); + WriteEntry(archive, manifest.EntryPoint, script); + foreach (var entry in extraEntries) WriteEntry(archive, entry.Path, entry.Content); + } + stream.Position = 0; + return stream; + } + + private static void WriteEntry(ZipArchive archive, string path, string content) + { + var entry = archive.CreateEntry(path, CompressionLevel.Fastest); + using var writer = new StreamWriter(entry.Open(), new UTF8Encoding(encoderShouldEmitUTF8Identifier: false)); + writer.Write(content); + } + + private sealed class PluginPlatformFixture : IAsyncDisposable + { + private readonly IOptions _options; + private readonly HttpMessageHandler _httpHandler; + + public PluginPlatformFixture( + Action? configure = null, + HttpMessageHandler? httpHandler = null, + bool enableFailureInjection = false) + { + RootPath = Path.Combine(Path.GetTempPath(), $"sdw-plugin-tests-{Guid.NewGuid():N}"); + var options = new PluginPlatformOptions + { + RootPath = RootPath, + AllowUnsignedLocalPackages = true, + InvocationTimeoutMilliseconds = 2_000, + MaximumWorkerCpuMilliseconds = 1_500, + MaximumWorkerMemoryMegabytes = 256, + CircuitBreakerFailures = 3, + CircuitBreakerSeconds = 60 + }; + configure?.Invoke(options); + _options = Options.Create(options); + _httpHandler = httpHandler ?? new RecordingHttpMessageHandler(); + IPluginCatalogRepository repository = new PluginCatalogRepository(_options); + if (enableFailureInjection) + { + FailingRepository = new FailingPluginCatalogRepository(repository); + repository = FailingRepository; + } + SafeFileAccess = new PluginSafeFileAccess(); + Manager = CreateManager(repository); + } + + public string RootPath { get; } + public PluginManager Manager { get; } + public PluginSafeFileAccess SafeFileAccess { get; } + public FailingPluginCatalogRepository? FailingRepository { get; } + + public PluginManager CreateRestartedManager() + => CreateManager(new PluginCatalogRepository(_options)); + + public async Task PreviewAsync(PluginManifest manifest, string script) + { + await using var package = BuildPackage(manifest, script); + return await Manager.PreviewPackageAsync(package, $"{manifest.Id}.sdwpkg", CancellationToken.None); + } + + public async Task InstallAsync(PluginManifest manifest, string script) + { + var preview = await PreviewAsync(manifest, script); + await Manager.InstallPackageAsync(preview.Token, preview.PackageSha256, + preview.Manifest.Capabilities, CancellationToken.None); + } + + public async Task InstallAndEnableAsync(PluginManifest manifest, string script) + { + await InstallAsync(manifest, script); + await Manager.EnableAsync(manifest.Id, CancellationToken.None); + } + + public Task InvokeAsync(string id, string handler) + => InvokeAsync(id, handler, new { }); + + public Task InvokeAsync(string id, string handler, object input) + => Manager.InvokeAsync(id, handler, JsonSerializer.SerializeToElement(input), CancellationToken.None); + + public ValueTask DisposeAsync() + { + if (Directory.Exists(RootPath)) Directory.Delete(RootPath, recursive: true); + return ValueTask.CompletedTask; + } + + private PluginManager CreateManager(IPluginCatalogRepository repository) + { + var inspector = new PluginPackageInspector(_options); + var broker = new PluginCapabilityBroker( + new FixedHttpClientFactory(_httpHandler), _options, SafeFileAccess); + var executor = new PluginProcessExecutor(broker, _options); + return new PluginManager(repository, inspector, executor, _options, TimeProvider.System); + } + } + + private sealed class FailingPluginCatalogRepository(IPluginCatalogRepository inner) + : IPluginCatalogRepository + { + public bool FailNextRetainedRemoval { get; set; } + public bool FailNextCatalogRemoval { get; set; } + + public Task> GetAllAsync(CancellationToken cancellationToken) + => inner.GetAllAsync(cancellationToken); + + public Task FindAsync(string id, CancellationToken cancellationToken) + => inner.FindAsync(id, cancellationToken); + + public Task SaveAsync(PluginCatalogEntry entry, CancellationToken cancellationToken) + => inner.SaveAsync(entry, cancellationToken); + + public Task RemoveAsync(string id, CancellationToken cancellationToken) + { + if (FailNextCatalogRemoval) + { + FailNextCatalogRemoval = false; + throw new IOException("Injected catalog removal failure."); + } + return inner.RemoveAsync(id, cancellationToken); + } + + public Task FindRetainedAsync(string id, CancellationToken cancellationToken) + => inner.FindRetainedAsync(id, cancellationToken); + + public Task SaveRetainedAsync(RetainedPluginData retained, CancellationToken cancellationToken) + => inner.SaveRetainedAsync(retained, cancellationToken); + + public Task RemoveRetainedAsync(string id, CancellationToken cancellationToken) + { + if (FailNextRetainedRemoval) + { + FailNextRetainedRemoval = false; + throw new IOException("Injected failure after new catalog write."); + } + return inner.RemoveRetainedAsync(id, cancellationToken); + } + } + + private sealed class FixedHttpClientFactory(HttpMessageHandler handler) : IHttpClientFactory + { + public HttpClient CreateClient(string name) => new(handler, disposeHandler: false); + } + + private sealed class FixedDnsResolver(params IPAddress[] addresses) : IPluginDnsResolver + { + public Task ResolveAsync(string host, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Assert.AreEqual("approved.example", host); + return Task.FromResult(addresses); + } + } + + private sealed class RecordingHttpMessageHandler : HttpMessageHandler + { + public Uri? LastRequestUri { get; private set; } + public string? LastBody { get; private set; } + + protected override async Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + LastRequestUri = request.RequestUri; + LastBody = request.Content is null ? null : await request.Content.ReadAsStringAsync(cancellationToken); + return new HttpResponseMessage(HttpStatusCode.NoContent) + { + Content = new StringContent(string.Empty) + }; + } + } +} diff --git a/SecondDimensionWatcherReDive.Test/SecondDimensionWatcherReDive.Test.csproj b/SecondDimensionWatcherReDive.Test/SecondDimensionWatcherReDive.Test.csproj index 4ce878a..c6fe06e 100644 --- a/SecondDimensionWatcherReDive.Test/SecondDimensionWatcherReDive.Test.csproj +++ b/SecondDimensionWatcherReDive.Test/SecondDimensionWatcherReDive.Test.csproj @@ -28,4 +28,13 @@ + + + + + diff --git a/SecondDimensionWatcherReDive/Controllers/Converter.cs b/SecondDimensionWatcherReDive/Controllers/Converter.cs index 742d740..976f952 100644 --- a/SecondDimensionWatcherReDive/Controllers/Converter.cs +++ b/SecondDimensionWatcherReDive/Controllers/Converter.cs @@ -1,10 +1,84 @@ using SecondDimensionWatcherReDive.Framework.DataRepository; using SecondDimensionWatcherReDive.Framework.FileDownload; +using SecondDimensionWatcherReDive.Framework.Plugin; namespace SecondDimensionWatcherReDive.Controllers; internal static class Converter { + public static External.PluginCapabilities ToExternal(this PluginCapabilities capabilities) => + new(capabilities.NetworkDomains, + capabilities.FileRoots, + capabilities.Notifications, + capabilities.DownloadControl, + capabilities.StorageAccess, + capabilities.BackgroundTasks); + + public static PluginCapabilities ToDomain(this External.PluginCapabilities capabilities) => + new() + { + NetworkDomains = capabilities.NetworkDomains, + FileRoots = capabilities.FileRoots, + Notifications = capabilities.Notifications, + DownloadControl = capabilities.DownloadControl, + StorageAccess = capabilities.StorageAccess, + BackgroundTasks = capabilities.BackgroundTasks + }; + + public static External.PluginManifest ToExternal(this PluginManifest manifest) => + new(manifest.Id, + manifest.Name, + manifest.Version, + manifest.ApiVersion, + manifest.EntryPoint, + manifest.Description, + manifest.Dependencies.Select(dependency => + new External.PluginDependency(dependency.Id, dependency.MinimumVersion)).ToArray(), + manifest.Capabilities.ToExternal(), + manifest.Platforms, + manifest.Integrity?.Files ?? new Dictionary(), + manifest.Signature?.Publisher, + manifest.Signature?.Algorithm, + manifest.Providers.Select(provider => new External.PluginProvider( + provider.Kind, + provider.Name, + provider.Handlers)).ToArray(), + manifest.DataVersion, + manifest.DataMigration is null + ? null + : new External.PluginDataMigration( + manifest.DataMigration.Strategy, + manifest.DataMigration.Description)); + + public static External.PluginHealth ToExternal(this PluginHealth health) => + new(health.Status, + health.ConsecutiveFailures, + health.LastSuccessAt, + health.LastFailureAt, + health.LastError, + health.CircuitOpenUntil); + + public static External.InstalledPlugin ToExternal(this InstalledPlugin plugin) => + new(plugin.Manifest.ToExternal(), + plugin.IsEnabled, + plugin.ApprovedCapabilities.ToExternal(), + plugin.CompatibilityErrors, + plugin.Health.ToExternal(), + plugin.Configuration.ValueKind == System.Text.Json.JsonValueKind.Object && + plugin.Configuration.EnumerateObject().Any()); + + public static External.PluginPackagePreview ToExternal(this PluginPackagePreview preview) => + new(preview.Token, + preview.PackageSha256, + preview.Manifest.ToExternal(), + preview.CompatibilityErrors, + preview.IsSignatureTrusted, + preview.SignatureStatus, + preview.ExpiresAt); + + public static External.PluginInstallResult ToExternal(this PluginInstallResult result) => + new(result.Id, result.Version, result.IsUpgrade, result.CompatibilityErrors); + public static External.AnimationInfo ToExternal(this AnimationInfo record) => new(record.Id, record.Title, diff --git a/SecondDimensionWatcherReDive/Controllers/External/AppJsonSerializerContext.cs b/SecondDimensionWatcherReDive/Controllers/External/AppJsonSerializerContext.cs index 0f20309..d30733e 100644 --- a/SecondDimensionWatcherReDive/Controllers/External/AppJsonSerializerContext.cs +++ b/SecondDimensionWatcherReDive/Controllers/External/AppJsonSerializerContext.cs @@ -65,4 +65,12 @@ namespace SecondDimensionWatcherReDive.Controllers.External; [JsonSerializable(typeof(QueueMediaLibraryScanResponse))] [JsonSerializable(typeof(ApplicationSettingsResponse))] [JsonSerializable(typeof(PatchApplicationSettingsRequest))] +[JsonSerializable(typeof(PluginPackagePreview))] +[JsonSerializable(typeof(PluginInstallResult))] +[JsonSerializable(typeof(InstalledPlugin))] +[JsonSerializable(typeof(IReadOnlyList))] +[JsonSerializable(typeof(InstallPluginRequest))] +[JsonSerializable(typeof(UpdatePluginConfigurationRequest))] +[JsonSerializable(typeof(RemotePluginInstallRequest))] +[JsonSerializable(typeof(PluginOperationError))] internal partial class AppJsonSerializerContext : JsonSerializerContext; diff --git a/SecondDimensionWatcherReDive/Controllers/External/PluginModels.cs b/SecondDimensionWatcherReDive/Controllers/External/PluginModels.cs new file mode 100644 index 0000000..8d787e0 --- /dev/null +++ b/SecondDimensionWatcherReDive/Controllers/External/PluginModels.cs @@ -0,0 +1,78 @@ +using System.Text.Json; +namespace SecondDimensionWatcherReDive.Controllers.External; + +internal sealed record InstallPluginRequest( + string PreviewToken, + string ExpectedSha256, + PluginCapabilities ApprovedCapabilities); + +internal sealed record UpdatePluginConfigurationRequest(JsonElement Configuration); + +internal sealed record RemotePluginInstallRequest(string Url, string? ExpectedSha256); + +internal sealed record PluginOperationError(string Code, string Message); + +internal sealed record PluginCapabilities( + IReadOnlyList NetworkDomains, + IReadOnlyList FileRoots, + bool Notifications, + bool DownloadControl, + bool StorageAccess, + bool BackgroundTasks); + +internal sealed record PluginDependency(string Id, string MinimumVersion); + +internal sealed record PluginProvider( + string Kind, + string Name, + IReadOnlyDictionary Handlers); + +internal sealed record PluginDataMigration(string Strategy, string? Description); + +internal sealed record PluginManifest( + string Id, + string Name, + string Version, + string ApiVersion, + string EntryPoint, + string? Description, + IReadOnlyList Dependencies, + PluginCapabilities Capabilities, + IReadOnlyList Platforms, + IReadOnlyDictionary FileSha256, + string? SignaturePublisher, + string? SignatureAlgorithm, + IReadOnlyList Providers, + int DataVersion, + PluginDataMigration? DataMigration); + +internal sealed record PluginHealth( + string Status, + int ConsecutiveFailures, + DateTimeOffset? LastSuccessAt, + DateTimeOffset? LastFailureAt, + string? LastError, + DateTimeOffset? CircuitOpenUntil); + +internal sealed record InstalledPlugin( + PluginManifest Manifest, + bool IsEnabled, + PluginCapabilities ApprovedCapabilities, + IReadOnlyList CompatibilityErrors, + PluginHealth Health, + bool HasConfiguration); + +internal sealed record PluginPackagePreview( + string Token, + string PackageSha256, + PluginManifest Manifest, + IReadOnlyList CompatibilityErrors, + bool IsSignatureTrusted, + string SignatureStatus, + DateTimeOffset ExpiresAt); + +internal sealed record PluginInstallResult( + string Id, + string Version, + bool IsUpgrade, + IReadOnlyList CompatibilityErrors); diff --git a/SecondDimensionWatcherReDive/Controllers/PluginController.cs b/SecondDimensionWatcherReDive/Controllers/PluginController.cs new file mode 100644 index 0000000..a00e74a --- /dev/null +++ b/SecondDimensionWatcherReDive/Controllers/PluginController.cs @@ -0,0 +1,133 @@ +using Microsoft.AspNetCore.Authentication.JwtBearer; +using System.Text.Json; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using SecondDimensionWatcherReDive.Controllers.External; +using SecondDimensionWatcherReDive.Framework.Plugin; +using SecondDimensionWatcherReDive.PluginPlatform; + +namespace SecondDimensionWatcherReDive.Controllers; + +[ApiController] +[Route("api/plugins")] +[Authorize(AuthenticationSchemes = JwtBearerDefaults.AuthenticationScheme)] +internal sealed class PluginController( + IPluginManager manager, + IJavaScriptPluginLoader packageLoader) : ControllerBase +{ + [HttpGet] + public async Task>> GetAll(CancellationToken cancellationToken) + => Ok((await manager.GetAllAsync(cancellationToken)).Select(plugin => plugin.ToExternal()).ToArray()); + + [HttpPost("preview")] + [Consumes("multipart/form-data")] + [RequestSizeLimit(8 * 1024 * 1024)] + public async Task> Preview( + [FromForm] IFormFile package, + CancellationToken cancellationToken) + { + if (package.Length == 0) return BadRequest(Error("empty_package", "A non-empty plugin package is required.")); + try + { + await using var stream = package.OpenReadStream(); + return Ok((await packageLoader.PreviewPackageAsync(stream, package.FileName, cancellationToken)).ToExternal()); + } + catch (Exception exception) when (exception is InvalidDataException or IOException or JsonException) + { + return BadRequest(Error("invalid_package", exception.Message)); + } + catch (InvalidOperationException exception) + { + return Conflict(Error("plugin_preview_capacity_reached", exception.Message)); + } + } + + [HttpPost("preview-remote")] + public ActionResult PreviewRemote([FromBody] RemotePluginInstallRequest request) + => StatusCode(StatusCodes.Status403Forbidden, Error( + "remote_install_disabled", + $"Remote JavaScript installation is disabled. Download '{request.Url}' through a trusted administrative channel, verify its provenance, then upload it for checksum, signature and capability review.")); + + [HttpPost("install")] + public async Task> Install( + [FromBody] InstallPluginRequest request, + CancellationToken cancellationToken) + => await ExecuteMutationAsync(async () => (await packageLoader.InstallPackageAsync( + request.PreviewToken, + request.ExpectedSha256, + request.ApprovedCapabilities.ToDomain(), + cancellationToken)).ToExternal()); + + [HttpPost("{id}/upgrade")] + public async Task> Upgrade( + string id, + [FromBody] InstallPluginRequest request, + CancellationToken cancellationToken) + => await ExecuteMutationAsync(async () => (await manager.UpgradeAsync( + id, + request.PreviewToken, + request.ExpectedSha256, + request.ApprovedCapabilities.ToDomain(), + cancellationToken)).ToExternal()); + + [HttpPost("{id}/enable")] + public async Task Enable(string id, CancellationToken cancellationToken) + => await ExecuteEmptyMutationAsync(() => manager.EnableAsync(id, cancellationToken)); + + [HttpPost("{id}/disable")] + public async Task Disable(string id, CancellationToken cancellationToken) + => await ExecuteEmptyMutationAsync(() => manager.DisableAsync(id, cancellationToken)); + + [HttpPut("{id}/configuration")] + public async Task UpdateConfiguration( + string id, + [FromBody] UpdatePluginConfigurationRequest request, + CancellationToken cancellationToken) + => await ExecuteEmptyMutationAsync(() => manager.UpdateConfigurationAsync( + id, + request.Configuration, + cancellationToken)); + + [HttpDelete("{id}")] + public async Task Uninstall( + string id, + CancellationToken cancellationToken, + [FromQuery] bool deleteData = false) + => await ExecuteEmptyMutationAsync(() => manager.UninstallAsync(id, deleteData, cancellationToken)); + + private async Task> ExecuteMutationAsync(Func> operation) + { + try + { + return Ok(await operation()); + } + catch (KeyNotFoundException exception) + { + return NotFound(Error("plugin_not_found", exception.Message)); + } + catch (ArgumentException exception) + { + return BadRequest(Error("invalid_plugin_request", exception.Message)); + } + catch (UnauthorizedAccessException exception) + { + return StatusCode(StatusCodes.Status403Forbidden, Error("capability_or_trust_denied", exception.Message)); + } + catch (Exception exception) when (exception is InvalidDataException or InvalidOperationException or IOException) + { + return Conflict(Error("plugin_operation_rejected", exception.Message)); + } + } + + private async Task ExecuteEmptyMutationAsync(Func operation) + { + var result = await ExecuteMutationAsync(async () => + { + await operation(); + return true; + }); + return result.Result ?? NoContent(); + } + + private static PluginOperationError Error(string code, string message) => new(code, message); +} diff --git a/SecondDimensionWatcherReDive/Plugin/PluginEvent.cs b/SecondDimensionWatcherReDive/Plugin/PluginEvent.cs index 8d25dfe..4f55d5c 100644 --- a/SecondDimensionWatcherReDive/Plugin/PluginEvent.cs +++ b/SecondDimensionWatcherReDive/Plugin/PluginEvent.cs @@ -5,12 +5,53 @@ namespace SecondDimensionWatcherReDive.Plugin; public class PluginEvent : IPluginEventRegister, IPluginEventTrigger { private readonly List> _handlers = []; + private readonly object _gate = new(); + private readonly TimeSpan _handlerTimeout; + private readonly Action? _onHandlerError; - public void Register(Func action) => _handlers.Add(action); + public PluginEvent(TimeSpan? handlerTimeout = null, Action? onHandlerError = null) + { + _handlerTimeout = handlerTimeout ?? TimeSpan.FromSeconds(5); + _onHandlerError = onHandlerError; + } + + public void Register(Func action) + { + ArgumentNullException.ThrowIfNull(action); + lock (_gate) _handlers.Add(action); + } public async Task InvokeAsync(T value, CancellationToken cancellationToken = default) { - foreach (var handler in _handlers) - await handler(value, cancellationToken); + Func[] handlers; + lock (_gate) handlers = _handlers.ToArray(); + foreach (var handler in handlers) + { + cancellationToken.ThrowIfCancellationRequested(); + using var handlerCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + handlerCancellation.CancelAfter(_handlerTimeout); + try + { + await handler(value, handlerCancellation.Token) + .WaitAsync(_handlerTimeout, cancellationToken); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + _onHandlerError?.Invoke(new TimeoutException( + $"Plugin event handler exceeded {_handlerTimeout.TotalMilliseconds:0} ms.")); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (TimeoutException exception) + { + _onHandlerError?.Invoke(exception); + } + catch (Exception exception) + { + _onHandlerError?.Invoke(exception); + } + } } } diff --git a/SecondDimensionWatcherReDive/Plugin/PluginHelper.cs b/SecondDimensionWatcherReDive/Plugin/PluginHelper.cs index c2805ad..135bd8d 100644 --- a/SecondDimensionWatcherReDive/Plugin/PluginHelper.cs +++ b/SecondDimensionWatcherReDive/Plugin/PluginHelper.cs @@ -14,9 +14,9 @@ public static WebApplicationBuilder InitializePlugin(this WebApplicationBuilder webApplicationBuilder.Services.AddSingleton>(beforeDownloadStarted); webApplicationBuilder.Services.AddSingleton>(onFileDownloadCompleted); - webApplicationBuilder.Services.AddSingleton(sp => + webApplicationBuilder.Services.AddSingleton(_ => { - var services = new PluginServices(sp); + var services = new PluginServices(); services.AddEvent(PluginEventName.BeforeDownloadStarted, beforeDownloadStarted); services.AddEvent(PluginEventName.OnFileDownloadCompleted, onFileDownloadCompleted); return services; diff --git a/SecondDimensionWatcherReDive/Plugin/PluginServices.cs b/SecondDimensionWatcherReDive/Plugin/PluginServices.cs index f23873b..a5f6794 100644 --- a/SecondDimensionWatcherReDive/Plugin/PluginServices.cs +++ b/SecondDimensionWatcherReDive/Plugin/PluginServices.cs @@ -3,12 +3,10 @@ namespace SecondDimensionWatcherReDive.Plugin; -public class PluginServices(IServiceProvider serviceProvider) : IPluginServices +public class PluginServices : IPluginServices { private readonly Dictionary _events = new(); - public IServiceProvider ServiceProvider => serviceProvider; - public void AddEvent(string eventName, PluginEvent pluginEvent) => _events[eventName] = pluginEvent; diff --git a/SecondDimensionWatcherReDive/PluginPlatform/IPluginCapabilityBroker.cs b/SecondDimensionWatcherReDive/PluginPlatform/IPluginCapabilityBroker.cs new file mode 100644 index 0000000..0e94dd6 --- /dev/null +++ b/SecondDimensionWatcherReDive/PluginPlatform/IPluginCapabilityBroker.cs @@ -0,0 +1,13 @@ +using System.Text.Json; +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.PluginPlatform; + +internal interface IPluginCapabilityBroker +{ + Task ExecuteAsync( + PluginCatalogEntry plugin, + string capability, + JsonElement payload, + CancellationToken cancellationToken); +} diff --git a/SecondDimensionWatcherReDive/PluginPlatform/IPluginManager.cs b/SecondDimensionWatcherReDive/PluginPlatform/IPluginManager.cs new file mode 100644 index 0000000..5033a43 --- /dev/null +++ b/SecondDimensionWatcherReDive/PluginPlatform/IPluginManager.cs @@ -0,0 +1,26 @@ +using System.Text.Json; +using SecondDimensionWatcherReDive.Framework.Plugin; + +namespace SecondDimensionWatcherReDive.PluginPlatform; + +internal interface IPluginManager +{ + Task InitializeAsync(CancellationToken cancellationToken); + Task> GetAllAsync(CancellationToken cancellationToken); + Task EnableAsync(string id, CancellationToken cancellationToken); + Task DisableAsync(string id, CancellationToken cancellationToken); + Task UpgradeAsync( + string id, + string previewToken, + string expectedSha256, + PluginCapabilities approvedCapabilities, + CancellationToken cancellationToken); + Task UninstallAsync(string id, bool deleteData, CancellationToken cancellationToken); + Task UpdateConfigurationAsync(string id, JsonElement configuration, CancellationToken cancellationToken); + Task InvokeAsync( + string id, + string handler, + JsonElement input, + CancellationToken cancellationToken); + IReadOnlyList GetSnapshot(); +} diff --git a/SecondDimensionWatcherReDive/PluginPlatform/PluginCapabilityBroker.cs b/SecondDimensionWatcherReDive/PluginPlatform/PluginCapabilityBroker.cs new file mode 100644 index 0000000..add9fc7 --- /dev/null +++ b/SecondDimensionWatcherReDive/PluginPlatform/PluginCapabilityBroker.cs @@ -0,0 +1,324 @@ +using System.Net; +using System.Collections.Concurrent; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Options; +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.PluginPlatform; + +internal sealed class PluginCapabilityBroker( + IHttpClientFactory httpClientFactory, + IOptions options, + PluginSafeFileAccess fileAccess) : IPluginCapabilityBroker +{ + private static readonly JsonSerializerOptions WebJsonOptions = new(JsonSerializerDefaults.Web); + private readonly PluginPlatformOptions _options = options.Value; + private readonly string _dataRoot = Path.Combine(Path.GetFullPath(options.Value.RootPath), "data"); + private readonly ConcurrentDictionary _dataGates = new(StringComparer.Ordinal); + + public Task ExecuteAsync( + PluginCatalogEntry plugin, + string capability, + JsonElement payload, + CancellationToken cancellationToken) + => capability switch + { + "network.request" => NetworkRequestAsync(plugin, payload, cancellationToken), + "file.read" => ReadFileAsync(plugin, payload, cancellationToken), + "file.list" => ListFilesAsync(plugin, payload, cancellationToken), + "data.read" => ReadDataAsync(plugin, payload, cancellationToken), + "data.write" => WriteDataAsync(plugin, payload, cancellationToken), + "data.list" => ListDataAsync(plugin, payload, cancellationToken), + "data.exists" => DataExistsAsync(plugin, payload, cancellationToken), + "data.info" => DataInfoAsync(plugin, payload, cancellationToken), + _ => throw new UnauthorizedAccessException($"Capability operation '{capability}' is not available.") + }; + + private async Task NetworkRequestAsync( + PluginCatalogEntry plugin, + JsonElement payload, + CancellationToken cancellationToken) + { + var request = payload.Deserialize(WebJsonOptions) + ?? throw new InvalidDataException("Invalid network request."); + if (!Uri.TryCreate(request.Url, UriKind.Absolute, out var uri) || + uri.Scheme is not ("http" or "https") || string.IsNullOrWhiteSpace(uri.Host)) + throw new UnauthorizedAccessException("Only absolute HTTP(S) URLs are allowed."); + if (!IsDomainAllowed(uri.IdnHost, plugin.ApprovedCapabilities.NetworkDomains)) + throw new UnauthorizedAccessException($"Network target '{uri.IdnHost}' was not approved."); + if (!Enum.TryParse(request.Method, ignoreCase: true, out var methodName)) + throw new InvalidDataException("Unsupported HTTP method."); + + using var message = new HttpRequestMessage(new HttpMethod(methodName.ToString().ToUpperInvariant()), uri); + if (request.Body is not null) + message.Content = new StringContent(request.Body, Encoding.UTF8, request.ContentType ?? "application/json"); + using var response = await httpClientFactory.CreateClient("PluginPlatform").SendAsync( + message, + HttpCompletionOption.ResponseHeadersRead, + cancellationToken); + var body = await ReadBoundedAsync(await response.Content.ReadAsStreamAsync(cancellationToken), + _options.MaximumResponseBytes, cancellationToken); + return JsonSerializer.SerializeToElement(new + { + status = (int)response.StatusCode, + contentType = response.Content.Headers.ContentType?.ToString(), + body = Encoding.UTF8.GetString(body) + }); + } + + private async Task ReadFileAsync( + PluginCatalogEntry plugin, + JsonElement payload, + CancellationToken cancellationToken) + { + var path = GetRequiredString(payload, "path"); + var (root, resolved) = ResolveApprovedFilePath(path, plugin.ApprovedCapabilities.FileRoots); + var bytes = await fileAccess.ReadAsync(root, resolved, _options.MaximumResponseBytes, cancellationToken); + return JsonSerializer.SerializeToElement(new { base64 = Convert.ToBase64String(bytes) }); + } + + private Task ListFilesAsync( + PluginCatalogEntry plugin, + JsonElement payload, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var (root, path) = ResolveApprovedFilePath(GetRequiredString(payload, "path"), + plugin.ApprovedCapabilities.FileRoots); + var entries = fileAccess.List(root, path, 1_000) + .Select(item => new + { + name = item.Name, + isDirectory = item.IsDirectory + }) + .ToArray(); + return Task.FromResult(JsonSerializer.SerializeToElement(entries)); + } + + private async Task ReadDataAsync( + PluginCatalogEntry plugin, + JsonElement payload, + CancellationToken cancellationToken) + { + EnsureStorageCapability(plugin); + var root = GetPluginDataRoot(plugin.Manifest.Id); + var path = ResolvePluginDataPath(plugin.Manifest.Id, GetRequiredString(payload, "path")); + var bytes = await fileAccess.ReadAsync(root, path, _options.MaximumResponseBytes, cancellationToken); + return JsonSerializer.SerializeToElement(new { base64 = Convert.ToBase64String(bytes) }); + } + + private async Task WriteDataAsync( + PluginCatalogEntry plugin, + JsonElement payload, + CancellationToken cancellationToken) + { + EnsureStorageCapability(plugin); + var root = GetPluginDataRoot(plugin.Manifest.Id); + var path = ResolvePluginDataPath(plugin.Manifest.Id, GetRequiredString(payload, "path")); + var base64 = GetRequiredString(payload, "base64"); + byte[] bytes; + try + { + bytes = Convert.FromBase64String(base64); + } + catch (FormatException) + { + throw new InvalidDataException("Data payload must be valid base64."); + } + if (bytes.Length > _options.MaximumResponseBytes) + throw new InvalidDataException("Data write exceeds the configured size limit."); + var gate = _dataGates.GetOrAdd(plugin.Manifest.Id, _ => new SemaphoreSlim(1, 1)); + await gate.WaitAsync(cancellationToken); + try + { + var usage = MeasureUsage(root); + var existing = fileAccess.Info(root, path); + var projectedFiles = usage.Files + (existing is null ? 1 : 0); + var projectedBytes = checked(usage.Bytes - (existing?.Length ?? 0) + bytes.Length); + if (projectedFiles > _options.MaximumPluginDataFiles || + projectedBytes > _options.MaximumPluginDataBytes) + throw new InvalidDataException("Plugin data quota would be exceeded."); + await fileAccess.WriteAsync(root, path, bytes, cancellationToken); + } + finally + { + gate.Release(); + } + + return JsonSerializer.SerializeToElement(new { written = bytes.Length }); + } + + private Task ListDataAsync( + PluginCatalogEntry plugin, + JsonElement payload, + CancellationToken cancellationToken) + { + EnsureStorageCapability(plugin); + cancellationToken.ThrowIfCancellationRequested(); + var root = GetPluginDataRoot(plugin.Manifest.Id); + var path = ResolvePluginDataPath(plugin.Manifest.Id, GetRequiredString(payload, "path", allowEmpty: true)); + if (!fileAccess.Exists(root, path)) + return Task.FromResult(JsonSerializer.SerializeToElement(Array.Empty())); + var entries = fileAccess.List(root, path, 1_000) + .Select(item => new + { + name = item.Name, + isDirectory = item.IsDirectory, + length = item.Length, + lastModifiedUtc = item.LastModifiedUtc + }) + .ToArray(); + return Task.FromResult(JsonSerializer.SerializeToElement(entries)); + } + + private Task DataExistsAsync( + PluginCatalogEntry plugin, + JsonElement payload, + CancellationToken cancellationToken) + { + EnsureStorageCapability(plugin); + cancellationToken.ThrowIfCancellationRequested(); + var root = GetPluginDataRoot(plugin.Manifest.Id); + var path = ResolvePluginDataPath(plugin.Manifest.Id, GetRequiredString(payload, "path", allowEmpty: true)); + var info = fileAccess.Info(root, path); + return Task.FromResult(JsonSerializer.SerializeToElement(new + { + exists = info is not null, + isDirectory = info?.IsDirectory ?? false + })); + } + + private Task DataInfoAsync( + PluginCatalogEntry plugin, + JsonElement payload, + CancellationToken cancellationToken) + { + EnsureStorageCapability(plugin); + cancellationToken.ThrowIfCancellationRequested(); + var relativePath = GetRequiredString(payload, "path", allowEmpty: true); + var root = GetPluginDataRoot(plugin.Manifest.Id); + var path = ResolvePluginDataPath(plugin.Manifest.Id, relativePath); + var info = fileAccess.Info(root, path) + ?? throw new FileNotFoundException("Plugin data path does not exist."); + return Task.FromResult(JsonSerializer.SerializeToElement(new + { + isDirectory = info.IsDirectory, + path = relativePath, + fileName = info.Name, + length = info.Length, + lastModifiedUtc = info.LastModifiedUtc + })); + } + + private (string Root, string Path) ResolveApprovedFilePath(string path, IReadOnlyList approvedRoots) + { + if (approvedRoots.Count == 0) throw new UnauthorizedAccessException("No file roots were approved."); + if (!Path.IsPathFullyQualified(path)) throw new UnauthorizedAccessException("File paths must be absolute."); + var candidate = Path.GetFullPath(path); + var root = approvedRoots.Select(Path.GetFullPath).FirstOrDefault(value => IsWithin(candidate, value)); + if (root is null) throw new UnauthorizedAccessException($"File path '{path}' is outside approved roots."); + return (root, candidate); + } + + private string ResolvePluginDataPath(string pluginId, string relativePath) + { + if (!PluginManifestValidator.IsSafeRelativePath(relativePath) && !string.IsNullOrEmpty(relativePath)) + throw new UnauthorizedAccessException("Plugin data paths must be relative and cannot contain traversal."); + var root = GetPluginDataRoot(pluginId); + var depth = relativePath.Split( + [Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar], + StringSplitOptions.RemoveEmptyEntries).Length; + if (depth > _options.MaximumPluginDataPathDepth) + throw new InvalidDataException("Plugin data path exceeds the configured depth limit."); + var candidate = Path.GetFullPath(Path.Combine(root, relativePath)); + if (!IsWithin(candidate, root)) throw new UnauthorizedAccessException("Plugin data path escapes its root."); + return candidate; + } + + private string GetPluginDataRoot(string pluginId) => Path.Combine(_dataRoot, pluginId); + + private static void EnsureStorageCapability(PluginCatalogEntry plugin) + { + if (!plugin.ApprovedCapabilities.StorageAccess) + throw new UnauthorizedAccessException("Storage access was not approved for this plugin."); + } + + private static string GetRequiredString(JsonElement payload, string property, bool allowEmpty = false) + { + if (!payload.TryGetProperty(property, out var value) || value.ValueKind != JsonValueKind.String) + throw new InvalidDataException($"Capability request requires string property '{property}'."); + var result = value.GetString() ?? string.Empty; + if (!allowEmpty && string.IsNullOrWhiteSpace(result)) + throw new InvalidDataException($"Capability request property '{property}' cannot be empty."); + return result; + } + + private static bool IsDomainAllowed(string host, IEnumerable domains) + => domains.Any(pattern => pattern.StartsWith("*.", StringComparison.Ordinal) + ? host.EndsWith(pattern[1..], StringComparison.OrdinalIgnoreCase) && + host.Length > pattern.Length - 1 + : host.Equals(pattern, StringComparison.OrdinalIgnoreCase)); + + private static bool IsWithin(string candidate, string root) + { + var relative = Path.GetRelativePath(root, candidate); + return relative != ".." && !relative.StartsWith($"..{Path.DirectorySeparatorChar}", StringComparison.Ordinal) && + !Path.IsPathFullyQualified(relative); + } + + private (long Bytes, int Files) MeasureUsage(string root) + { + if (!Directory.Exists(root)) return (0, 0); + long bytes = 0; + var files = 0; + var pending = new Stack(); + pending.Push(root); + while (pending.TryPop(out var directory)) + { + foreach (var entry in fileAccess.List(root, directory, _options.MaximumPluginDataFiles + 1)) + { + var path = Path.Combine(directory, entry.Name); + if (entry.IsDirectory) pending.Push(path); + else + { + files = checked(files + 1); + bytes = checked(bytes + (entry.Length ?? 0)); + } + } + } + return (bytes, files); + } + + private static async Task ReadBoundedAsync( + Stream stream, + int maximumBytes, + CancellationToken cancellationToken) + { + using var memory = new MemoryStream(Math.Min(maximumBytes, 64 * 1024)); + var buffer = new byte[64 * 1024]; + int read; + while ((read = await stream.ReadAsync(buffer, cancellationToken)) > 0) + { + if (memory.Length + read > maximumBytes) + throw new InvalidDataException("Capability response exceeds the configured size limit."); + await memory.WriteAsync(buffer.AsMemory(0, read), cancellationToken); + } + return memory.ToArray(); + } + + private sealed record NetworkCapabilityRequest( + string Method, + string Url, + string? Body, + string? ContentType); + + private enum HttpMethodName + { + Get, + Post, + Put, + Patch, + Delete + } +} diff --git a/SecondDimensionWatcherReDive/PluginPlatform/PluginLifecycleCoordinator.cs b/SecondDimensionWatcherReDive/PluginPlatform/PluginLifecycleCoordinator.cs new file mode 100644 index 0000000..f95305d --- /dev/null +++ b/SecondDimensionWatcherReDive/PluginPlatform/PluginLifecycleCoordinator.cs @@ -0,0 +1,133 @@ +using System.Collections.Concurrent; + +namespace SecondDimensionWatcherReDive.PluginPlatform; + +internal sealed class PluginInvocationInterruptedException(string message) : InvalidOperationException(message); + +internal sealed class PluginLifecycleCoordinator +{ + private readonly ConcurrentDictionary _gates = new(StringComparer.Ordinal); + + public InvocationLease EnterInvocation(string pluginId, CancellationToken callerCancellationToken) + => _gates.GetOrAdd(pluginId, _ => new Gate()).Enter(callerCancellationToken); + + public Task BeginLifecycleAsync( + string pluginId, + TimeSpan timeout, + CancellationToken cancellationToken) + => _gates.GetOrAdd(pluginId, _ => new Gate()).BeginLifecycleAsync(timeout, cancellationToken); + + internal sealed class InvocationLease : IDisposable + { + private readonly Gate _owner; + private readonly CancellationTokenSource _linkedCancellation; + private int _disposed; + + public InvocationLease( + Gate owner, + CancellationToken lifecycleCancellationToken, + CancellationToken callerCancellationToken) + { + _owner = owner; + LifecycleCancellationToken = lifecycleCancellationToken; + _linkedCancellation = CancellationTokenSource.CreateLinkedTokenSource( + lifecycleCancellationToken, callerCancellationToken); + } + + public CancellationToken Token => _linkedCancellation.Token; + public CancellationToken LifecycleCancellationToken { get; } + + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) return; + _linkedCancellation.Dispose(); + _owner.ExitInvocation(); + } + } + + internal sealed class Gate + { + private readonly object _sync = new(); + private CancellationTokenSource _lifecycleCancellation = new(); + private TaskCompletionSource? _drained; + private int _active; + private bool _lifecycleActive; + + public InvocationLease Enter(CancellationToken callerCancellationToken) + { + lock (_sync) + { + if (_lifecycleActive) + throw new PluginCapacityExceededException( + "Plugin is being disabled, upgraded, or uninstalled."); + _active++; + return new InvocationLease(this, _lifecycleCancellation.Token, callerCancellationToken); + } + } + + public async Task BeginLifecycleAsync( + TimeSpan timeout, + CancellationToken cancellationToken) + { + Task drained; + lock (_sync) + { + if (_lifecycleActive) + throw new InvalidOperationException("A lifecycle operation is already in progress for this plugin."); + _lifecycleActive = true; + _lifecycleCancellation.Cancel(); + if (_active == 0) + { + drained = Task.CompletedTask; + } + else + { + _drained = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + drained = _drained.Task; + } + } + + try + { + await drained.WaitAsync(timeout, cancellationToken); + return new LifecycleLease(this); + } + catch + { + EndLifecycle(); + throw; + } + } + + public void ExitInvocation() + { + lock (_sync) + { + _active--; + if (_active == 0) _drained?.TrySetResult(); + } + } + + private void EndLifecycle() + { + lock (_sync) + { + if (!_lifecycleActive) return; + _lifecycleCancellation.Dispose(); + _lifecycleCancellation = new CancellationTokenSource(); + _drained = null; + _lifecycleActive = false; + } + } + + private sealed class LifecycleLease(Gate owner) : IDisposable + { + private int _disposed; + + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) owner.EndLifecycle(); + } + } + } +} diff --git a/SecondDimensionWatcherReDive/PluginPlatform/PluginLifecycleJournal.cs b/SecondDimensionWatcherReDive/PluginPlatform/PluginLifecycleJournal.cs new file mode 100644 index 0000000..f7f59ba --- /dev/null +++ b/SecondDimensionWatcherReDive/PluginPlatform/PluginLifecycleJournal.cs @@ -0,0 +1,32 @@ +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.PluginPlatform; + +internal static class PluginLifecycleJournalValues +{ + public const string Upgrade = "upgrade"; + public const string Uninstall = "uninstall"; + public const string Prepared = "prepared"; + public const string Committed = "committed"; +} + +internal sealed record PluginLifecycleJournal( + string Operation, + string PluginId, + string Phase, + PluginCatalogEntry[] OriginalEntries, + RetainedPluginData? OriginalRetained, + RetainedPluginData? IntendedRetained, + bool DeleteData); + +internal enum PluginLifecycleCheckpoint +{ + AfterMove, + AfterCommit +} + +/// +/// Test-only abrupt-termination signal. The manager deliberately bypasses its in-process +/// rollback for this exception so a new manager can exercise durable startup recovery. +/// +internal sealed class PluginProcessCrashSimulationException() : Exception("Simulated plugin host termination."); diff --git a/SecondDimensionWatcherReDive/PluginPlatform/PluginManager.cs b/SecondDimensionWatcherReDive/PluginPlatform/PluginManager.cs new file mode 100644 index 0000000..27b6159 --- /dev/null +++ b/SecondDimensionWatcherReDive/PluginPlatform/PluginManager.cs @@ -0,0 +1,989 @@ +using System.Security.Cryptography; +using System.Runtime.ExceptionServices; +using System.Diagnostics; +using System.Text.Json; +using Microsoft.Extensions.Options; +using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Framework.Plugin; + +namespace SecondDimensionWatcherReDive.PluginPlatform; + +internal sealed class PluginManager( + IPluginCatalogRepository repository, + PluginPackageInspector packageInspector, + IPluginProcessExecutor processExecutor, + IOptions options, + TimeProvider timeProvider) : IPluginManager, IJavaScriptPluginLoader +{ + private const string LifecycleJournalSuffix = ".journal.json"; + + private static readonly JsonSerializerOptions JournalJsonOptions = new(JsonSerializerDefaults.Web) + { + WriteIndented = true + }; + + private readonly SemaphoreSlim _gate = new(1, 1); + private readonly PluginLifecycleCoordinator _lifecycle = new(); + private readonly Dictionary _entries = new(StringComparer.Ordinal); + private readonly HashSet _pendingLifecyclePluginIds = new(StringComparer.Ordinal); + private InstalledPlugin[] _snapshot = []; + private readonly PluginPlatformOptions _options = options.Value; + private readonly string _rootPath = Path.GetFullPath(options.Value.RootPath); + private bool _initialized; + + internal Action? BeforeInvocationLeaseForTesting { get; set; } + internal Action? LifecycleCheckpointForTesting { get; set; } + + public async Task InitializeAsync(CancellationToken cancellationToken) + { + await _gate.WaitAsync(cancellationToken); + try + { + if (_initialized) return; + Directory.CreateDirectory(_rootPath); + RestrictDirectory(_rootPath); + await RecoverLifecycleTransactionsAsync(cancellationToken); + foreach (var entry in await repository.GetAllAsync(cancellationToken)) + _entries[entry.Manifest.Id] = entry; + await DisableMissingPackagePluginsAsync(cancellationToken); + await DisableIncompatiblePluginsAsync(cancellationToken); + CleanupUnreferencedPackages(); + UpdateSnapshot(); + _initialized = true; + } + finally + { + _gate.Release(); + } + } + + public async Task> GetAllAsync(CancellationToken cancellationToken) + { + await EnsureInitializedAsync(cancellationToken); + await _gate.WaitAsync(cancellationToken); + try + { + UpdateSnapshot(); + return _snapshot; + } + finally + { + _gate.Release(); + } + } + + public IReadOnlyList GetSnapshot() => Volatile.Read(ref _snapshot); + + public async Task PreviewPackageAsync( + Stream package, + string fileName, + CancellationToken cancellationToken) + { + await EnsureInitializedAsync(cancellationToken); + var inspected = await packageInspector.StageAndInspectAsync(package, fileName, cancellationToken); + await _gate.WaitAsync(cancellationToken); + try + { + return new PluginPackagePreview( + inspected.Token, + inspected.PackageSha256, + inspected.Manifest, + GetCompatibilityErrors(inspected.Manifest), + inspected.IsSignatureTrusted, + inspected.SignatureStatus, + inspected.ExpiresAt); + } + finally + { + _gate.Release(); + } + } + + public Task InstallPackageAsync( + string previewToken, + string expectedSha256, + PluginCapabilities approvedCapabilities, + CancellationToken cancellationToken) + => InstallOrUpgradeAsync(null, previewToken, expectedSha256, approvedCapabilities, cancellationToken); + + public Task UpgradeAsync( + string id, + string previewToken, + string expectedSha256, + PluginCapabilities approvedCapabilities, + CancellationToken cancellationToken) + => InstallOrUpgradeAsync(id, previewToken, expectedSha256, approvedCapabilities, cancellationToken); + + public async Task EnableAsync(string id, CancellationToken cancellationToken) + { + await EnsureInitializedAsync(cancellationToken); + await _gate.WaitAsync(cancellationToken); + try + { + var entry = GetRequiredEntry(id); + EnsureNoPendingLifecycleManagement(); + var errors = GetCompatibilityErrors(entry.Manifest); + if (errors.Count > 0) + throw new InvalidOperationException($"Plugin cannot be enabled: {string.Join(" ", errors)}"); + if (entry.IsEnabled) return; + entry = entry with + { + IsEnabled = true, + Health = entry.Health with + { + Status = "healthy", + ConsecutiveFailures = 0, + CircuitOpenUntil = null, + LastError = null + } + }; + await repository.SaveAsync(entry, cancellationToken); + _entries[id] = entry; + UpdateSnapshot(); + } + finally + { + _gate.Release(); + } + } + + public async Task DisableAsync(string id, CancellationToken cancellationToken) + { + await EnsureInitializedAsync(cancellationToken); + await _gate.WaitAsync(cancellationToken); + try + { + var entry = GetRequiredEntry(id); + EnsureNoPendingLifecycleManagement(); + using var lifecycle = await _lifecycle.BeginLifecycleAsync( + id, LifecycleWaitTimeout, cancellationToken); + if (entry.IsEnabled) + { + entry = entry with { IsEnabled = false }; + await repository.SaveAsync(entry, cancellationToken); + _entries[id] = entry; + } + await DisableIncompatiblePluginsAsync(cancellationToken); + UpdateSnapshot(); + } + finally + { + _gate.Release(); + } + } + + public async Task UninstallAsync(string id, bool deleteData, CancellationToken cancellationToken) + { + await EnsureInitializedAsync(cancellationToken); + await _gate.WaitAsync(cancellationToken); + string? transactionPath = null; + string? packageBackupPath = null; + string? dataBackupPath = null; + RetainedPluginData? originalRetained = null; + PluginLifecycleJournal? journal = null; + var journalCommitted = false; + var originalEntries = _entries.ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal); + try + { + var entry = GetRequiredEntry(id); + EnsureNoPendingLifecycleManagement(); + using var lifecycle = await _lifecycle.BeginLifecycleAsync( + id, LifecycleWaitTimeout, cancellationToken); + originalRetained = await repository.FindRetainedAsync(id, cancellationToken); + var intendedRetained = deleteData + ? null + : new RetainedPluginData( + id, + entry.ConfigurationJson, + entry.DataVersion, + timeProvider.GetUtcNow(), + entry.PublisherFingerprint); + transactionPath = CreateTransactionPath(PluginLifecycleJournalValues.Uninstall, id); + journal = new PluginLifecycleJournal( + PluginLifecycleJournalValues.Uninstall, + id, + PluginLifecycleJournalValues.Prepared, + originalEntries.Values.ToArray(), + originalRetained, + intendedRetained, + deleteData); + WriteLifecycleJournal(transactionPath, journal); + packageBackupPath = MoveToTransaction(entry.PackageDirectory, + Path.Combine(transactionPath, "package")); + if (deleteData) + dataBackupPath = MoveToTransaction(GetDataPath(id), Path.Combine(transactionPath, "data")); + LifecycleCheckpointForTesting?.Invoke(PluginLifecycleCheckpoint.AfterMove); + + if (!deleteData) + { + await repository.SaveRetainedAsync(intendedRetained!, cancellationToken); + } + else + { + await repository.RemoveRetainedAsync(id, cancellationToken); + } + + await repository.RemoveAsync(id, cancellationToken); + _entries.Remove(id); + await DisableIncompatiblePluginsAsync(cancellationToken); + UpdateSnapshot(); + journal = journal with { Phase = PluginLifecycleJournalValues.Committed }; + WriteLifecycleJournal(transactionPath, journal); + journalCommitted = true; + LifecycleCheckpointForTesting?.Invoke(PluginLifecycleCheckpoint.AfterCommit); + DeleteLifecycleTransaction(transactionPath, id); + } + catch (PluginProcessCrashSimulationException) + { + throw; + } + catch (Exception) when (journalCommitted) + { + // The catalog change is durably committed. Keep the journal so startup can + // retry idempotent cleanup rather than attempting an unsafe partial rollback. + throw; + } + catch (Exception failure) + { + try + { + foreach (var originalEntry in originalEntries.Values) + await repository.SaveAsync(originalEntry, CancellationToken.None); + if (originalRetained is null) + await repository.RemoveRetainedAsync(id, CancellationToken.None); + else + await repository.SaveRetainedAsync(originalRetained, CancellationToken.None); + RestoreTransactionDirectory(packageBackupPath, + originalEntries.TryGetValue(id, out var original) ? original.PackageDirectory : null); + RestoreTransactionDirectory(dataBackupPath, GetDataPath(id)); + _entries.Clear(); + foreach (var originalEntry in originalEntries) + _entries[originalEntry.Key] = originalEntry.Value; + UpdateSnapshot(); + DeleteLifecycleTransactionBestEffort(transactionPath, id); + } + catch (Exception rollbackFailure) + { + throw new AggregateException("Plugin uninstall failed and rollback was incomplete.", + failure, rollbackFailure); + } + throw; + } + finally + { + _gate.Release(); + } + } + + public async Task UpdateConfigurationAsync( + string id, + JsonElement configuration, + CancellationToken cancellationToken) + { + await EnsureInitializedAsync(cancellationToken); + if (configuration.ValueKind != JsonValueKind.Object) + throw new InvalidDataException("Plugin configuration must be a JSON object."); + var json = configuration.GetRawText(); + if (json.Length > 64 * 1024) throw new InvalidDataException("Plugin configuration is too large."); + await _gate.WaitAsync(cancellationToken); + try + { + var entry = GetRequiredEntry(id) with { ConfigurationJson = json }; + EnsureNoPendingLifecycleManagement(); + await repository.SaveAsync(entry, cancellationToken); + _entries[id] = entry; + UpdateSnapshot(); + } + finally + { + _gate.Release(); + } + } + + public async Task InvokeAsync( + string id, + string handler, + JsonElement input, + CancellationToken cancellationToken) + { + await EnsureInitializedAsync(cancellationToken); + if (!PluginManifestValidator.IsValidHandlerName(handler)) + throw new InvalidDataException("Invalid plugin handler name."); + PluginCatalogEntry entry; + PluginLifecycleCoordinator.InvocationLease? invocation = null; + await _gate.WaitAsync(cancellationToken); + try + { + entry = GetRequiredEntry(id); + EnsureNoPendingLifecycleManagement(); + if (!entry.IsEnabled) throw new InvalidOperationException($"Plugin '{id}' is disabled."); + var errors = GetCompatibilityErrors(entry.Manifest); + if (errors.Count > 0) + throw new InvalidOperationException($"Plugin '{id}' is incompatible: {string.Join(" ", errors)}"); + if (entry.Health.CircuitOpenUntil is { } openUntil && openUntil > timeProvider.GetUtcNow()) + throw new InvalidOperationException($"Plugin '{id}' circuit is open until {openUntil:O}."); + BeforeInvocationLeaseForTesting?.Invoke(); + invocation = _lifecycle.EnterInvocation(id, cancellationToken); + } + finally + { + _gate.Release(); + } + + JsonElement result = default; + Exception? failure = null; + var interruptedByLifecycle = false; + using (var activeInvocation = invocation ?? throw new UnreachableException()) + { + try + { + result = await processExecutor.InvokeAsync(entry, handler, input, activeInvocation.Token); + interruptedByLifecycle = activeInvocation.LifecycleCancellationToken.IsCancellationRequested && + !cancellationToken.IsCancellationRequested; + if (interruptedByLifecycle) + failure = new OperationCanceledException("Invocation completed during lifecycle cancellation."); + } + catch (Exception exception) + { + failure = exception; + interruptedByLifecycle = activeInvocation.LifecycleCancellationToken.IsCancellationRequested && + !cancellationToken.IsCancellationRequested; + } + } + + if (failure is null) + { + await RecordSuccessAsync(id, cancellationToken); + return result; + } + + if (interruptedByLifecycle) + throw new PluginInvocationInterruptedException( + $"Plugin '{id}' invocation was cancelled by a lifecycle operation."); + if (failure is not PluginCapacityExceededException && + (failure is not OperationCanceledException || !cancellationToken.IsCancellationRequested)) + await RecordFailureAsync(id, failure, CancellationToken.None); + ExceptionDispatchInfo.Capture(failure).Throw(); + throw new UnreachableException(); + } + + private async Task InstallOrUpgradeAsync( + string? upgradeId, + string previewToken, + string expectedSha256, + PluginCapabilities approvedCapabilities, + CancellationToken cancellationToken) + { + await EnsureInitializedAsync(cancellationToken); + var inspected = await packageInspector.InspectStagedAsync(previewToken, expectedSha256, cancellationToken); + if (!PluginManifestValidator.CapabilitiesEqual(inspected.Manifest.Capabilities, approvedCapabilities)) + throw new UnauthorizedAccessException("Approved capabilities do not exactly match the reviewed manifest."); + if (!inspected.IsSignatureTrusted && + !(_options.AllowUnsignedLocalPackages && inspected.Manifest.Signature is null)) + throw new UnauthorizedAccessException( + $"Package is not signed by a trusted publisher. {inspected.SignatureStatus}"); + + await _gate.WaitAsync(cancellationToken); + string? extractedPath = null; + string? transactionPath = null; + string? dataBackupPath = null; + PluginCatalogEntry? existing = null; + RetainedPluginData? retained = null; + PluginLifecycleJournal? journal = null; + IDisposable? lifecycle = null; + var catalogWritten = false; + var journalCommitted = false; + var originalEntries = _entries.ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal); + try + { + _entries.TryGetValue(inspected.Manifest.Id, out existing); + EnsureNoPendingLifecycleManagement(); + if (upgradeId is null && existing is not null) + throw new InvalidOperationException("Plugin is already installed; use the upgrade operation."); + if (upgradeId is not null) + { + if (existing is null || !string.Equals(upgradeId, inspected.Manifest.Id, StringComparison.Ordinal)) + throw new InvalidOperationException("Upgrade package id does not match the installed plugin."); + if (!PluginManifestValidator.TryParseVersion(existing.Manifest.Version, out var oldVersion) || + !PluginManifestValidator.TryParseVersion(inspected.Manifest.Version, out var newVersion) || + newVersion <= oldVersion) + throw new InvalidOperationException("Upgrade version must be newer than the installed version."); + EnsurePublisherContinuity(existing.PublisherFingerprint, inspected.PublisherFingerprint); + lifecycle = await _lifecycle.BeginLifecycleAsync( + inspected.Manifest.Id, LifecycleWaitTimeout, cancellationToken); + } + + retained = existing is null + ? await repository.FindRetainedAsync(inspected.Manifest.Id, cancellationToken) + : null; + if (retained is not null) + EnsurePublisherContinuity(retained.PublisherFingerprint, inspected.PublisherFingerprint); + var previousDataVersion = existing?.DataVersion ?? retained?.DataVersion; + if (previousDataVersion is not null && previousDataVersion != inspected.Manifest.DataVersion) + { + if (!string.Equals(inspected.Manifest.DataMigration?.Strategy, "reset", StringComparison.OrdinalIgnoreCase)) + throw new InvalidOperationException( + $"Data version changes from {previousDataVersion} to {inspected.Manifest.DataVersion}; manifest must explicitly declare dataMigration.strategy = 'reset'."); + transactionPath = CreateTransactionPath( + PluginLifecycleJournalValues.Upgrade, inspected.Manifest.Id); + journal = new PluginLifecycleJournal( + PluginLifecycleJournalValues.Upgrade, + inspected.Manifest.Id, + PluginLifecycleJournalValues.Prepared, + originalEntries.Values.ToArray(), + retained, + null, + DeleteData: false); + WriteLifecycleJournal(transactionPath, journal); + dataBackupPath = MoveToTransaction( + GetDataPath(inspected.Manifest.Id), Path.Combine(transactionPath, "data")); + LifecycleCheckpointForTesting?.Invoke(PluginLifecycleCheckpoint.AfterMove); + } + + extractedPath = await packageInspector.ExtractAsync(inspected, cancellationToken); + var configuration = existing?.ConfigurationJson ?? retained?.ConfigurationJson ?? "{}"; + var entry = new PluginCatalogEntry( + inspected.Manifest, + false, + approvedCapabilities, + new PluginHealth("healthy", 0, null, null, null, null), + extractedPath, + configuration, + inspected.Manifest.DataVersion, + inspected.PublisherFingerprint); + await repository.SaveAsync(entry, cancellationToken); + _entries[inspected.Manifest.Id] = entry; + catalogWritten = true; + await DisableIncompatiblePluginsAsync(cancellationToken); + await repository.RemoveRetainedAsync(inspected.Manifest.Id, cancellationToken); + UpdateSnapshot(); + + if (journal is not null) + { + journal = journal with { Phase = PluginLifecycleJournalValues.Committed }; + WriteLifecycleJournal(transactionPath!, journal); + journalCommitted = true; + LifecycleCheckpointForTesting?.Invoke(PluginLifecycleCheckpoint.AfterCommit); + DeleteLifecycleTransaction(transactionPath, inspected.Manifest.Id); + } + + if (existing is not null && !PathsEqual(existing.PackageDirectory, extractedPath) && + Directory.Exists(existing.PackageDirectory)) + { + try { Directory.Delete(existing.PackageDirectory, recursive: true); } + catch (IOException) { } + catch (UnauthorizedAccessException) { } + } + try { packageInspector.Consume(inspected); } + catch (IOException) { } + catch (UnauthorizedAccessException) { } + return new PluginInstallResult( + inspected.Manifest.Id, + inspected.Manifest.Version, + existing is not null, + GetCompatibilityErrors(inspected.Manifest)); + } + catch (PluginProcessCrashSimulationException) + { + throw; + } + catch (Exception) when (journalCommitted) + { + // The new catalog is committed; startup will finish journal cleanup. + throw; + } + catch (Exception failure) + { + try + { + if (catalogWritten) + { + if (existing is null) + await repository.RemoveAsync(inspected.Manifest.Id, CancellationToken.None); + foreach (var original in originalEntries.Values) + await repository.SaveAsync(original, CancellationToken.None); + if (retained is not null) + await repository.SaveRetainedAsync(retained, CancellationToken.None); + } + _entries.Clear(); + foreach (var original in originalEntries) _entries[original.Key] = original.Value; + UpdateSnapshot(); + if (extractedPath is not null && Directory.Exists(extractedPath)) + Directory.Delete(extractedPath, recursive: true); + RestoreTransactionDirectory(dataBackupPath, GetDataPath(inspected.Manifest.Id)); + DeleteLifecycleTransactionBestEffort(transactionPath, inspected.Manifest.Id); + } + catch (Exception rollbackFailure) + { + throw new AggregateException("Plugin installation failed and rollback was incomplete.", + failure, rollbackFailure); + } + throw; + } + finally + { + lifecycle?.Dispose(); + _gate.Release(); + } + } + + private async Task RecordSuccessAsync(string id, CancellationToken cancellationToken) + { + await _gate.WaitAsync(cancellationToken); + try + { + if (_pendingLifecyclePluginIds.Count > 0) return; + if (!_entries.TryGetValue(id, out var entry)) return; + entry = entry with + { + Health = entry.Health with + { + Status = "healthy", + ConsecutiveFailures = 0, + LastSuccessAt = timeProvider.GetUtcNow(), + LastError = null, + CircuitOpenUntil = null + } + }; + await repository.SaveAsync(entry, cancellationToken); + _entries[id] = entry; + UpdateSnapshot(); + } + finally + { + _gate.Release(); + } + } + + private async Task RecordFailureAsync(string id, Exception exception, CancellationToken cancellationToken) + { + await _gate.WaitAsync(cancellationToken); + try + { + if (_pendingLifecyclePluginIds.Count > 0) return; + if (!_entries.TryGetValue(id, out var entry)) return; + var failures = checked(entry.Health.ConsecutiveFailures + 1); + DateTimeOffset? openUntil = failures >= _options.CircuitBreakerFailures + ? timeProvider.GetUtcNow().AddSeconds(_options.CircuitBreakerSeconds) + : null; + entry = entry with + { + Health = entry.Health with + { + Status = openUntil is null ? "degraded" : "circuit-open", + ConsecutiveFailures = failures, + LastFailureAt = timeProvider.GetUtcNow(), + LastError = Truncate(exception.Message, 1_024), + CircuitOpenUntil = openUntil + } + }; + await repository.SaveAsync(entry, cancellationToken); + _entries[id] = entry; + UpdateSnapshot(); + } + finally + { + _gate.Release(); + } + } + + private async Task DisableIncompatiblePluginsAsync(CancellationToken cancellationToken) + { + bool changed; + do + { + changed = false; + foreach (var pair in _entries.OrderBy(value => value.Key, StringComparer.Ordinal).ToArray()) + { + if (!pair.Value.IsEnabled || GetCompatibilityErrors(pair.Value.Manifest).Count == 0) continue; + using var lifecycle = await _lifecycle.BeginLifecycleAsync( + pair.Key, LifecycleWaitTimeout, cancellationToken); + var disabled = pair.Value with { IsEnabled = false }; + await repository.SaveAsync(disabled, cancellationToken); + _entries[pair.Key] = disabled; + changed = true; + } + } while (changed); + } + + private IReadOnlyList GetCompatibilityErrors(PluginManifest manifest) + { + var installed = _entries.ToDictionary( + pair => pair.Key, + pair => new PluginCatalogEntryView(pair.Value.Manifest.Version, pair.Value.IsEnabled), + StringComparer.Ordinal); + return PluginManifestValidator.GetCompatibilityErrors(manifest, installed); + } + + private IReadOnlyList CreateSnapshot() + => _entries.Values + .OrderBy(entry => entry.Manifest.Name, StringComparer.OrdinalIgnoreCase) + .Select(entry => new InstalledPlugin( + entry.Manifest, + entry.IsEnabled, + entry.ApprovedCapabilities, + GetCompatibilityErrors(entry.Manifest), + entry.Health, + ParseConfiguration(entry.ConfigurationJson))) + .ToArray(); + + private void UpdateSnapshot() => Volatile.Write(ref _snapshot, CreateSnapshot().ToArray()); + + private PluginCatalogEntry GetRequiredEntry(string id) + { + if (!PluginManifestValidator.IsValidId(id)) throw new ArgumentException("Invalid plugin id.", nameof(id)); + return _entries.TryGetValue(id, out var entry) + ? entry + : throw new KeyNotFoundException($"Plugin '{id}' is not installed."); + } + + private void EnsureNoPendingLifecycleManagement() + { + if (_pendingLifecyclePluginIds.Count > 0) + throw new InvalidOperationException( + $"A pending lifecycle recovery exists for '{string.Join("', '", _pendingLifecyclePluginIds.Order())}'. " + + "Restart the service to finalize it before another plugin operation."); + } + + private async Task EnsureInitializedAsync(CancellationToken cancellationToken) + { + if (!_initialized) await InitializeAsync(cancellationToken); + } + + private string CreateTransactionPath(string operation, string id) + { + if (!PluginManifestValidator.IsValidId(id)) throw new ArgumentException("Invalid plugin id.", nameof(id)); + var transactionRoot = Path.Combine(_rootPath, "transactions"); + Directory.CreateDirectory(transactionRoot); + RestrictDirectory(transactionRoot); + var transactionPath = Path.Combine(transactionRoot, $"{operation}-{id}-{Guid.NewGuid():N}"); + Directory.CreateDirectory(transactionPath); + RestrictDirectory(transactionPath); + return transactionPath; + } + + private void WriteLifecycleJournal(string transactionPath, PluginLifecycleJournal journal) + { + var transactionRoot = Path.GetFullPath(Path.Combine(_rootPath, "transactions")); + var fullTransactionPath = Path.GetFullPath(transactionPath); + if (!IsStrictlyWithin(fullTransactionPath, transactionRoot)) + throw new UnauthorizedAccessException("Plugin lifecycle transaction is outside the transaction root."); + var journalPath = GetLifecycleJournalPath(fullTransactionPath); + var temporaryPath = $"{journalPath}.{Guid.NewGuid():N}.tmp"; + try + { + using (var stream = new FileStream(temporaryPath, FileMode.CreateNew, FileAccess.Write, + FileShare.None, 16 * 1024, FileOptions.WriteThrough)) + { + JsonSerializer.Serialize(stream, journal, JournalJsonOptions); + stream.Flush(flushToDisk: true); + } + RestrictFile(temporaryPath); + File.Move(temporaryPath, journalPath, overwrite: true); + _pendingLifecyclePluginIds.Add(journal.PluginId); + } + finally + { + if (File.Exists(temporaryPath)) File.Delete(temporaryPath); + } + } + + private async Task RecoverLifecycleTransactionsAsync(CancellationToken cancellationToken) + { + var transactionRoot = Path.Combine(_rootPath, "transactions"); + if (!Directory.Exists(transactionRoot)) return; + RestrictDirectory(transactionRoot); + var pending = new List<(string TransactionPath, PluginLifecycleJournal Journal)>(); + foreach (var journalPath in Directory.EnumerateFiles( + transactionRoot, $"*{LifecycleJournalSuffix}", SearchOption.TopDirectoryOnly) + .Order(StringComparer.Ordinal)) + { + cancellationToken.ThrowIfCancellationRequested(); + var transactionPath = journalPath[..^LifecycleJournalSuffix.Length]; + if (!IsStrictlyWithin(Path.GetFullPath(transactionPath), Path.GetFullPath(transactionRoot))) + throw new InvalidDataException("Plugin lifecycle transaction path is invalid."); + + PluginLifecycleJournal? journal; + await using (var stream = File.OpenRead(journalPath)) + journal = await JsonSerializer.DeserializeAsync( + stream, JournalJsonOptions, cancellationToken); + ValidateLifecycleJournal(journal, transactionPath); + pending.Add((transactionPath, journal!)); + } + + if (pending.GroupBy(item => item.Journal.PluginId, StringComparer.Ordinal) + .Any(group => group.Count() > 1)) + throw new InvalidDataException("Multiple pending lifecycle journals exist for the same plugin."); + foreach (var item in pending) _pendingLifecyclePluginIds.Add(item.Journal.PluginId); + + foreach (var item in pending) + { + var current = await repository.FindAsync(item.Journal.PluginId, cancellationToken); + var catalogCommitted = CatalogReflectsCommittedJournal(item.Journal, current); + if (item.Journal.Phase == PluginLifecycleJournalValues.Committed && catalogCommitted) + await FinalizeCommittedJournalAsync( + item.Journal, item.TransactionPath, cancellationToken); + else + await RollbackPreparedJournalAsync( + item.Journal, item.TransactionPath, cancellationToken); + } + + foreach (var temporaryJournal in Directory.EnumerateFiles( + transactionRoot, $"*{LifecycleJournalSuffix}.*.tmp", SearchOption.TopDirectoryOnly)) + File.Delete(temporaryJournal); + foreach (var transactionPath in Directory.EnumerateDirectories(transactionRoot).ToArray()) + { + if (File.Exists(GetLifecycleJournalPath(transactionPath))) continue; + if (Directory.Exists(Path.Combine(transactionPath, "package")) || + Directory.Exists(Path.Combine(transactionPath, "data"))) + throw new InvalidDataException( + $"Plugin lifecycle transaction '{Path.GetFileName(transactionPath)}' has payload but no recovery journal."); + DeleteTransaction(transactionPath); + } + } + + private void ValidateLifecycleJournal(PluginLifecycleJournal? journal, string transactionPath) + { + if (journal is null || + journal.Operation is not (PluginLifecycleJournalValues.Upgrade or PluginLifecycleJournalValues.Uninstall) || + journal.Phase is not (PluginLifecycleJournalValues.Prepared or PluginLifecycleJournalValues.Committed) || + !PluginManifestValidator.IsValidId(journal.PluginId) || + journal.OriginalEntries.GroupBy(entry => entry.Manifest.Id, StringComparer.Ordinal) + .Any(group => group.Count() > 1)) + throw new InvalidDataException( + $"Plugin lifecycle transaction '{Path.GetFileName(transactionPath)}' has an invalid journal."); + + var packagesRoot = Path.GetFullPath(Path.Combine(_rootPath, "packages")); + foreach (var entry in journal.OriginalEntries) + { + if (!PluginManifestValidator.IsValidId(entry.Manifest.Id) || + !IsStrictlyWithin(Path.GetFullPath(entry.PackageDirectory), packagesRoot)) + throw new InvalidDataException("Plugin lifecycle journal contains an invalid catalog path."); + } + + var original = journal.OriginalEntries.SingleOrDefault(entry => entry.Manifest.Id == journal.PluginId); + if (journal.Operation == PluginLifecycleJournalValues.Uninstall && original is null) + throw new InvalidDataException("Uninstall recovery journal is missing the original plugin catalog entry."); + if (journal.OriginalRetained is { } retained && retained.Id != journal.PluginId || + journal.IntendedRetained is { } intended && intended.Id != journal.PluginId) + throw new InvalidDataException("Plugin lifecycle journal contains retained data for another plugin."); + } + + private static bool CatalogReflectsCommittedJournal( + PluginLifecycleJournal journal, + PluginCatalogEntry? current) + { + if (journal.Operation == PluginLifecycleJournalValues.Uninstall) return current is null; + if (current is null) return false; + var original = journal.OriginalEntries.SingleOrDefault(entry => entry.Manifest.Id == journal.PluginId); + return original is null || + !string.Equals(current.Manifest.Version, original.Manifest.Version, StringComparison.Ordinal) || + !PathsEqual(current.PackageDirectory, original.PackageDirectory); + } + + private async Task RollbackPreparedJournalAsync( + PluginLifecycleJournal journal, + string transactionPath, + CancellationToken cancellationToken) + { + var original = journal.OriginalEntries.SingleOrDefault(entry => entry.Manifest.Id == journal.PluginId); + if (journal.Operation == PluginLifecycleJournalValues.Uninstall) + RestoreTransactionDirectory( + Path.Combine(transactionPath, "package"), original!.PackageDirectory); + RestoreTransactionDirectory( + Path.Combine(transactionPath, "data"), GetDataPath(journal.PluginId)); + + if (journal.OriginalRetained is null) + await repository.RemoveRetainedAsync(journal.PluginId, cancellationToken); + else + await repository.SaveRetainedAsync(journal.OriginalRetained, cancellationToken); + + if (original is null) + await repository.RemoveAsync(journal.PluginId, cancellationToken); + foreach (var entry in journal.OriginalEntries) + await repository.SaveAsync(entry, cancellationToken); + DeleteLifecycleTransaction(transactionPath, journal.PluginId); + } + + private async Task FinalizeCommittedJournalAsync( + PluginLifecycleJournal journal, + string transactionPath, + CancellationToken cancellationToken) + { + if (journal.Operation == PluginLifecycleJournalValues.Uninstall) + { + await repository.RemoveAsync(journal.PluginId, cancellationToken); + if (journal.DeleteData) + { + await repository.RemoveRetainedAsync(journal.PluginId, cancellationToken); + DeleteDirectoryWithinRoot(GetDataPath(journal.PluginId)); + } + else if (journal.IntendedRetained is not null) + { + await repository.SaveRetainedAsync(journal.IntendedRetained, cancellationToken); + } + } + else + { + await repository.RemoveRetainedAsync(journal.PluginId, cancellationToken); + } + + DeleteLifecycleTransaction(transactionPath, journal.PluginId); + } + + private async Task DisableMissingPackagePluginsAsync(CancellationToken cancellationToken) + { + var packagesRoot = Path.GetFullPath(Path.Combine(_rootPath, "packages")); + foreach (var pair in _entries.OrderBy(value => value.Key, StringComparer.Ordinal).ToArray()) + { + var packagePath = Path.GetFullPath(pair.Value.PackageDirectory); + if (IsStrictlyWithin(packagePath, packagesRoot) && Directory.Exists(packagePath)) continue; + var disabled = pair.Value with + { + IsEnabled = false, + Health = pair.Value.Health with + { + Status = "missing-package", + LastError = "The installed plugin package directory is missing or invalid." + } + }; + await repository.SaveAsync(disabled, cancellationToken); + _entries[pair.Key] = disabled; + } + } + + private void CleanupUnreferencedPackages() + { + var packagesRoot = Path.GetFullPath(Path.Combine(_rootPath, "packages")); + if (!Directory.Exists(packagesRoot)) return; + var referenced = _entries.Values + .Select(entry => Path.GetFullPath(entry.PackageDirectory)) + .Where(path => IsStrictlyWithin(path, packagesRoot)) + .ToHashSet(OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal); + foreach (var pluginDirectory in Directory.EnumerateDirectories(packagesRoot)) + { + foreach (var packageDirectory in Directory.EnumerateDirectories(pluginDirectory)) + { + if (!referenced.Contains(Path.GetFullPath(packageDirectory))) + DeleteTransactionBestEffort(packageDirectory); + } + if (!Directory.EnumerateFileSystemEntries(pluginDirectory).Any()) + DeleteTransactionBestEffort(pluginDirectory); + } + } + + private void DeleteDirectoryWithinRoot(string path) + { + var fullPath = Path.GetFullPath(path); + if (!IsStrictlyWithin(fullPath, _rootPath)) + throw new UnauthorizedAccessException("Plugin lifecycle cleanup path is outside the platform root."); + if (Directory.Exists(fullPath)) Directory.Delete(fullPath, recursive: true); + } + + private string GetDataPath(string id) => Path.Combine(_rootPath, "data", id); + + private TimeSpan LifecycleWaitTimeout => + TimeSpan.FromMilliseconds(_options.InvocationTimeoutMilliseconds + 2_000); + + private static void EnsurePublisherContinuity(string? existingFingerprint, string? incomingFingerprint) + { + if (string.Equals(existingFingerprint, incomingFingerprint, StringComparison.Ordinal)) return; + throw new UnauthorizedAccessException( + "Plugin publisher identity does not match the installed or retained owner. Delete retained data before transferring ownership."); + } + + private string? MoveToTransaction(string source, string destination) + { + if (!Directory.Exists(source)) return null; + var fullSource = Path.GetFullPath(source); + if (!IsWithin(fullSource, _rootPath)) + throw new UnauthorizedAccessException("Plugin lifecycle path is outside the plugin platform root."); + Directory.Move(fullSource, destination); + return destination; + } + + private static void RestoreTransactionDirectory(string? backup, string? destination) + { + if (backup is null || destination is null || !Directory.Exists(backup)) return; + if (Directory.Exists(destination)) Directory.Delete(destination, recursive: true); + Directory.CreateDirectory(Path.GetDirectoryName(destination)!); + Directory.Move(backup, destination); + } + + private static void DeleteTransactionBestEffort(string? transactionPath) + { + if (transactionPath is null || !Directory.Exists(transactionPath)) return; + try { Directory.Delete(transactionPath, recursive: true); } + catch (IOException) { } + catch (UnauthorizedAccessException) { } + } + + private static void DeleteTransaction(string? transactionPath) + { + if (transactionPath is not null && Directory.Exists(transactionPath)) + Directory.Delete(transactionPath, recursive: true); + } + + private static string GetLifecycleJournalPath(string transactionPath) + => $"{transactionPath}{LifecycleJournalSuffix}"; + + private void DeleteLifecycleTransaction(string? transactionPath, string pluginId) + { + if (transactionPath is null) return; + DeleteTransaction(transactionPath); + var journalPath = GetLifecycleJournalPath(transactionPath); + if (File.Exists(journalPath)) File.Delete(journalPath); + if (!Directory.Exists(transactionPath) && !File.Exists(journalPath)) + _pendingLifecyclePluginIds.Remove(pluginId); + } + + private void DeleteLifecycleTransactionBestEffort(string? transactionPath, string pluginId) + { + if (transactionPath is null) return; + DeleteTransactionBestEffort(transactionPath); + if (Directory.Exists(transactionPath)) return; + try + { + var journalPath = GetLifecycleJournalPath(transactionPath); + if (File.Exists(journalPath)) File.Delete(journalPath); + if (!File.Exists(journalPath)) _pendingLifecyclePluginIds.Remove(pluginId); + } + catch (IOException) { } + catch (UnauthorizedAccessException) { } + } + + private static JsonElement ParseConfiguration(string json) + { + using var document = JsonDocument.Parse(json); + return document.RootElement.Clone(); + } + + private static bool PathsEqual(string left, string right) + => string.Equals(Path.GetFullPath(left), Path.GetFullPath(right), OperatingSystem.IsWindows() + ? StringComparison.OrdinalIgnoreCase + : StringComparison.Ordinal); + + private static bool IsWithin(string candidate, string root) + { + var relative = Path.GetRelativePath(Path.GetFullPath(root), candidate); + return relative != ".." && !relative.StartsWith($"..{Path.DirectorySeparatorChar}", StringComparison.Ordinal) && + !Path.IsPathFullyQualified(relative); + } + + private static bool IsStrictlyWithin(string candidate, string root) + => !PathsEqual(candidate, root) && IsWithin(candidate, root); + + private static string Truncate(string value, int length) => value.Length <= length ? value : value[..length]; + + private static void RestrictDirectory(string path) + { + if (!OperatingSystem.IsWindows()) + File.SetUnixFileMode(path, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + } + + private static void RestrictFile(string path) + { + if (!OperatingSystem.IsWindows()) + File.SetUnixFileMode(path, UnixFileMode.UserRead | UnixFileMode.UserWrite); + } +} diff --git a/SecondDimensionWatcherReDive/PluginPlatform/PluginManifestValidator.cs b/SecondDimensionWatcherReDive/PluginPlatform/PluginManifestValidator.cs new file mode 100644 index 0000000..dab1fe0 --- /dev/null +++ b/SecondDimensionWatcherReDive/PluginPlatform/PluginManifestValidator.cs @@ -0,0 +1,333 @@ +using System.Runtime.InteropServices; +using System.Text.RegularExpressions; +using SecondDimensionWatcherReDive.Framework.Plugin; + +namespace SecondDimensionWatcherReDive.PluginPlatform; + +internal static partial class PluginManifestValidator +{ + [GeneratedRegex("^[a-z](?:[a-z0-9-]*[a-z0-9])?(?:\\.[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)*$", + RegexOptions.CultureInvariant)] + private static partial Regex IdPattern(); + + [GeneratedRegex("^[A-Za-z][A-Za-z0-9_.-]{0,63}$", RegexOptions.CultureInvariant)] + private static partial Regex HandlerNamePattern(); + + private const int MaximumVersionLength = 128; + + public static bool IsValidId(string id) + => !string.IsNullOrWhiteSpace(id) && id.Length is >= 3 and <= 64 && IdPattern().IsMatch(id); + public static bool IsValidHandlerName(string name) + => !string.IsNullOrWhiteSpace(name) && HandlerNamePattern().IsMatch(name); + + public static IReadOnlyList Validate(PluginManifest manifest) + { + var errors = new List(); + if (!IsValidId(manifest.Id)) + errors.Add("Plugin id must be 3-64 lowercase ASCII characters in non-empty dot-separated segments."); + if (string.IsNullOrWhiteSpace(manifest.Name) || manifest.Name.Length > 128 || + manifest.Name.Any(char.IsControl)) + errors.Add("Plugin name is required, must be at most 128 characters, and cannot contain control characters."); + if (manifest.Description is { } description && + (description.Length > 2_048 || description.Any(IsDisallowedDescriptionCharacter))) + errors.Add("Plugin description must be at most 2048 characters and cannot contain unsafe control characters."); + if (!TryParseVersion(manifest.Version, out _)) errors.Add("Plugin version must be a valid semantic version."); + if (!TryParseApiVersion(manifest.ApiVersion, out _)) errors.Add("API version must be a valid API version."); + if (!IsSafeRelativePath(manifest.EntryPoint) || !manifest.EntryPoint.EndsWith(".js", StringComparison.OrdinalIgnoreCase)) + errors.Add("Entry point must be a relative JavaScript file path."); + if (manifest.DataVersion < 1) errors.Add("Data version must be at least 1."); + + foreach (var dependency in manifest.Dependencies) + { + if (!IsValidId(dependency.Id)) errors.Add($"Dependency id '{dependency.Id}' is invalid."); + if (!TryParseVersion(dependency.MinimumVersion, out _)) + errors.Add($"Dependency '{dependency.Id}' has an invalid minimum version."); + } + + if (manifest.Dependencies.GroupBy(x => x.Id, StringComparer.Ordinal).Any(group => group.Count() > 1)) + errors.Add("Dependencies must not contain duplicate ids."); + if (manifest.Providers.GroupBy(x => $"{x.Kind}:{x.Name}", StringComparer.Ordinal).Any(group => group.Count() > 1)) + errors.Add("Provider declarations must have unique kind/name pairs."); + + foreach (var provider in manifest.Providers) + { + if (provider.Kind is not ("notification" or "storage")) + errors.Add($"Provider '{provider.Name}' has unsupported kind '{provider.Kind}'."); + if (string.IsNullOrWhiteSpace(provider.Name) || provider.Handlers.Count == 0) + errors.Add("Provider declarations require a name and at least one handler."); + if (!IsValidIdentifier(provider.Name)) + errors.Add($"Provider '{provider.Name}' name must be a 1-64 character ASCII identifier."); + if (provider.Handlers.Keys.Any(operation => !IsValidIdentifier(operation))) + errors.Add($"Provider '{provider.Name}' contains an invalid operation name."); + if (provider.Handlers.Values.Any(handler => !IsValidHandlerName(handler))) + errors.Add($"Provider '{provider.Name}' contains an invalid handler name."); + if (provider.Kind == "notification" && !provider.Handlers.ContainsKey("send")) + errors.Add($"Notification provider '{provider.Name}' requires a send handler."); + if (provider.Kind == "storage" && + new[] { "exists", "info", "read", "list" }.Any(operation => !provider.Handlers.ContainsKey(operation))) + errors.Add($"Storage provider '{provider.Name}' requires exists, info, read and list handlers."); + } + + if (manifest.Providers.Any(provider => provider.Kind == "storage") && !manifest.Capabilities.StorageAccess) + errors.Add("Storage providers require the storageAccess capability."); + if (manifest.Providers.Any(provider => provider.Kind == "notification") && + !manifest.Capabilities.Notifications) + errors.Add("Notification providers require the notifications capability."); + + foreach (var domain in manifest.Capabilities.NetworkDomains) + { + if (!IsValidDomainPattern(domain)) errors.Add($"Network domain '{domain}' is invalid."); + } + + foreach (var root in manifest.Capabilities.FileRoots) + { + if (!Path.IsPathFullyQualified(root)) errors.Add($"File root '{root}' must be absolute."); + } + + if (manifest.Integrity?.Files is not { Count: > 0 } files) + { + errors.Add("Integrity metadata must contain a SHA-256 digest for every package file."); + } + else + { + foreach (var file in files) + { + if (!IsSafeArchivePath(file.Key) || + file.Key.Equals("manifest.json", StringComparison.OrdinalIgnoreCase)) + errors.Add($"Integrity path '{file.Key}' is invalid."); + if (!IsSha256(file.Value)) + errors.Add($"Integrity digest for '{file.Key}' must be a valid SHA-256 value."); + } + if (!files.ContainsKey(manifest.EntryPoint.Replace('\\', '/'))) + errors.Add("Integrity metadata must include the entry point."); + } + if (manifest.Signature is { Algorithm: not "RSA-SHA256" }) + errors.Add("Only RSA-SHA256 signatures are supported."); + if (manifest.Signature is { } signature && !IsValidIdentifier(signature.Publisher)) + errors.Add("Signature publisher must be a 1-64 character ASCII identifier."); + if (manifest.DataMigration is { } migration && + !string.Equals(migration.Strategy, "preserve", StringComparison.OrdinalIgnoreCase) && + !string.Equals(migration.Strategy, "reset", StringComparison.OrdinalIgnoreCase)) + errors.Add("Data migration strategy must be 'preserve' or 'reset'."); + if (manifest.DataMigration?.Description is { } migrationDescription && + (migrationDescription.Length > 2_048 || + migrationDescription.Any(IsDisallowedDescriptionCharacter))) + errors.Add("Data migration description must be at most 2048 characters and cannot contain unsafe control characters."); + + return errors; + } + + public static IReadOnlyList GetCompatibilityErrors( + PluginManifest manifest, + IReadOnlyDictionary installed) + { + var errors = new List(); + if (!TryParseApiVersion(manifest.ApiVersion, out var requested) || + !TryParseApiVersion(PluginApi.CurrentVersion, out var current) || + requested.Major != current.Major || requested > current) + { + errors.Add($"Plugin API {manifest.ApiVersion} is incompatible with host API {PluginApi.CurrentVersion}."); + } + + var platform = $"{GetOs()}-{RuntimeInformation.ProcessArchitecture.ToString().ToLowerInvariant()}"; + if (manifest.Platforms.Count > 0 && + !manifest.Platforms.Contains("any", StringComparer.OrdinalIgnoreCase) && + !manifest.Platforms.Contains(platform, StringComparer.OrdinalIgnoreCase)) + { + errors.Add($"Plugin does not support host platform '{platform}'."); + } + if (OperatingSystem.IsWindows() && + (manifest.Capabilities.StorageAccess || manifest.Capabilities.FileRoots.Count > 0 || + manifest.Providers.Any(provider => provider.Kind == "storage"))) + { + errors.Add("Plugin file and storage capabilities are not supported on Windows in API 1.0."); + } + + foreach (var dependency in manifest.Dependencies) + { + if (!installed.TryGetValue(dependency.Id, out var installedDependency)) + { + errors.Add($"Required dependency '{dependency.Id}' is not installed."); + continue; + } + + if (!installedDependency.IsEnabled) + errors.Add($"Required dependency '{dependency.Id}' is disabled."); + if (!TryParseVersion(installedDependency.Version, out var actual) || + !TryParseVersion(dependency.MinimumVersion, out var minimum) || actual < minimum) + { + errors.Add($"Dependency '{dependency.Id}' requires >= {dependency.MinimumVersion}; installed version is {installedDependency.Version}."); + } + } + + return errors; + } + + public static bool CapabilitiesEqual(PluginCapabilities left, PluginCapabilities right) + => left.Notifications == right.Notifications && + left.DownloadControl == right.DownloadControl && + left.StorageAccess == right.StorageAccess && + left.BackgroundTasks == right.BackgroundTasks && + SetEqual(left.NetworkDomains, right.NetworkDomains, StringComparer.OrdinalIgnoreCase) && + SetEqual(left.FileRoots.Select(Path.GetFullPath), right.FileRoots.Select(Path.GetFullPath), PathComparer); + + public static bool TryParseVersion(string value, out SemanticVersion version) + => TryParseSemanticVersion(value, requirePatch: true, out version); + + private static bool TryParseApiVersion(string value, out SemanticVersion version) + => TryParseSemanticVersion(value, requirePatch: false, out version); + + private static bool TryParseSemanticVersion( + string value, + bool requirePatch, + out SemanticVersion version) + { + version = default; + if (string.IsNullOrEmpty(value) || value.Length > MaximumVersionLength || + value.Any(character => character > 0x7f)) + return false; + + var buildSeparator = value.IndexOf('+'); + var withoutBuild = buildSeparator < 0 ? value : value[..buildSeparator]; + if (buildSeparator >= 0) + { + var build = value[(buildSeparator + 1)..]; + if (!AreValidIdentifiers(build, rejectNumericLeadingZero: false)) return false; + } + + var prereleaseSeparator = withoutBuild.IndexOf('-'); + var core = prereleaseSeparator < 0 ? withoutBuild : withoutBuild[..prereleaseSeparator]; + var prerelease = prereleaseSeparator < 0 ? [] : withoutBuild[(prereleaseSeparator + 1)..].Split('.'); + if (prereleaseSeparator >= 0 && !AreValidIdentifiers( + withoutBuild[(prereleaseSeparator + 1)..], rejectNumericLeadingZero: true)) + return false; + + var components = core.Split('.'); + if (components.Length != 3 && (requirePatch || components.Length != 2)) return false; + var patch = 0; + if (!TryParseNumericComponent(components[0], out var major) || + !TryParseNumericComponent(components[1], out var minor) || + (components.Length == 3 && !TryParseNumericComponent(components[2], out patch))) + return false; + + version = new SemanticVersion(major, minor, patch, prerelease); + return true; + } + + private static bool TryParseNumericComponent(string value, out int component) + { + component = 0; + return value.Length > 0 && (value.Length == 1 || value[0] != '0') && + value.All(IsAsciiDigit) && + int.TryParse(value, System.Globalization.NumberStyles.None, + System.Globalization.CultureInfo.InvariantCulture, out component); + } + + private static bool AreValidIdentifiers(string value, bool rejectNumericLeadingZero) + { + var identifiers = value.Split('.'); + return identifiers.All(identifier => + identifier.Length > 0 && + identifier.All(character => IsAsciiLetterOrDigit(character) || character == '-') && + (!rejectNumericLeadingZero || !identifier.All(IsAsciiDigit) || + identifier.Length == 1 || identifier[0] != '0')); + } + + private static bool IsAsciiDigit(char value) => value is >= '0' and <= '9'; + + private static bool IsAsciiLetterOrDigit(char value) + => IsAsciiDigit(value) || value is >= 'A' and <= 'Z' or >= 'a' and <= 'z'; + + private static bool IsValidIdentifier(string value) + => !string.IsNullOrEmpty(value) && HandlerNamePattern().IsMatch(value); + + private static bool IsDisallowedDescriptionCharacter(char value) + => char.IsControl(value) && value is not '\r' and not '\n' and not '\t'; + + public static bool IsSafeRelativePath(string value) + { + if (string.IsNullOrWhiteSpace(value) || Path.IsPathFullyQualified(value)) return false; + var normalized = value.Replace('\\', '/'); + return normalized.Split('/', StringSplitOptions.RemoveEmptyEntries) + .All(segment => segment is not "." and not ".."); + } + + public static bool IsSafeArchivePath(string value) + { + if (string.IsNullOrWhiteSpace(value) || value.Contains('\\') || value.StartsWith('/') || + value.EndsWith('/') || value.Contains("//", StringComparison.Ordinal) || + value.Contains(':')) + return false; + return IsSafeRelativePath(value); + } + + private static bool IsSha256(string value) + => value.Length == 64 && value.All(Uri.IsHexDigit); + + private static bool IsValidDomainPattern(string value) + { + var domain = value.StartsWith("*.", StringComparison.Ordinal) ? value[2..] : value; + return Uri.CheckHostName(domain) == UriHostNameType.Dns && + !domain.Equals("localhost", StringComparison.OrdinalIgnoreCase); + } + + private static bool SetEqual(IEnumerable left, IEnumerable right, StringComparer comparer) + => new HashSet(left, comparer).SetEquals(right); + + private static string GetOs() + => OperatingSystem.IsWindows() ? "win" : OperatingSystem.IsMacOS() ? "osx" : "linux"; + + private static StringComparer PathComparer => OperatingSystem.IsWindows() + ? StringComparer.OrdinalIgnoreCase + : StringComparer.Ordinal; +} + +internal readonly record struct SemanticVersion( + int Major, + int Minor, + int Patch, + IReadOnlyList Prerelease) : IComparable +{ + public int CompareTo(SemanticVersion other) + { + var core = Major.CompareTo(other.Major); + if (core == 0) core = Minor.CompareTo(other.Minor); + if (core == 0) core = Patch.CompareTo(other.Patch); + if (core != 0) return core; + + if (Prerelease.Count == 0) return other.Prerelease.Count == 0 ? 0 : 1; + if (other.Prerelease.Count == 0) return -1; + for (var index = 0; index < Math.Min(Prerelease.Count, other.Prerelease.Count); index++) + { + var left = Prerelease[index]; + var right = other.Prerelease[index]; + var leftNumeric = left.All(character => character is >= '0' and <= '9'); + var rightNumeric = right.All(character => character is >= '0' and <= '9'); + int comparison; + if (leftNumeric && rightNumeric) + { + comparison = left.Length.CompareTo(right.Length); + if (comparison == 0) comparison = string.CompareOrdinal(left, right); + } + else if (leftNumeric != rightNumeric) + { + comparison = leftNumeric ? -1 : 1; + } + else + { + comparison = string.CompareOrdinal(left, right); + } + + if (comparison != 0) return comparison; + } + + return Prerelease.Count.CompareTo(other.Prerelease.Count); + } + + public static bool operator <(SemanticVersion left, SemanticVersion right) => left.CompareTo(right) < 0; + public static bool operator >(SemanticVersion left, SemanticVersion right) => left.CompareTo(right) > 0; + public static bool operator <=(SemanticVersion left, SemanticVersion right) => left.CompareTo(right) <= 0; + public static bool operator >=(SemanticVersion left, SemanticVersion right) => left.CompareTo(right) >= 0; +} + +internal sealed record PluginCatalogEntryView(string Version, bool IsEnabled); diff --git a/SecondDimensionWatcherReDive/PluginPlatform/PluginNetworkConnectionFactory.cs b/SecondDimensionWatcherReDive/PluginPlatform/PluginNetworkConnectionFactory.cs new file mode 100644 index 0000000..a681725 --- /dev/null +++ b/SecondDimensionWatcherReDive/PluginPlatform/PluginNetworkConnectionFactory.cs @@ -0,0 +1,119 @@ +using System.Net; +using System.Net.Sockets; + +namespace SecondDimensionWatcherReDive.PluginPlatform; + +internal interface IPluginDnsResolver +{ + Task ResolveAsync(string host, CancellationToken cancellationToken); +} + +internal sealed class SystemPluginDnsResolver : IPluginDnsResolver +{ + public Task ResolveAsync(string host, CancellationToken cancellationToken) + => IPAddress.TryParse(host, out var address) + ? Task.FromResult(new[] { address }) + : Dns.GetHostAddressesAsync(host, cancellationToken); +} + +internal static class PluginNetworkConnectionFactory +{ + public static SocketsHttpHandler Create(IPluginDnsResolver resolver) + => new() + { + AllowAutoRedirect = false, + UseCookies = false, + UseProxy = false, + ConnectCallback = (context, cancellationToken) => + ConnectAsync(context.DnsEndPoint, resolver, cancellationToken) + }; + + private static async ValueTask ConnectAsync( + DnsEndPoint endpoint, + IPluginDnsResolver resolver, + CancellationToken cancellationToken) + { + var addresses = await resolver.ResolveAsync(endpoint.Host, cancellationToken); + if (addresses.Length == 0) + throw new HttpRequestException($"Plugin network target '{endpoint.Host}' did not resolve."); + if (addresses.Any(address => !IsPublicAddress(address))) + throw new UnauthorizedAccessException( + $"Plugin network target '{endpoint.Host}' resolved to a non-public address."); + + List? failures = null; + foreach (var address in addresses) + { + var socket = new Socket(address.AddressFamily, SocketType.Stream, ProtocolType.Tcp) + { + NoDelay = true + }; + try + { + await socket.ConnectAsync(new IPEndPoint(address, endpoint.Port), cancellationToken); + return new NetworkStream(socket, ownsSocket: true); + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + socket.Dispose(); + (failures ??= []).Add(exception); + } + } + + throw new HttpRequestException( + $"Could not connect to approved plugin network target '{endpoint.Host}'.", + failures is { Count: 1 } ? failures[0] : new AggregateException(failures ?? [])); + } + + internal static bool IsPublicAddress(IPAddress address) + { + if (address.IsIPv4MappedToIPv6) address = address.MapToIPv4(); + var bytes = address.GetAddressBytes(); + if (address.AddressFamily == AddressFamily.InterNetwork) + { + return !InCidr(bytes, [0, 0, 0, 0], 8) && + !InCidr(bytes, [10, 0, 0, 0], 8) && + !InCidr(bytes, [100, 64, 0, 0], 10) && + !InCidr(bytes, [127, 0, 0, 0], 8) && + !InCidr(bytes, [169, 254, 0, 0], 16) && + !InCidr(bytes, [172, 16, 0, 0], 12) && + !InCidr(bytes, [192, 0, 0, 0], 24) && + !InCidr(bytes, [192, 0, 2, 0], 24) && + !InCidr(bytes, [192, 88, 99, 0], 24) && + !InCidr(bytes, [192, 168, 0, 0], 16) && + !InCidr(bytes, [198, 18, 0, 0], 15) && + !InCidr(bytes, [198, 51, 100, 0], 24) && + !InCidr(bytes, [203, 0, 113, 0], 24) && + !InCidr(bytes, [224, 0, 0, 0], 4) && + !InCidr(bytes, [240, 0, 0, 0], 4); + } + + if (address.AddressFamily != AddressFamily.InterNetworkV6) return false; + return InCidr(bytes, [0x20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 3) && + !address.Equals(IPAddress.IPv6Any) && + !address.Equals(IPAddress.IPv6Loopback) && + !address.IsIPv6LinkLocal && + !address.IsIPv6Multicast && + !InCidr(bytes, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 96) && + !InCidr(bytes, [0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 0, 0, 0, 0, 0, 0], 96) && + !InCidr(bytes, [0x00, 0x64, 0xff, 0x9b, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 96) && + !InCidr(bytes, [0x00, 0x64, 0xff, 0x9b, 0x00, 0x01, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 48) && + !InCidr(bytes, [0xfc, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 7) && + !InCidr(bytes, [0xfe, 0xc0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 10) && + !InCidr(bytes, [0x01, 0x00, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 64) && + !InCidr(bytes, [0x20, 0x01, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 23) && + !InCidr(bytes, [0x20, 0x01, 0x00, 0x02, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 48) && + !InCidr(bytes, [0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 32) && + !InCidr(bytes, [0x20, 0x02, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 16) && + !InCidr(bytes, [0x3f, 0xff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 20); + } + + private static bool InCidr(ReadOnlySpan address, ReadOnlySpan network, int prefixLength) + { + var wholeBytes = prefixLength / 8; + var remainingBits = prefixLength % 8; + if (!address[..wholeBytes].SequenceEqual(network[..wholeBytes])) return false; + if (remainingBits == 0) return true; + var mask = (byte)(0xff << (8 - remainingBits)); + return (address[wholeBytes] & mask) == (network[wholeBytes] & mask); + } +} diff --git a/SecondDimensionWatcherReDive/PluginPlatform/PluginPackageInspector.cs b/SecondDimensionWatcherReDive/PluginPlatform/PluginPackageInspector.cs new file mode 100644 index 0000000..6cf9717 --- /dev/null +++ b/SecondDimensionWatcherReDive/PluginPlatform/PluginPackageInspector.cs @@ -0,0 +1,414 @@ +using System.IO.Compression; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Options; +using SecondDimensionWatcherReDive.Framework.Plugin; + +namespace SecondDimensionWatcherReDive.PluginPlatform; + +internal sealed class PluginPackageInspector(IOptions options) +{ + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true + }; + + private readonly PluginPlatformOptions _options = options.Value; + private readonly string _rootPath = Path.GetFullPath(options.Value.RootPath); + private readonly SemaphoreSlim _stagingGate = new(1, 1); + + public async Task StageAndInspectAsync( + Stream package, + string fileName, + CancellationToken cancellationToken) + { + if (!fileName.EndsWith(".sdwpkg", StringComparison.OrdinalIgnoreCase) && + !fileName.EndsWith(".zip", StringComparison.OrdinalIgnoreCase)) + throw new InvalidDataException("Plugin packages must use the .sdwpkg or .zip extension."); + + await _stagingGate.WaitAsync(cancellationToken); + try + { + var stagingPath = Path.Combine(_rootPath, "staging"); + Directory.CreateDirectory(stagingPath); + RestrictDirectory(stagingPath); + CleanupExpiredPreviews(stagingPath); + var stagedPackages = Directory.EnumerateFiles( + stagingPath, "*.sdwpkg", SearchOption.TopDirectoryOnly).ToArray(); + if (stagedPackages.Length >= _options.MaximumStagedPackages) + throw new InvalidOperationException("The plugin preview staging limit has been reached."); + var stagedBytes = stagedPackages.Sum(path => new FileInfo(path).Length); + + var token = Convert.ToHexString(RandomNumberGenerator.GetBytes(24)).ToLowerInvariant(); + var packagePath = Path.Combine(stagingPath, $"{token}.sdwpkg"); + using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + try + { + await using (var target = new FileStream(packagePath, FileMode.CreateNew, FileAccess.Write, + FileShare.None, 64 * 1024, + FileOptions.Asynchronous | FileOptions.SequentialScan)) + { + var buffer = new byte[64 * 1024]; + long total = 0; + int read; + while ((read = await package.ReadAsync(buffer, cancellationToken)) > 0) + { + total = checked(total + read); + if (total > _options.MaximumPackageBytes) + throw new InvalidDataException( + $"Plugin package exceeds {_options.MaximumPackageBytes} bytes."); + if (stagedBytes + total > _options.MaximumStagedPackageBytes) + throw new InvalidOperationException("The plugin preview staging byte limit has been reached."); + hash.AppendData(buffer, 0, read); + await target.WriteAsync(buffer.AsMemory(0, read), cancellationToken); + } + } + + RestrictFile(packagePath); + var sha256 = Convert.ToHexString(hash.GetHashAndReset()).ToLowerInvariant(); + return await InspectAsync(token, packagePath, sha256, cancellationToken); + } + catch + { + if (File.Exists(packagePath)) File.Delete(packagePath); + throw; + } + } + finally + { + _stagingGate.Release(); + } + } + + public async Task InspectStagedAsync( + string token, + string expectedSha256, + CancellationToken cancellationToken) + { + if (string.IsNullOrEmpty(token) || token.Length != 48 || + token.Any(character => !Uri.IsHexDigit(character))) + throw new InvalidDataException("Invalid preview token."); + if (string.IsNullOrEmpty(expectedSha256) || expectedSha256.Length != 64 || + expectedSha256.Any(character => !Uri.IsHexDigit(character))) + throw new InvalidDataException("Expected package checksum must be a 64-character SHA-256 value."); + var path = Path.Combine(_rootPath, "staging", $"{token}.sdwpkg"); + if (!File.Exists(path)) throw new FileNotFoundException("Plugin preview expired or does not exist."); + if (File.GetLastWriteTimeUtc(path).AddMinutes(_options.PreviewLifetimeMinutes) < DateTime.UtcNow) + { + File.Delete(path); + throw new InvalidDataException("Plugin preview has expired."); + } + + var actualSha256 = await ComputeSha256Async(path, cancellationToken); + if (!CryptographicOperations.FixedTimeEquals( + Encoding.ASCII.GetBytes(actualSha256), Encoding.ASCII.GetBytes(expectedSha256.ToLowerInvariant()))) + throw new InvalidDataException("Package checksum no longer matches the approved preview."); + + return await InspectAsync(token, path, actualSha256, cancellationToken); + } + + public async Task ExtractAsync( + InspectedPluginPackage package, + CancellationToken cancellationToken) + { + var packagesRoot = Path.GetFullPath(Path.Combine(_rootPath, "packages")); + var pluginRoot = GetContainedChildPath(packagesRoot, package.Manifest.Id, "plugin id"); + var finalPath = GetContainedChildPath(pluginRoot, package.Manifest.Version, "plugin version"); + var temporaryPath = GetContainedChildPath( + pluginRoot, + $".{package.Manifest.Version}.{Guid.NewGuid():N}.tmp", + "temporary package path"); + Directory.CreateDirectory(pluginRoot); + RestrictDirectory(pluginRoot); + if (Directory.Exists(finalPath)) + throw new InvalidOperationException("This plugin version is already present on disk."); + Directory.CreateDirectory(temporaryPath); + RestrictDirectory(temporaryPath); + + try + { + using var archive = ZipFile.OpenRead(package.PackagePath); + var extractedFiles = new Dictionary(StringComparer.Ordinal); + long extractedBytes = 0; + foreach (var entry in archive.Entries) + { + cancellationToken.ThrowIfCancellationRequested(); + ValidateArchiveEntry(entry); + var destination = Path.GetFullPath(Path.Combine(temporaryPath, entry.FullName)); + if (!IsWithin(destination, temporaryPath)) + throw new InvalidDataException($"Archive entry '{entry.FullName}' escapes the package root."); + if (string.IsNullOrEmpty(entry.Name)) + { + Directory.CreateDirectory(destination); + continue; + } + + Directory.CreateDirectory(Path.GetDirectoryName(destination)!); + await using var source = entry.Open(); + await using var target = new FileStream(destination, FileMode.CreateNew, FileAccess.Write, + FileShare.None, 64 * 1024, FileOptions.Asynchronous); + using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + var buffer = new byte[64 * 1024]; + int read; + while ((read = await source.ReadAsync(buffer, cancellationToken)) > 0) + { + extractedBytes = checked(extractedBytes + read); + if (extractedBytes > _options.MaximumExpandedBytes) + throw new InvalidDataException( + $"Expanded package exceeds {_options.MaximumExpandedBytes} bytes."); + hash.AppendData(buffer, 0, read); + await target.WriteAsync(buffer.AsMemory(0, read), cancellationToken); + } + RestrictFile(destination); + if (!entry.FullName.Equals("manifest.json", StringComparison.Ordinal)) + { + extractedFiles[entry.FullName] = + Convert.ToHexString(hash.GetHashAndReset()).ToLowerInvariant(); + } + } + + VerifyFileManifest(package.Manifest, extractedFiles); + + Directory.Move(temporaryPath, finalPath); + return finalPath; + } + catch + { + if (Directory.Exists(temporaryPath)) Directory.Delete(temporaryPath, recursive: true); + throw; + } + } + + public void Consume(InspectedPluginPackage package) + { + if (File.Exists(package.PackagePath)) File.Delete(package.PackagePath); + } + + private async Task InspectAsync( + string token, + string packagePath, + string sha256, + CancellationToken cancellationToken) + { + using var archive = ZipFile.OpenRead(packagePath); + if (archive.Entries.Count == 0 || archive.Entries.Count > _options.MaximumPackageFiles) + throw new InvalidDataException($"Plugin package must contain 1-{_options.MaximumPackageFiles} files."); + long expandedBytes = 0; + if (archive.Entries.GroupBy(entry => entry.FullName, StringComparer.OrdinalIgnoreCase) + .Any(group => group.Count() > 1)) + throw new InvalidDataException("Plugin package contains duplicate archive paths."); + foreach (var entry in archive.Entries) + { + ValidateArchiveEntry(entry); + expandedBytes = checked(expandedBytes + entry.Length); + if (expandedBytes > _options.MaximumExpandedBytes) + throw new InvalidDataException($"Expanded package exceeds {_options.MaximumExpandedBytes} bytes."); + } + + var manifestEntry = archive.GetEntry("manifest.json") + ?? throw new InvalidDataException("Plugin package is missing manifest.json."); + if (manifestEntry.Length > 256 * 1024) throw new InvalidDataException("Plugin manifest is too large."); + PluginManifest? manifest; + await using (var stream = manifestEntry.Open()) + manifest = await JsonSerializer.DeserializeAsync(stream, JsonOptions, cancellationToken); + if (manifest is null) throw new InvalidDataException("Plugin manifest is invalid."); + manifest = Normalize(manifest); + + var validationErrors = PluginManifestValidator.Validate(manifest); + if (validationErrors.Count > 0) throw new InvalidDataException(string.Join(" ", validationErrors)); + var actualFiles = new Dictionary(StringComparer.Ordinal); + foreach (var entry in archive.Entries.Where(entry => !string.IsNullOrEmpty(entry.Name) && + !entry.FullName.Equals("manifest.json", StringComparison.Ordinal))) + { + actualFiles[entry.FullName] = await ComputeSha256Async(entry, cancellationToken); + } + VerifyFileManifest(manifest, actualFiles); + + var (trusted, status, publisherFingerprint) = VerifySignature(manifest); + return new InspectedPluginPackage( + token, + packagePath, + sha256, + manifest, + trusted, + status, + publisherFingerprint, + new DateTimeOffset(File.GetLastWriteTimeUtc(packagePath).AddMinutes(_options.PreviewLifetimeMinutes), + TimeSpan.Zero)); + } + + private (bool IsTrusted, string Status, string? PublisherFingerprint) VerifySignature(PluginManifest manifest) + { + if (manifest.Signature is null) return (false, "Package is unsigned.", null); + if (!_options.TrustedPublisherPublicKeys.TryGetValue(manifest.Signature.Publisher, out var publicKey)) + return (false, $"Publisher '{manifest.Signature.Publisher}' is not trusted by this deployment.", null); + try + { + using var rsa = RSA.Create(); + rsa.ImportFromPem(publicKey); + if (rsa.KeySize < 2048) + return (false, "Trusted publisher RSA keys must be at least 2048 bits.", null); + var payload = PluginSignaturePayload.Create(manifest); + var signature = Convert.FromBase64String(manifest.Signature.Value); + var valid = rsa.VerifyData(payload, signature, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); + var fingerprint = Convert.ToHexString(SHA256.HashData(rsa.ExportSubjectPublicKeyInfo())) + .ToLowerInvariant(); + return valid + ? (true, $"Signature verified for trusted publisher '{manifest.Signature.Publisher}'.", fingerprint) + : (false, "Package signature is invalid.", null); + } + catch (Exception exception) when (exception is CryptographicException or FormatException) + { + return (false, $"Package signature could not be verified: {exception.Message}", null); + } + } + + private static PluginManifest Normalize(PluginManifest manifest) + { + var capabilities = manifest.Capabilities ?? new PluginCapabilities(); + capabilities = capabilities with + { + NetworkDomains = capabilities.NetworkDomains?.Where(value => value is not null).ToArray() ?? [], + FileRoots = capabilities.FileRoots?.Where(value => value is not null).ToArray() ?? [] + }; + return manifest with + { + Id = manifest.Id ?? string.Empty, + Name = manifest.Name ?? string.Empty, + Version = manifest.Version ?? string.Empty, + ApiVersion = manifest.ApiVersion ?? string.Empty, + EntryPoint = manifest.EntryPoint ?? string.Empty, + Dependencies = manifest.Dependencies?.Where(value => value is not null).Select(value => + new PluginDependency(value.Id ?? string.Empty, value.MinimumVersion ?? string.Empty)).ToArray() ?? [], + Capabilities = capabilities, + Platforms = manifest.Platforms?.Where(value => value is not null).ToArray() ?? [], + Providers = manifest.Providers?.Where(value => value is not null).Select(value => value with + { + Kind = value.Kind ?? string.Empty, + Name = value.Name ?? string.Empty, + Handlers = value.Handlers ?? new Dictionary() + }).ToArray() ?? [], + Integrity = manifest.Integrity is null + ? null + : new PluginIntegrity + { + Files = manifest.Integrity.Files? + .Where(value => value.Key is not null && value.Value is not null) + .ToDictionary(value => value.Key, value => value.Value, StringComparer.Ordinal) + ?? new Dictionary(StringComparer.Ordinal) + }, + Signature = manifest.Signature is null + ? null + : new PluginSignature( + manifest.Signature.Publisher ?? string.Empty, + manifest.Signature.Algorithm ?? string.Empty, + manifest.Signature.Value ?? string.Empty), + DataMigration = manifest.DataMigration is null + ? null + : manifest.DataMigration with { Strategy = manifest.DataMigration.Strategy ?? string.Empty } + }; + } + + private void CleanupExpiredPreviews(string stagingPath) + { + foreach (var path in Directory.EnumerateFiles(stagingPath, "*.sdwpkg")) + { + if (File.GetLastWriteTimeUtc(path).AddMinutes(_options.PreviewLifetimeMinutes) < DateTime.UtcNow) + File.Delete(path); + } + } + + private static async Task ComputeSha256Async(string path, CancellationToken cancellationToken) + { + await using var stream = File.OpenRead(path); + var digest = await SHA256.HashDataAsync(stream, cancellationToken); + return Convert.ToHexString(digest).ToLowerInvariant(); + } + + private static async Task ComputeSha256Async( + ZipArchiveEntry entry, + CancellationToken cancellationToken) + { + await using var stream = entry.Open(); + var digest = await SHA256.HashDataAsync(stream, cancellationToken); + return Convert.ToHexString(digest).ToLowerInvariant(); + } + + private static void ValidateArchiveEntry(ZipArchiveEntry entry) + { + var path = string.IsNullOrEmpty(entry.Name) + ? entry.FullName.TrimEnd('/') + : entry.FullName; + if (!PluginManifestValidator.IsSafeArchivePath(path)) + throw new InvalidDataException($"Archive entry '{entry.FullName}' is unsafe."); + var unixFileType = (entry.ExternalAttributes >> 16) & 0xF000; + if (unixFileType == 0xA000) + throw new InvalidDataException($"Archive entry '{entry.FullName}' is a symbolic link."); + } + + private static void VerifyFileManifest( + PluginManifest manifest, + IReadOnlyDictionary actualFiles) + { + var expectedFiles = manifest.Integrity?.Files + ?? throw new InvalidDataException("Plugin integrity metadata is missing."); + var missing = expectedFiles.Keys.Except(actualFiles.Keys, StringComparer.Ordinal).Order().ToArray(); + var unlisted = actualFiles.Keys.Except(expectedFiles.Keys, StringComparer.Ordinal).Order().ToArray(); + if (missing.Length > 0 || unlisted.Length > 0) + { + throw new InvalidDataException( + $"Package file list does not match signed integrity metadata. Missing: {FormatPaths(missing)}; unlisted: {FormatPaths(unlisted)}."); + } + + foreach (var actual in actualFiles) + { + if (!actual.Value.Equals(expectedFiles[actual.Key], StringComparison.OrdinalIgnoreCase)) + throw new InvalidDataException($"Package file '{actual.Key}' failed its integrity check."); + } + } + + private static string FormatPaths(IReadOnlyList paths) + => paths.Count == 0 ? "none" : string.Join(", ", paths); + + private static bool IsWithin(string candidate, string root) + { + var relative = Path.GetRelativePath(root, candidate); + return relative != ".." && !relative.StartsWith($"..{Path.DirectorySeparatorChar}", StringComparison.Ordinal) && + !Path.IsPathFullyQualified(relative); + } + + private static string GetContainedChildPath(string root, string child, string fieldName) + { + var fullRoot = Path.GetFullPath(root); + var candidate = Path.GetFullPath(Path.Combine(fullRoot, child)); + if (!IsWithin(candidate, fullRoot) || + string.Equals(candidate, fullRoot, OperatingSystem.IsWindows() + ? StringComparison.OrdinalIgnoreCase + : StringComparison.Ordinal)) + throw new InvalidDataException($"The {fieldName} escapes its package directory."); + return candidate; + } + + private static void RestrictDirectory(string path) + { + if (!OperatingSystem.IsWindows()) + File.SetUnixFileMode(path, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + } + + private static void RestrictFile(string path) + { + if (!OperatingSystem.IsWindows()) + File.SetUnixFileMode(path, UnixFileMode.UserRead | UnixFileMode.UserWrite); + } +} + +internal sealed record InspectedPluginPackage( + string Token, + string PackagePath, + string PackageSha256, + PluginManifest Manifest, + bool IsSignatureTrusted, + string SignatureStatus, + string? PublisherFingerprint, + DateTimeOffset ExpiresAt); diff --git a/SecondDimensionWatcherReDive/PluginPlatform/PluginPlatformOptions.cs b/SecondDimensionWatcherReDive/PluginPlatform/PluginPlatformOptions.cs new file mode 100644 index 0000000..06882a7 --- /dev/null +++ b/SecondDimensionWatcherReDive/PluginPlatform/PluginPlatformOptions.cs @@ -0,0 +1,27 @@ +namespace SecondDimensionWatcherReDive.PluginPlatform; + +public sealed class PluginPlatformOptions +{ + public const string SectionName = "PluginPlatform"; + + public string RootPath { get; set; } = "./plugin-data"; + public bool AllowUnsignedLocalPackages { get; set; } + public long MaximumPackageBytes { get; set; } = 4 * 1024 * 1024; + public long MaximumExpandedBytes { get; set; } = 16 * 1024 * 1024; + public int MaximumPackageFiles { get; set; } = 128; + public int MaximumStagedPackages { get; set; } = 32; + public long MaximumStagedPackageBytes { get; set; } = 64 * 1024 * 1024; + public int InvocationTimeoutMilliseconds { get; set; } = 5_000; + public int MaximumWorkerMemoryMegabytes { get; set; } = 256; + public int MaximumWorkerCpuMilliseconds { get; set; } = 4_000; + public int MaximumConcurrentWorkers { get; set; } = 4; + public int MaximumConcurrentWorkersPerPlugin { get; set; } = 1; + public int MaximumResponseBytes { get; set; } = 2 * 1024 * 1024; + public long MaximumPluginDataBytes { get; set; } = 64 * 1024 * 1024; + public int MaximumPluginDataFiles { get; set; } = 1_000; + public int MaximumPluginDataPathDepth { get; set; } = 8; + public int CircuitBreakerFailures { get; set; } = 3; + public int CircuitBreakerSeconds { get; set; } = 60; + public int PreviewLifetimeMinutes { get; set; } = 30; + public Dictionary TrustedPublisherPublicKeys { get; set; } = new(StringComparer.Ordinal); +} diff --git a/SecondDimensionWatcherReDive/PluginPlatform/PluginPlatformServiceExtensions.cs b/SecondDimensionWatcherReDive/PluginPlatform/PluginPlatformServiceExtensions.cs new file mode 100644 index 0000000..dda4b9c --- /dev/null +++ b/SecondDimensionWatcherReDive/PluginPlatform/PluginPlatformServiceExtensions.cs @@ -0,0 +1,71 @@ +using Microsoft.Extensions.DependencyInjection.Extensions; +using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Framework.Plugin; +using SecondDimensionWatcherReDive.Repositories; + +namespace SecondDimensionWatcherReDive.PluginPlatform; + +internal static class PluginPlatformServiceExtensions +{ + public static IServiceCollection AddPluginPlatform( + this IServiceCollection services, + IConfiguration configuration) + { + services.AddOptions() + .Bind(configuration.GetSection(PluginPlatformOptions.SectionName)) + .Validate(options => options.MaximumPackageBytes is >= 1_024 and <= 64 * 1024 * 1024, + "MaximumPackageBytes must be between 1 KiB and 64 MiB.") + .Validate(options => options.MaximumExpandedBytes >= options.MaximumPackageBytes, + "MaximumExpandedBytes must be at least MaximumPackageBytes.") + .Validate(options => options.MaximumExpandedBytes <= 256 * 1024 * 1024, + "MaximumExpandedBytes must not exceed 256 MiB.") + .Validate(options => options.MaximumPackageFiles is >= 1 and <= 4_096, + "MaximumPackageFiles must be between 1 and 4,096.") + .Validate(options => options.MaximumStagedPackages is >= 1 and <= 1_024, + "MaximumStagedPackages must be between 1 and 1,024.") + .Validate(options => options.MaximumStagedPackageBytes >= options.MaximumPackageBytes && + options.MaximumStagedPackageBytes <= 4L * 1024 * 1024 * 1024, + "MaximumStagedPackageBytes must be at least MaximumPackageBytes and no greater than 4 GiB.") + .Validate(options => options.InvocationTimeoutMilliseconds is >= 100 and <= 60_000, + "Invocation timeout must be between 100 ms and 60 seconds.") + .Validate(options => options.MaximumWorkerMemoryMegabytes is >= 32 and <= 1_024, + "Worker memory must be between 32 MiB and 1 GiB.") + .Validate(options => options.MaximumWorkerCpuMilliseconds is >= 100 and <= 60_000, + "Worker CPU time must be between 100 ms and 60 seconds.") + .Validate(options => options.MaximumConcurrentWorkers is >= 1 and <= 32, + "MaximumConcurrentWorkers must be between 1 and 32.") + .Validate(options => options.MaximumConcurrentWorkersPerPlugin >= 1 && + options.MaximumConcurrentWorkersPerPlugin <= options.MaximumConcurrentWorkers, + "The per-plugin worker limit must be positive and no greater than the global worker limit.") + .Validate(options => options.MaximumPluginDataBytes is >= 1_024 and <= 10L * 1024 * 1024 * 1024, + "MaximumPluginDataBytes must be between 1 KiB and 10 GiB.") + .Validate(options => options.MaximumPluginDataFiles is >= 1 and <= 100_000, + "MaximumPluginDataFiles must be between 1 and 100,000.") + .Validate(options => options.MaximumPluginDataPathDepth is >= 1 and <= 64, + "MaximumPluginDataPathDepth must be between 1 and 64.") + .Validate(options => options.MaximumResponseBytes is >= 1_024 and <= 8 * 1024 * 1024, + "MaximumResponseBytes must be between 1 KiB and 8 MiB.") + .Validate(options => options.CircuitBreakerFailures is >= 1 and <= 100, + "CircuitBreakerFailures must be between 1 and 100.") + .Validate(options => options.CircuitBreakerSeconds is >= 1 and <= 86_400, + "CircuitBreakerSeconds must be between 1 second and 1 day.") + .Validate(options => options.PreviewLifetimeMinutes is >= 1 and <= 1_440, + "PreviewLifetimeMinutes must be between 1 minute and 1 day.") + .ValidateOnStart(); + services.TryAddSingleton(TimeProvider.System); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(provider => provider.GetRequiredService()); + services.AddSingleton(provider => provider.GetRequiredService()); + services.AddSingleton(); + services.AddHttpClient("PluginPlatform") + .ConfigurePrimaryHttpMessageHandler(provider => + PluginNetworkConnectionFactory.Create(provider.GetRequiredService())); + return services; + } +} diff --git a/SecondDimensionWatcherReDive/PluginPlatform/PluginProcessExecutor.cs b/SecondDimensionWatcherReDive/PluginPlatform/PluginProcessExecutor.cs new file mode 100644 index 0000000..578c0d0 --- /dev/null +++ b/SecondDimensionWatcherReDive/PluginPlatform/PluginProcessExecutor.cs @@ -0,0 +1,291 @@ +using System.Diagnostics; +using System.Collections.Concurrent; +using System.Reflection; +using System.Security.Cryptography; +using System.Text.Json; +using Microsoft.Extensions.Options; +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.PluginPlatform; + +internal interface IPluginProcessExecutor +{ + Task InvokeAsync( + PluginCatalogEntry plugin, + string handler, + JsonElement input, + CancellationToken cancellationToken); +} + +internal sealed class PluginCapacityExceededException(string message) : InvalidOperationException(message); + +internal sealed class PluginProcessExecutor( + IPluginCapabilityBroker capabilityBroker, + IOptions options) : IPluginProcessExecutor +{ + private readonly PluginPlatformOptions _options = options.Value; + private readonly SemaphoreSlim _globalGate = new( + options.Value.MaximumConcurrentWorkers, + options.Value.MaximumConcurrentWorkers); + private readonly ConcurrentDictionary _pluginGates = new(StringComparer.Ordinal); + + public async Task InvokeAsync( + PluginCatalogEntry plugin, + string handler, + JsonElement input, + CancellationToken cancellationToken) + { + var pluginGate = _pluginGates.GetOrAdd(plugin.Manifest.Id, _ => new SemaphoreSlim( + _options.MaximumConcurrentWorkersPerPlugin, + _options.MaximumConcurrentWorkersPerPlugin)); + if (!await pluginGate.WaitAsync(0, cancellationToken)) + throw new PluginCapacityExceededException( + $"Plugin '{plugin.Manifest.Id}' has reached its concurrent worker limit."); + try + { + if (!await _globalGate.WaitAsync(0, cancellationToken)) + throw new PluginCapacityExceededException("The global plugin worker limit has been reached."); + try + { + return await InvokeCoreAsync(plugin, handler, input, cancellationToken); + } + finally + { + _globalGate.Release(); + } + } + finally + { + pluginGate.Release(); + } + } + + private async Task InvokeCoreAsync( + PluginCatalogEntry plugin, + string handler, + JsonElement input, + CancellationToken cancellationToken) + { + var entryBytes = await ReadAndVerifyPackageAsync(plugin, cancellationToken); + var script = System.Text.Encoding.UTF8.GetString(entryBytes); + using var configDocument = JsonDocument.Parse(plugin.ConfigurationJson); + var invocation = new PluginWorkerInvocation( + script, + handler, + input.Clone(), + configDocument.RootElement.Clone(), + Math.Clamp(_options.MaximumWorkerMemoryMegabytes / 4, 16, 64), + _options.MaximumResponseBytes); + + using var process = new Process { StartInfo = CreateStartInfo() }; + if (!process.Start()) throw new InvalidOperationException("Could not start plugin worker process."); + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeout.CancelAfter(TimeSpan.FromMilliseconds(_options.InvocationTimeoutMilliseconds)); + var resourceViolation = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var monitor = MonitorProcessAsync(process, resourceViolation, timeout.Token); + + try + { + var invocationJson = JsonSerializer.Serialize( + invocation, + PluginWorkerJsonContext.Default.PluginWorkerInvocation); + await process.StandardInput.WriteLineAsync(invocationJson.AsMemory(), timeout.Token); + await process.StandardInput.FlushAsync(timeout.Token); + + while (true) + { + var line = await process.StandardOutput.ReadLineAsync(timeout.Token); + if (line is null) + { + if (resourceViolation.Task.IsCompletedSuccessfully) + throw new TimeoutException(resourceViolation.Task.Result); + var stderr = await ReadErrorAsync(process); + throw new InvalidOperationException( + $"Plugin worker exited before returning a result (exit {process.ExitCode}): {stderr}"); + } + if (line.Length > _options.MaximumResponseBytes * 2) + throw new InvalidDataException("Plugin worker protocol message is too large."); + var message = JsonSerializer.Deserialize( + line, + PluginWorkerJsonContext.Default.PluginWorkerMessage) + ?? throw new InvalidDataException("Plugin worker sent invalid protocol data."); + switch (message.Type) + { + case "capability": + await HandleCapabilityAsync(process, plugin, message, timeout.Token); + break; + case "result" when message.Result is not null: + return message.Result.Value.Clone(); + case "error": + throw new InvalidOperationException(message.Error ?? "Plugin execution failed."); + default: + throw new InvalidDataException($"Unexpected plugin worker message '{message.Type}'."); + } + } + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new TimeoutException( + $"Plugin invocation exceeded {_options.InvocationTimeoutMilliseconds} ms or a resource limit."); + } + finally + { + if (!process.HasExited) process.Kill(entireProcessTree: true); + timeout.Cancel(); + try { await monitor; } catch (OperationCanceledException) { } + } + } + + private async Task HandleCapabilityAsync( + Process process, + PluginCatalogEntry plugin, + PluginWorkerMessage message, + CancellationToken cancellationToken) + { + PluginWorkerMessage response; + try + { + if (message.Id is null || message.Capability is null || message.Payload is null) + throw new InvalidDataException("Capability request is incomplete."); + var result = await capabilityBroker.ExecuteAsync( + plugin, + message.Capability, + message.Payload.Value, + cancellationToken); + response = new PluginWorkerMessage + { + Type = "capability-result", + Id = message.Id, + Result = result + }; + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + response = new PluginWorkerMessage + { + Type = "capability-error", + Id = message.Id, + Error = exception.Message.Length <= 1_024 ? exception.Message : exception.Message[..1_024] + }; + } + + var json = JsonSerializer.Serialize(response, PluginWorkerJsonContext.Default.PluginWorkerMessage); + await process.StandardInput.WriteLineAsync(json.AsMemory(), cancellationToken); + await process.StandardInput.FlushAsync(cancellationToken); + } + + private async Task MonitorProcessAsync( + Process process, + TaskCompletionSource resourceViolation, + CancellationToken cancellationToken) + { + var maximumWorkingSet = (long)_options.MaximumWorkerMemoryMegabytes * 1024 * 1024; + var maximumCpu = TimeSpan.FromMilliseconds(_options.MaximumWorkerCpuMilliseconds); + while (!process.HasExited) + { + cancellationToken.ThrowIfCancellationRequested(); + process.Refresh(); + if (process.WorkingSet64 > maximumWorkingSet || process.TotalProcessorTime > maximumCpu) + { + resourceViolation.TrySetResult( + $"Plugin worker exceeded its CPU or {_options.MaximumWorkerMemoryMegabytes} MiB memory budget."); + process.Kill(entireProcessTree: true); + return; + } + await Task.Delay(25, cancellationToken); + } + } + + private static ProcessStartInfo CreateStartInfo() + { + var hostAssembly = typeof(PluginWorkerHost).Assembly; + var entryAssembly = Assembly.GetEntryAssembly(); + var processPath = entryAssembly == hostAssembly + ? Environment.ProcessPath + : null; + processPath ??= "dotnet"; + var startInfo = new ProcessStartInfo + { + FileName = processPath, + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true + }; + if (entryAssembly != hostAssembly || + string.Equals(Path.GetFileNameWithoutExtension(processPath), "dotnet", StringComparison.OrdinalIgnoreCase)) + startInfo.ArgumentList.Add(hostAssembly.Location); + startInfo.ArgumentList.Add(PluginWorkerHost.WorkerArgument); + return startInfo; + } + + private static async Task ReadErrorAsync(Process process) + { + var value = await process.StandardError.ReadToEndAsync(); + return value.Length <= 1_024 ? value : value[..1_024]; + } + + private static bool IsWithin(string candidate, string root) + { + var relative = Path.GetRelativePath(Path.GetFullPath(root), candidate); + return relative != ".." && !relative.StartsWith($"..{Path.DirectorySeparatorChar}", StringComparison.Ordinal) && + !Path.IsPathFullyQualified(relative); + } + + private static async Task ReadAndVerifyPackageAsync( + PluginCatalogEntry plugin, + CancellationToken cancellationToken) + { + var root = Path.GetFullPath(plugin.PackageDirectory); + if (!Directory.Exists(root) || new DirectoryInfo(root).LinkTarget is not null) + throw new InvalidDataException("Installed plugin package directory is missing or unsafe."); + var expected = plugin.Manifest.Integrity?.Files + ?? throw new InvalidDataException("Installed plugin integrity metadata is missing."); + var actualPaths = new HashSet(StringComparer.Ordinal); + byte[]? entryBytes = null; + var entryPath = plugin.Manifest.EntryPoint.Replace('\\', '/'); + + foreach (var path in EnumerateRegularFiles(root)) + { + cancellationToken.ThrowIfCancellationRequested(); + var relative = Path.GetRelativePath(root, path).Replace('\\', '/'); + if (relative.Equals("manifest.json", StringComparison.Ordinal)) continue; + actualPaths.Add(relative); + if (!expected.TryGetValue(relative, out var expectedDigest)) + throw new InvalidDataException($"Installed plugin contains unlisted file '{relative}'."); + var bytes = await File.ReadAllBytesAsync(path, cancellationToken); + var actualDigest = Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant(); + if (!actualDigest.Equals(expectedDigest, StringComparison.OrdinalIgnoreCase)) + throw new InvalidDataException($"Installed plugin file '{relative}' failed its integrity check."); + if (relative.Equals(entryPath, StringComparison.Ordinal)) entryBytes = bytes; + } + + var missing = expected.Keys.Except(actualPaths, StringComparer.Ordinal).Order().ToArray(); + if (missing.Length > 0) + throw new InvalidDataException($"Installed plugin files are missing: {string.Join(", ", missing)}."); + return entryBytes ?? throw new InvalidDataException("Installed plugin entry point is missing or unsafe."); + } + + private static IEnumerable EnumerateRegularFiles(string root) + { + var pending = new Stack(); + pending.Push(new DirectoryInfo(root)); + while (pending.TryPop(out var directory)) + { + foreach (var entry in directory.EnumerateFileSystemInfos()) + { + if (entry.LinkTarget is not null) + throw new InvalidDataException("Symbolic links are not allowed in installed plugin packages."); + if (entry is DirectoryInfo child) + { + pending.Push(child); + } + else if (entry is FileInfo) + { + yield return entry.FullName; + } + } + } + } +} diff --git a/SecondDimensionWatcherReDive/PluginPlatform/PluginProviderRegistry.cs b/SecondDimensionWatcherReDive/PluginPlatform/PluginProviderRegistry.cs new file mode 100644 index 0000000..2b79007 --- /dev/null +++ b/SecondDimensionWatcherReDive/PluginPlatform/PluginProviderRegistry.cs @@ -0,0 +1,116 @@ +using System.Runtime.CompilerServices; +using System.Text.Json; +using SecondDimensionWatcherReDive.Framework.FileStore; +using SecondDimensionWatcherReDive.Framework.Plugin; + +namespace SecondDimensionWatcherReDive.PluginPlatform; + +internal static class PluginProviderIdentity +{ + public static string Create(string pluginId, string providerName) + => $"plugin:{pluginId}:{providerName}"; +} + +public interface IPluginProviderRegistry +{ + IReadOnlyList GetFileStores(); + IReadOnlyList GetNotificationProviders(); +} + +internal sealed class PluginProviderRegistry(IPluginManager manager) : IPluginProviderRegistry +{ + public IReadOnlyList GetFileStores() + => manager.GetSnapshot() + .Where(IsAvailable) + .SelectMany(plugin => plugin.Manifest.Providers + .Where(provider => provider.Kind == "storage") + .Select(provider => (IFileStore)new JavaScriptFileStore( + plugin.Manifest.Id, + provider, + manager))) + .ToArray(); + + public IReadOnlyList GetNotificationProviders() + => manager.GetSnapshot() + .Where(IsAvailable) + .SelectMany(plugin => plugin.Manifest.Providers + .Where(provider => provider.Kind == "notification") + .Select(provider => (INotificationProvider)new JavaScriptNotificationProvider( + plugin.Manifest.Id, + provider, + manager))) + .ToArray(); + + private static bool IsAvailable(InstalledPlugin plugin) + => plugin.IsEnabled && plugin.CompatibilityErrors.Count == 0 && + (plugin.Health.CircuitOpenUntil is null || plugin.Health.CircuitOpenUntil <= DateTimeOffset.UtcNow); +} + +internal sealed class JavaScriptNotificationProvider( + string pluginId, + PluginProviderDeclaration declaration, + IPluginManager manager) : INotificationProvider +{ + public string Name => PluginProviderIdentity.Create(pluginId, declaration.Name); + + public async Task SendAsync(PluginNotification notification, CancellationToken cancellationToken) + { + if (!declaration.Handlers.TryGetValue("send", out var handler)) + throw new InvalidOperationException($"Notification provider '{Name}' has no send handler."); + var input = JsonSerializer.SerializeToElement(notification); + var result = await manager.InvokeAsync(pluginId, handler, input, cancellationToken); + if (result.ValueKind == JsonValueKind.Object && + result.TryGetProperty("success", out var success) && success.ValueKind == JsonValueKind.False) + throw new InvalidOperationException($"Notification provider '{Name}' rejected the notification."); + } +} + +internal sealed class JavaScriptFileStore( + string pluginId, + PluginProviderDeclaration declaration, + IPluginManager manager) : IFileStore +{ + private static readonly JsonSerializerOptions WebJsonOptions = new(JsonSerializerDefaults.Web); + public string Name => PluginProviderIdentity.Create(pluginId, declaration.Name); + + public async Task OpenReadStreamAsync(string path, CancellationToken cancellationToken) + { + var result = await InvokeAsync("read", path, cancellationToken); + if (!result.TryGetProperty("base64", out var base64) || base64.ValueKind != JsonValueKind.String) + throw new InvalidDataException("Storage provider read result must contain base64 data."); + return new MemoryStream(Convert.FromBase64String(base64.GetString()!), writable: false); + } + + public async Task FileInfoAsync(string path, CancellationToken cancellationToken) + { + var result = await InvokeAsync("info", path, cancellationToken); + return result.Deserialize(WebJsonOptions) + ?? throw new InvalidDataException("Storage provider returned invalid file information."); + } + + public async Task ExistAsync(string path, CancellationToken cancellationToken) + { + var result = await InvokeAsync("exists", path, cancellationToken); + return result.TryGetProperty("exists", out var exists) && exists.GetBoolean(); + } + + public IAsyncEnumerable EnumerateDirectory(string path) + => EnumerateDirectoryCore(path, CancellationToken.None); + + private async IAsyncEnumerable EnumerateDirectoryCore( + string path, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + var result = await InvokeAsync("list", path, cancellationToken); + var entries = result.Deserialize(WebJsonOptions) + ?? throw new InvalidDataException("Storage provider returned an invalid directory listing."); + foreach (var entry in entries) yield return entry; + } + + private Task InvokeAsync(string operation, string path, CancellationToken cancellationToken) + { + if (!declaration.Handlers.TryGetValue(operation, out var handler)) + throw new InvalidOperationException($"Storage provider '{Name}' has no {operation} handler."); + return manager.InvokeAsync(pluginId, handler, JsonSerializer.SerializeToElement(new { path }), cancellationToken); + } +} diff --git a/SecondDimensionWatcherReDive/PluginPlatform/PluginSafeFileAccess.cs b/SecondDimensionWatcherReDive/PluginPlatform/PluginSafeFileAccess.cs new file mode 100644 index 0000000..b9d80d2 --- /dev/null +++ b/SecondDimensionWatcherReDive/PluginPlatform/PluginSafeFileAccess.cs @@ -0,0 +1,438 @@ +using System.Runtime.InteropServices; +using System.Text; +using Microsoft.Win32.SafeHandles; + +namespace SecondDimensionWatcherReDive.PluginPlatform; + +internal sealed record PluginFileEntry( + string Name, + bool IsDirectory, + long? Length, + DateTimeOffset LastModifiedUtc); + +/// +/// Opens every POSIX path component relative to an already-open directory with O_NOFOLLOW. +/// This pins the object being accessed and closes rename/symlink races after lexical approval. +/// Windows reads validate the final path attached to the opened handle. +/// +internal sealed class PluginSafeFileAccess +{ + private const int ReadOnly = 0; + private const int WriteOnly = 1; + private const int LinuxCreate = 0x40; + private const int LinuxExclusive = 0x80; + private const int LinuxDirectory = 0x10000; + private const int LinuxNoFollow = 0x20000; + private const int LinuxCloseOnExec = 0x80000; + private const int LinuxNonBlocking = 0x800; + private const int MacCreate = 0x200; + private const int MacExclusive = 0x800; + private const int MacDirectory = 0x100000; + private const int MacNoFollow = 0x100; + private const int MacCloseOnExec = 0x1000000; + private const int MacNonBlocking = 0x4; + private const int MissingPathError = 2; + + internal Action? BeforeOpenForTesting { get; set; } + + public async Task ReadAsync( + string root, + string path, + int maximumBytes, + CancellationToken cancellationToken) + { + ValidateLexicalPath(root, path); + BeforeOpenForTesting?.Invoke(); + await using var stream = OpenRead(root, path); + using var memory = new MemoryStream(Math.Min(maximumBytes, 64 * 1024)); + var buffer = new byte[64 * 1024]; + int read; + while ((read = await stream.ReadAsync(buffer, cancellationToken)) > 0) + { + if (memory.Length + read > maximumBytes) + throw new InvalidDataException("Capability response exceeds the configured size limit."); + await memory.WriteAsync(buffer.AsMemory(0, read), cancellationToken); + } + return memory.ToArray(); + } + + public IReadOnlyList List(string root, string path, int maximumEntries) + { + ValidateLexicalPath(root, path); + BeforeOpenForTesting?.Invoke(); + if (IsPosix) + { + using var directory = OpenPosixAbsolute(path, directory: true, out _); + var descriptorPath = GetDescriptorPath(directory); + var result = new List(); + foreach (var item in Directory.EnumerateFileSystemEntries(descriptorPath)) + { + if (result.Count >= maximumEntries) + throw new InvalidDataException("Directory contains too many entries."); + var name = Path.GetFileName(item); + SafeFileHandle child; + try + { + child = OpenPosixAt(directory, name, directory: false, out _); + } + catch (UnauthorizedAccessException exception) when (exception.InnerException is FileNotFoundException) + { + continue; + } + using (child) + { + var childPath = GetDescriptorPath(child); + var isDirectory = Directory.Exists(childPath); + result.Add(new PluginFileEntry( + name, + isDirectory, + isDirectory ? null : RandomAccess.GetLength(child), + new DateTimeOffset(File.GetLastWriteTimeUtc(childPath), TimeSpan.Zero))); + } + } + return result; + } + + throw new PlatformNotSupportedException( + "Plugin directory capabilities are disabled on Windows until handle-relative enumeration is available."); + } + + public PluginFileEntry? Info(string root, string path) + { + ValidateLexicalPath(root, path); + BeforeOpenForTesting?.Invoke(); + if (IsPosix) + { + SafeFileHandle handle; + try + { + handle = OpenPosixAbsolute(path, directory: false, out _); + } + catch (UnauthorizedAccessException exception) when (exception.InnerException is FileNotFoundException) + { + return null; + } + using (handle) + { + var descriptorPath = GetDescriptorPath(handle); + var isDirectory = Directory.Exists(descriptorPath); + return new PluginFileEntry( + Path.GetFileName(path), + isDirectory, + isDirectory ? null : RandomAccess.GetLength(handle), + new DateTimeOffset(File.GetLastWriteTimeUtc(descriptorPath), TimeSpan.Zero)); + } + } + + throw new PlatformNotSupportedException( + "Plugin metadata capabilities are disabled on Windows until handle-relative inspection is available."); + } + + public bool Exists(string root, string path) => Info(root, path) is not null; + + public async Task WriteAsync( + string root, + string path, + ReadOnlyMemory content, + CancellationToken cancellationToken) + { + ValidateLexicalPath(root, path); + BeforeOpenForTesting?.Invoke(); + if (IsPosix) + { + await WritePosixAsync(root, path, content, cancellationToken); + return; + } + + throw new PlatformNotSupportedException( + "Plugin data writes are disabled on Windows until handle-relative creation is available."); + } + + private static Stream OpenRead(string root, string path) + { + if (IsPosix) + { + var handle = OpenPosixAbsolute(path, directory: false, out _); + try { return new FileStream(handle, FileAccess.Read); } + catch { handle.Dispose(); throw; } + } + + var windowsHandle = OpenWindowsPath(root, path, directory: false); + try { return new FileStream(windowsHandle, FileAccess.Read); } + catch { windowsHandle.Dispose(); throw; } + } + + private static async Task WritePosixAsync( + string root, + string path, + ReadOnlyMemory content, + CancellationToken cancellationToken) + { + using var rootHandle = OpenOrCreatePosixDirectoryAbsolute(root); + var relative = Path.GetRelativePath(root, path); + var segments = relative.Split(Path.DirectorySeparatorChar, StringSplitOptions.RemoveEmptyEntries); + if (segments.Length == 0) throw new UnauthorizedAccessException("A plugin data root cannot be overwritten."); + + SafeFileHandle current = DuplicateHandleReference(rootHandle); + try + { + foreach (var segment in segments[..^1]) + { + SafeFileHandle next; + try + { + next = OpenPosixAt(current, segment, directory: true, out _); + } + catch (UnauthorizedAccessException exception) when (exception.InnerException is FileNotFoundException) + { + if (PosixMkdirAt(current.DangerousGetHandle().ToInt32(), segment, Convert.ToUInt32("700", 8)) != 0 && + Marshal.GetLastPInvokeError() != 17) + ThrowPosixError(Path.Combine(root, relative)); + next = OpenPosixAt(current, segment, directory: true, out _); + } + current.Dispose(); + current = next; + } + + var temporaryName = $".sdw-{Guid.NewGuid():N}.tmp"; + var descriptor = PosixOpenAt( + current.DangerousGetHandle().ToInt32(), + temporaryName, + WriteOnly | CreateFlag | ExclusiveFlag | NoFollowFlag | CloseOnExecFlag, + Convert.ToUInt32("600", 8)); + if (descriptor < 0) ThrowPosixError(path); + try + { + await using (var stream = new FileStream( + new SafeFileHandle((nint)descriptor, ownsHandle: false), FileAccess.Write)) + { + await stream.WriteAsync(content, cancellationToken); + await stream.FlushAsync(cancellationToken); + } + if (PosixRenameAt( + current.DangerousGetHandle().ToInt32(), temporaryName, + current.DangerousGetHandle().ToInt32(), segments[^1]) != 0) + ThrowPosixError(path); + } + finally + { + _ = PosixUnlinkAt(current.DangerousGetHandle().ToInt32(), temporaryName, 0); + new SafeFileHandle((nint)descriptor, ownsHandle: true).Dispose(); + } + } + finally + { + current.Dispose(); + } + } + + private static SafeFileHandle DuplicateHandleReference(SafeFileHandle handle) + { + var duplicate = PosixOpenAt(handle.DangerousGetHandle().ToInt32(), ".", + ReadOnly | DirectoryFlag | NoFollowFlag | CloseOnExecFlag, 0); + if (duplicate < 0) ThrowPosixError("."); + return new SafeFileHandle((nint)duplicate, ownsHandle: true); + } + + private static SafeFileHandle OpenPosixAbsolute(string path, bool directory, out int error) + { + var normalized = Path.GetFullPath(path); + var root = Path.GetPathRoot(normalized)!; + var descriptor = PosixOpen(root, ReadOnly | DirectoryFlag | NoFollowFlag | CloseOnExecFlag, 0); + if (descriptor < 0) ThrowPosixError(path); + var current = new SafeFileHandle((nint)descriptor, ownsHandle: true); + try + { + var segments = Path.GetRelativePath(root, normalized) + .Split(Path.DirectorySeparatorChar, StringSplitOptions.RemoveEmptyEntries); + for (var index = 0; index < segments.Length; index++) + { + var next = OpenPosixAt(current, segments[index], directory && index == segments.Length - 1, + out error); + current.Dispose(); + current = next; + } + error = 0; + var result = current; + current = new SafeFileHandle(nint.Zero, ownsHandle: false); + return result; + } + finally + { + current.Dispose(); + } + } + + private static SafeFileHandle OpenOrCreatePosixDirectoryAbsolute(string path) + { + var normalized = Path.GetFullPath(path); + var root = Path.GetPathRoot(normalized)!; + var descriptor = PosixOpen(root, ReadOnly | DirectoryFlag | NoFollowFlag | CloseOnExecFlag, 0); + if (descriptor < 0) ThrowPosixError(path); + var current = new SafeFileHandle((nint)descriptor, ownsHandle: true); + try + { + foreach (var segment in Path.GetRelativePath(root, normalized) + .Split(Path.DirectorySeparatorChar, StringSplitOptions.RemoveEmptyEntries)) + { + SafeFileHandle next; + try + { + next = OpenPosixAt(current, segment, directory: true, out _); + } + catch (UnauthorizedAccessException exception) when (exception.InnerException is FileNotFoundException) + { + if (PosixMkdirAt(current.DangerousGetHandle().ToInt32(), segment, Convert.ToUInt32("700", 8)) != 0 && + Marshal.GetLastPInvokeError() != 17) + ThrowPosixError(path); + next = OpenPosixAt(current, segment, directory: true, out _); + } + current.Dispose(); + current = next; + } + var result = current; + current = new SafeFileHandle(nint.Zero, ownsHandle: false); + return result; + } + finally + { + current.Dispose(); + } + } + + private static SafeFileHandle OpenPosixAt( + SafeFileHandle parent, + string name, + bool directory, + out int error) + { + var flags = ReadOnly | NoFollowFlag | CloseOnExecFlag | NonBlockingFlag; + if (directory) flags |= DirectoryFlag; + var descriptor = PosixOpenAt(parent.DangerousGetHandle().ToInt32(), name, flags, 0); + if (descriptor < 0) + { + error = Marshal.GetLastPInvokeError(); + ThrowPosixError(name, error); + } + error = 0; + return new SafeFileHandle((nint)descriptor, ownsHandle: true); + } + + private static void ValidateLexicalPath(string root, string path) + { + root = Path.GetFullPath(root); + path = Path.GetFullPath(path); + var relative = Path.GetRelativePath(root, path); + if (relative == ".." || relative.StartsWith($"..{Path.DirectorySeparatorChar}", StringComparison.Ordinal) || + Path.IsPathFullyQualified(relative)) + throw new UnauthorizedAccessException("Plugin file path escapes its approved root."); + } + + private static SafeFileHandle OpenWindowsPath(string root, string path, bool directory) + { + if (!OperatingSystem.IsWindows()) throw new PlatformNotSupportedException(); + const uint genericRead = 0x80000000; + const uint shareAll = 0x00000007; + const uint openExisting = 3; + const uint backupSemantics = 0x02000000; + const uint openReparsePoint = 0x00200000; + var handle = WindowsCreateFile(path, genericRead, shareAll, 0, openExisting, + backupSemantics | openReparsePoint, 0); + if (handle.IsInvalid) + { + handle.Dispose(); + throw new UnauthorizedAccessException($"Plugin file path '{path}' could not be opened safely."); + } + try + { + RejectWindowsReparsePoint(path); + var finalPath = GetWindowsFinalPath(handle); + ValidateLexicalPath(root, finalPath); + if (!Path.GetFullPath(path).Equals(Path.GetFullPath(finalPath), StringComparison.OrdinalIgnoreCase)) + throw new UnauthorizedAccessException("Plugin file path changed while it was being opened."); + return handle; + } + catch + { + handle.Dispose(); + throw; + } + } + + private static void RejectWindowsReparsePoint(string path) + { + if ((File.GetAttributes(path) & FileAttributes.ReparsePoint) != 0) + throw new UnauthorizedAccessException("Reparse points are not allowed in plugin file paths."); + } + + private static string GetWindowsFinalPath(SafeFileHandle handle) + { + const int maximumPath = 32768; + var buffer = new StringBuilder(maximumPath); + var length = WindowsGetFinalPathNameByHandle(handle, buffer, maximumPath, 0); + if (length == 0 || length >= maximumPath) + throw new UnauthorizedAccessException("Could not validate the opened plugin file handle."); + var path = buffer.ToString(); + const string uncPrefix = @"\\?\UNC\"; + const string devicePrefix = @"\\?\"; + if (path.StartsWith(uncPrefix, StringComparison.OrdinalIgnoreCase)) + return @"\\" + path[uncPrefix.Length..]; + return path.StartsWith(devicePrefix, StringComparison.OrdinalIgnoreCase) + ? path[devicePrefix.Length..] + : path; + } + + private static string GetDescriptorPath(SafeFileHandle handle) + => OperatingSystem.IsLinux() + ? $"/proc/self/fd/{handle.DangerousGetHandle().ToInt32()}" + : $"/dev/fd/{handle.DangerousGetHandle().ToInt32()}"; + + private static void ThrowPosixError(string path, int? knownError = null) + { + var error = knownError ?? Marshal.GetLastPInvokeError(); + Exception? inner = error == MissingPathError ? new FileNotFoundException(path) : null; + throw new UnauthorizedAccessException( + $"Plugin file path '{path}' could not be opened without following links (errno {error}).", inner); + } + + private static bool IsPosix => OperatingSystem.IsLinux() || OperatingSystem.IsMacOS(); + private static int DirectoryFlag => OperatingSystem.IsMacOS() ? MacDirectory : LinuxDirectory; + private static int NoFollowFlag => OperatingSystem.IsMacOS() ? MacNoFollow : LinuxNoFollow; + private static int CloseOnExecFlag => OperatingSystem.IsMacOS() ? MacCloseOnExec : LinuxCloseOnExec; + private static int CreateFlag => OperatingSystem.IsMacOS() ? MacCreate : LinuxCreate; + private static int ExclusiveFlag => OperatingSystem.IsMacOS() ? MacExclusive : LinuxExclusive; + private static int NonBlockingFlag => OperatingSystem.IsMacOS() ? MacNonBlocking : LinuxNonBlocking; + + private static void RestrictFile(string path) + { + if (!OperatingSystem.IsWindows()) + File.SetUnixFileMode(path, UnixFileMode.UserRead | UnixFileMode.UserWrite); + } + + [DllImport("libc", EntryPoint = "open", SetLastError = true, CharSet = CharSet.Ansi)] + private static extern int PosixOpen(string path, int flags, uint mode); + + [DllImport("libc", EntryPoint = "openat", SetLastError = true, CharSet = CharSet.Ansi)] + private static extern int PosixOpenAt(int directoryDescriptor, string path, int flags, uint mode); + + [DllImport("libc", EntryPoint = "mkdirat", SetLastError = true, CharSet = CharSet.Ansi)] + private static extern int PosixMkdirAt(int directoryDescriptor, string path, uint mode); + + [DllImport("libc", EntryPoint = "renameat", SetLastError = true, CharSet = CharSet.Ansi)] + private static extern int PosixRenameAt(int oldDirectoryDescriptor, string oldPath, + int newDirectoryDescriptor, string newPath); + + [DllImport("libc", EntryPoint = "unlinkat", SetLastError = true, CharSet = CharSet.Ansi)] + private static extern int PosixUnlinkAt(int directoryDescriptor, string path, int flags); + + [DllImport("kernel32.dll", EntryPoint = "CreateFileW", SetLastError = true, + CharSet = CharSet.Unicode, ExactSpelling = true)] + private static extern SafeFileHandle WindowsCreateFile( + string fileName, uint desiredAccess, uint shareMode, nint securityAttributes, + uint creationDisposition, uint flagsAndAttributes, nint templateFile); + + [DllImport("kernel32.dll", EntryPoint = "GetFinalPathNameByHandleW", SetLastError = true, + CharSet = CharSet.Unicode, ExactSpelling = true)] + private static extern uint WindowsGetFinalPathNameByHandle( + SafeFileHandle file, StringBuilder path, int pathLength, uint flags); +} diff --git a/SecondDimensionWatcherReDive/PluginPlatform/PluginWorkerHost.cs b/SecondDimensionWatcherReDive/PluginPlatform/PluginWorkerHost.cs new file mode 100644 index 0000000..9934581 --- /dev/null +++ b/SecondDimensionWatcherReDive/PluginPlatform/PluginWorkerHost.cs @@ -0,0 +1,154 @@ +using System.Text.Json; +using Microsoft.ClearScript; +using Microsoft.ClearScript.V8; + +namespace SecondDimensionWatcherReDive.PluginPlatform; + +public interface IPluginWorkerBridge +{ + string Request(string capability, string payloadJson); +} + +internal static class PluginWorkerHost +{ + public const string WorkerArgument = "--plugin-worker"; + + public static bool IsWorkerInvocation(string[] args) + => args.Length == 1 && string.Equals(args[0], WorkerArgument, StringComparison.Ordinal); + + public static async Task RunAsync(CancellationToken cancellationToken) + { + try + { + var line = await Console.In.ReadLineAsync(cancellationToken); + if (string.IsNullOrWhiteSpace(line)) throw new InvalidDataException("Missing worker invocation."); + var invocation = JsonSerializer.Deserialize( + line, + PluginWorkerJsonContext.Default.PluginWorkerInvocation) + ?? throw new InvalidDataException("Invalid worker invocation."); + Execute(invocation); + return 0; + } + catch (Exception exception) + { + WriteMessage(new PluginWorkerMessage + { + Type = "error", + Error = SanitizeError(exception.Message) + }); + return 1; + } + } + + private static void Execute(PluginWorkerInvocation invocation) + { + var heapMiB = Math.Clamp(invocation.MaximumHeapMegabytes, 16, 512); + var constraints = new V8RuntimeConstraints + { + MaxOldSpaceSize = heapMiB, + MaxArrayBufferAllocation = checked((nuint)heapMiB * 1024 * 1024 / 2) + }; + using var runtime = new V8Runtime(constraints) + { + MaxHeapSize = checked((nuint)Math.Max(8, heapMiB - 8) * 1024 * 1024), + MaxStackUsage = 2 * 1024 * 1024, + EnableInterruptPropagation = true, + HeapSizeViolationPolicy = V8RuntimeViolationPolicy.Interrupt + }; + using var engine = runtime.CreateScriptEngine( + V8ScriptEngineFlags.DisableGlobalMembers | V8ScriptEngineFlags.HideHostExceptions); + var bridgeName = $"__sdwBridge_{Guid.NewGuid():N}"; + engine.AddRestrictedHostObject(bridgeName, new PluginWorkerBridge()); + engine.Execute("sdw-sdk.js", $$""" + 'use strict'; + globalThis.sdw = ((bridge) => { + return Object.freeze({ + request(capability, payload) { + const response = JSON.parse(bridge.Request(String(capability), JSON.stringify(payload ?? {}))); + if (!response.Ok) throw new Error(response.Error || 'Capability request was denied.'); + return response.Result; + } + }); + })(globalThis[{{JsonSerializer.Serialize(bridgeName)}}]); + delete globalThis[{{JsonSerializer.Serialize(bridgeName)}}]; + """); + engine.Execute("plugin.js", invocation.Script); + + var handlerJson = JsonSerializer.Serialize(invocation.Handler); + var inputJson = JsonSerializer.Serialize(invocation.Input.GetRawText()); + var configurationJson = JsonSerializer.Serialize(invocation.Configuration.GetRawText()); + var maximumResponseBytes = Math.Clamp(invocation.MaximumResponseBytes, 1024, 8 * 1024 * 1024); + var expression = $$""" + (() => { + if (!globalThis.sdwPlugin || typeof globalThis.sdwPlugin.handlers !== 'object') + throw new Error('Plugin must define globalThis.sdwPlugin.handlers.'); + const handler = globalThis.sdwPlugin.handlers[{{handlerJson}}]; + if (typeof handler !== 'function') + throw new Error('Plugin handler is not defined: ' + {{handlerJson}}); + const value = handler(JSON.parse({{inputJson}}), Object.freeze(JSON.parse({{configurationJson}}))); + if (value && typeof value.then === 'function') + throw new Error('Async JavaScript handlers are not supported; use synchronous sdw.request calls.'); + return JSON.stringify(value === undefined ? null : value); + })() + """; + var serialized = Convert.ToString(engine.Evaluate(expression), System.Globalization.CultureInfo.InvariantCulture) + ?? "null"; + if (System.Text.Encoding.UTF8.GetByteCount(serialized) > maximumResponseBytes) + throw new InvalidDataException("Plugin result exceeds the configured response limit."); + using var document = JsonDocument.Parse(serialized); + WriteMessage(new PluginWorkerMessage + { + Type = "result", + Result = document.RootElement.Clone() + }); + } + + private static void WriteMessage(PluginWorkerMessage message) + { + Console.Out.WriteLine(JsonSerializer.Serialize(message, PluginWorkerJsonContext.Default.PluginWorkerMessage)); + Console.Out.Flush(); + } + + private static string SanitizeError(string message) + => message.Length <= 1_024 ? message : message[..1_024]; + + private sealed class PluginWorkerBridge : IPluginWorkerBridge + { + public string Request(string capability, string payloadJson) + { + if (capability.Length > 64 || payloadJson.Length > 2 * 1024 * 1024) + throw new InvalidDataException("Capability request is too large."); + using var payloadDocument = JsonDocument.Parse(payloadJson); + var id = Guid.NewGuid().ToString("N"); + WriteMessage(new PluginWorkerMessage + { + Type = "capability", + Id = id, + Capability = capability, + Payload = payloadDocument.RootElement.Clone() + }); + + var responseLine = Console.In.ReadLine(); + if (string.IsNullOrWhiteSpace(responseLine)) throw new IOException("Capability broker disconnected."); + var response = JsonSerializer.Deserialize( + responseLine, + PluginWorkerJsonContext.Default.PluginWorkerMessage) + ?? throw new InvalidDataException("Invalid capability response."); + if (!string.Equals(response.Id, id, StringComparison.Ordinal)) + throw new InvalidDataException("Capability response id does not match request."); + if (response.Type == "capability-error") + return JsonSerializer.Serialize(new PluginWorkerBridgeResponse + { + Ok = false, + Error = response.Error ?? "Capability request was denied." + }, PluginWorkerJsonContext.Default.PluginWorkerBridgeResponse); + if (response.Type != "capability-result" || response.Result is null) + throw new InvalidDataException("Invalid capability response type."); + return JsonSerializer.Serialize(new PluginWorkerBridgeResponse + { + Ok = true, + Result = response.Result.Value.Clone() + }, PluginWorkerJsonContext.Default.PluginWorkerBridgeResponse); + } + } +} diff --git a/SecondDimensionWatcherReDive/PluginPlatform/PluginWorkerProtocol.cs b/SecondDimensionWatcherReDive/PluginPlatform/PluginWorkerProtocol.cs new file mode 100644 index 0000000..87da455 --- /dev/null +++ b/SecondDimensionWatcherReDive/PluginPlatform/PluginWorkerProtocol.cs @@ -0,0 +1,34 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace SecondDimensionWatcherReDive.PluginPlatform; + +internal sealed record PluginWorkerInvocation( + string Script, + string Handler, + JsonElement Input, + JsonElement Configuration, + int MaximumHeapMegabytes, + int MaximumResponseBytes); + +internal sealed record PluginWorkerMessage +{ + public required string Type { get; init; } + public string? Id { get; init; } + public string? Capability { get; init; } + public JsonElement? Payload { get; init; } + public JsonElement? Result { get; init; } + public string? Error { get; init; } +} + +internal sealed record PluginWorkerBridgeResponse +{ + public required bool Ok { get; init; } + public JsonElement? Result { get; init; } + public string? Error { get; init; } +} + +[JsonSerializable(typeof(PluginWorkerInvocation))] +[JsonSerializable(typeof(PluginWorkerMessage))] +[JsonSerializable(typeof(PluginWorkerBridgeResponse))] +internal partial class PluginWorkerJsonContext : JsonSerializerContext; diff --git a/SecondDimensionWatcherReDive/Program.cs b/SecondDimensionWatcherReDive/Program.cs index 80f5f19..b187dda 100644 --- a/SecondDimensionWatcherReDive/Program.cs +++ b/SecondDimensionWatcherReDive/Program.cs @@ -26,6 +26,7 @@ using SecondDimensionWatcherReDive.Repositories; using SecondDimensionWatcherReDive.Chat; using SecondDimensionWatcherReDive.Plugin; +using SecondDimensionWatcherReDive.PluginPlatform; using SecondDimensionWatcherReDive.Services; using SecondDimensionWatcherReDive.MigrationTasks; using SecondDimensionWatcherReDive.Utils.Feed; @@ -35,6 +36,12 @@ using SecondDimensionWatcherReDive.Utils.Incidents; using SecondDimensionWatcherReDive.Utils.Scraper; +if (PluginWorkerHost.IsWorkerInvocation(args)) +{ + Environment.ExitCode = await PluginWorkerHost.RunAsync(CancellationToken.None); + return; +} + var builder = WebApplication.CreateBuilder(args); builder.Host.UseSystemd(); @@ -84,6 +91,7 @@ .SetApplicationName("SecondDimensionWatcherReDive") .PersistKeysToFileSystem(new DirectoryInfo(dataProtectionKeyRingPath)); builder.Services.AddApplicationRuntimeSettings(runtimeSettingsProvider); +builder.Services.AddPluginPlatform(builder.Configuration); builder.Services.Configure( builder.Configuration.GetSection(MediaLibraryOptions.SectionName)); @@ -372,4 +380,6 @@ await app.Services.GetRequiredService() // half-migrated database. await app.Services.GetRequiredService().RunAsync(CancellationToken.None); +await app.Services.GetRequiredService().InitializeAsync(CancellationToken.None); + await app.RunAsync(); diff --git a/SecondDimensionWatcherReDive/Repositories/PluginCatalogRepository.cs b/SecondDimensionWatcherReDive/Repositories/PluginCatalogRepository.cs new file mode 100644 index 0000000..c1ae997 --- /dev/null +++ b/SecondDimensionWatcherReDive/Repositories/PluginCatalogRepository.cs @@ -0,0 +1,191 @@ +using System.Text.Json; +using Microsoft.Extensions.Options; +using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.PluginPlatform; + +namespace SecondDimensionWatcherReDive.Repositories; + +internal sealed class PluginCatalogRepository : IPluginCatalogRepository +{ + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web) + { + WriteIndented = true + }; + + private readonly SemaphoreSlim _gate = new(1, 1); + private readonly string _catalogPath; + private readonly string _retainedPath; + + public PluginCatalogRepository(IOptions options) + { + var root = Path.GetFullPath(options.Value.RootPath); + Directory.CreateDirectory(root); + RestrictDirectory(root); + _catalogPath = Path.Combine(root, "catalog"); + _retainedPath = Path.Combine(root, "retained"); + Directory.CreateDirectory(_catalogPath); + Directory.CreateDirectory(_retainedPath); + RestrictDirectory(_catalogPath); + RestrictDirectory(_retainedPath); + } + + public async Task> GetAllAsync(CancellationToken cancellationToken) + { + await _gate.WaitAsync(cancellationToken); + try + { + var result = new List(); + foreach (var path in Directory.EnumerateFiles(_catalogPath, "*.json", SearchOption.TopDirectoryOnly)) + { + await using var stream = File.OpenRead(path); + var entry = await JsonSerializer.DeserializeAsync(stream, JsonOptions, + cancellationToken); + if (entry is not null) result.Add(entry); + } + + return result; + } + finally + { + _gate.Release(); + } + } + + public async Task FindAsync(string id, CancellationToken cancellationToken) + { + var path = GetPath(id); + await _gate.WaitAsync(cancellationToken); + try + { + if (!File.Exists(path)) return null; + await using var stream = File.OpenRead(path); + return await JsonSerializer.DeserializeAsync(stream, JsonOptions, cancellationToken); + } + finally + { + _gate.Release(); + } + } + + public async Task SaveAsync(PluginCatalogEntry entry, CancellationToken cancellationToken) + { + var path = GetPath(entry.Manifest.Id); + var temporaryPath = Path.Combine(_catalogPath, $".{entry.Manifest.Id}.{Guid.NewGuid():N}.tmp"); + + await _gate.WaitAsync(cancellationToken); + try + { + await using (var stream = new FileStream( + temporaryPath, + FileMode.CreateNew, + FileAccess.Write, + FileShare.None, + 16 * 1024, + FileOptions.Asynchronous | FileOptions.WriteThrough)) + { + await JsonSerializer.SerializeAsync(stream, entry, JsonOptions, cancellationToken); + await stream.FlushAsync(cancellationToken); + } + + RestrictFile(temporaryPath); + File.Move(temporaryPath, path, overwrite: true); + } + finally + { + if (File.Exists(temporaryPath)) File.Delete(temporaryPath); + _gate.Release(); + } + } + + public async Task RemoveAsync(string id, CancellationToken cancellationToken) + { + var path = GetPath(id); + await _gate.WaitAsync(cancellationToken); + try + { + if (File.Exists(path)) File.Delete(path); + } + finally + { + _gate.Release(); + } + } + + public async Task FindRetainedAsync(string id, CancellationToken cancellationToken) + { + var path = GetRetainedPath(id); + await _gate.WaitAsync(cancellationToken); + try + { + if (!File.Exists(path)) return null; + await using var stream = File.OpenRead(path); + return await JsonSerializer.DeserializeAsync(stream, JsonOptions, cancellationToken); + } + finally + { + _gate.Release(); + } + } + + public async Task SaveRetainedAsync(RetainedPluginData retained, CancellationToken cancellationToken) + { + var path = GetRetainedPath(retained.Id); + var temporaryPath = Path.Combine(_retainedPath, $".{retained.Id}.{Guid.NewGuid():N}.tmp"); + await _gate.WaitAsync(cancellationToken); + try + { + await using (var stream = new FileStream(temporaryPath, FileMode.CreateNew, FileAccess.Write, + FileShare.None, 16 * 1024, FileOptions.Asynchronous | FileOptions.WriteThrough)) + { + await JsonSerializer.SerializeAsync(stream, retained, JsonOptions, cancellationToken); + await stream.FlushAsync(cancellationToken); + } + + RestrictFile(temporaryPath); + File.Move(temporaryPath, path, overwrite: true); + } + finally + { + if (File.Exists(temporaryPath)) File.Delete(temporaryPath); + _gate.Release(); + } + } + + public async Task RemoveRetainedAsync(string id, CancellationToken cancellationToken) + { + var path = GetRetainedPath(id); + await _gate.WaitAsync(cancellationToken); + try + { + if (File.Exists(path)) File.Delete(path); + } + finally + { + _gate.Release(); + } + } + + private string GetPath(string id) + { + if (!PluginManifestValidator.IsValidId(id)) throw new ArgumentException("Invalid plugin id.", nameof(id)); + return Path.Combine(_catalogPath, $"{id}.json"); + } + + private string GetRetainedPath(string id) + { + if (!PluginManifestValidator.IsValidId(id)) throw new ArgumentException("Invalid plugin id.", nameof(id)); + return Path.Combine(_retainedPath, $"{id}.json"); + } + + private static void RestrictDirectory(string path) + { + if (!OperatingSystem.IsWindows()) + File.SetUnixFileMode(path, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + } + + private static void RestrictFile(string path) + { + if (!OperatingSystem.IsWindows()) + File.SetUnixFileMode(path, UnixFileMode.UserRead | UnixFileMode.UserWrite); + } +} diff --git a/SecondDimensionWatcherReDive/Utils/FileStore/FileStoreProvider.cs b/SecondDimensionWatcherReDive/Utils/FileStore/FileStoreProvider.cs index 65e05bb..3ebe10b 100644 --- a/SecondDimensionWatcherReDive/Utils/FileStore/FileStoreProvider.cs +++ b/SecondDimensionWatcherReDive/Utils/FileStore/FileStoreProvider.cs @@ -1,16 +1,30 @@ using SecondDimensionWatcherReDive.Framework.FileStore; +using SecondDimensionWatcherReDive.PluginPlatform; namespace SecondDimensionWatcherReDive.Utils.FileStore; -public class FileStoreProvider(IServiceProvider serviceProvider) : IFileStoreProvider +public class FileStoreProvider( + IServiceProvider serviceProvider, + IPluginProviderRegistry pluginProviderRegistry) : IFileStoreProvider { public IFileStore GetRequiredClient(string clientName) { - return serviceProvider.GetServices().First(c => c.Name == clientName); + return GetClient(clientName) + ?? throw new InvalidOperationException($"File store '{clientName}' is not registered."); } public IFileStore? GetClient(string clientName) { - return serviceProvider.GetServices().FirstOrDefault(c => c.Name == clientName); + var matches = GetClients().Where(client => client.Name == clientName).Take(2).ToArray(); + return matches.Length switch + { + 0 => null, + 1 => matches[0], + _ => throw new InvalidOperationException( + $"File store identity '{clientName}' is registered more than once.") + }; } -} \ No newline at end of file + + private IEnumerable GetClients() + => serviceProvider.GetServices().Concat(pluginProviderRegistry.GetFileStores()); +} diff --git a/SecondDimensionWatcherReDive/appsettings.example.json b/SecondDimensionWatcherReDive/appsettings.example.json index da9d5ab..ffcf1b1 100644 --- a/SecondDimensionWatcherReDive/appsettings.example.json +++ b/SecondDimensionWatcherReDive/appsettings.example.json @@ -18,6 +18,30 @@ "FileStore": { "Local": "/var/lib/sdw-redive/downloads" }, + // Controlled JavaScript plugins are stored separately from media. Unsigned packages are + // rejected by default. Trusted publisher values are PEM-encoded RSA public keys. + "PluginPlatform": { + "RootPath": "/var/lib/sdw-redive/plugins", + "AllowUnsignedLocalPackages": false, + "MaximumPackageBytes": 4194304, + "MaximumExpandedBytes": 16777216, + "MaximumPackageFiles": 128, + "MaximumStagedPackages": 32, + "MaximumStagedPackageBytes": 67108864, + "InvocationTimeoutMilliseconds": 5000, + "MaximumWorkerMemoryMegabytes": 256, + "MaximumWorkerCpuMilliseconds": 4000, + "MaximumConcurrentWorkers": 4, + "MaximumConcurrentWorkersPerPlugin": 1, + "MaximumResponseBytes": 2097152, + "MaximumPluginDataBytes": 67108864, + "MaximumPluginDataFiles": 1000, + "MaximumPluginDataPathDepth": 8, + "CircuitBreakerFailures": 3, + "CircuitBreakerSeconds": 60, + "PreviewLifetimeMinutes": 30, + "TrustedPublisherPublicKeys": {} + }, // Existing media library imports are read in place. Mount container paths read-only // and configure them from Settings; these values control polling, copy settling, // and how long database history survives a temporarily missing source item. diff --git a/deployments/podman-compose.yml b/deployments/podman-compose.yml index 7a4f463..97d8fb8 100644 --- a/deployments/podman-compose.yml +++ b/deployments/podman-compose.yml @@ -57,6 +57,7 @@ services: MediaLibrary__MissingGracePeriod: "1.00:00:00" PasswordFile: "/app/data/password.json" DataProtection__KeyRingPath: "/app/data/data-protection-keys" + PluginPlatform__RootPath: "/app/data/plugins" Torrent__Remote__Url: "http://qbittorrent:8080" Torrent__Remote__UserName: "" Torrent__Remote__Password: "" diff --git a/docs/container-deployment.md b/docs/container-deployment.md index f14757e..7e737f9 100644 --- a/docs/container-deployment.md +++ b/docs/container-deployment.md @@ -19,7 +19,7 @@ - `downloads` — sdw-redive 和 qbittorrent **共享**，用于下载文件的读写 - `pgdata` — PostgreSQL 数据持久化 - `valkeydata` — Valkey 缓存数据持久化 -- `appdata` — 登录密码文件与运行时敏感配置的 Data Protection 密钥环 +- `appdata` — 登录密码文件、运行时敏感配置的 Data Protection 密钥环，以及插件 catalog、安装包、配置与隔离数据 ## 快速开始 @@ -115,6 +115,7 @@ podman logs qbittorrent 2>&1 | grep "temporary password" | `ConnectionStrings__sdw` | PostgreSQL 连接字符串 | 必填 | | `JwtSecret` | JWT 签名密钥（>=32 字符） | 必填 | | `DataProtection__KeyRingPath` | 网页保存密钥/密码所用的持久化加密密钥环 | `/app/data/data-protection-keys` | +| `PluginPlatform__RootPath` | 插件 catalog、包、配置与隔离数据的持久化目录 | `/app/data/plugins` | | `FileStore__Local` | 下载文件存储路径 | `/downloads` | | `MediaLibrary__ScanInterval` | 持续监控目录的轮询间隔 | `00:05:00` | | `MediaLibrary__SettlingPeriod` | 新文件写入完成后的稳定等待时间 | `00:00:30` | @@ -137,7 +138,7 @@ podman logs qbittorrent 2>&1 | grep "temporary password" ### 网页运行时设置 -首次登录后可在「设置」中修改 AI/TMDB、qBittorrent、媒体库扫描、异常阈值和 NFS。网页值保存在 PostgreSQL，优先于上表的环境变量；敏感值加密后存储且不会通过 API 回显。`appdata` 卷中的 Data Protection 密钥环必须保留，否则重启后的应用无法解密已保存的密钥。 +首次登录后可在「设置」中修改 AI/TMDB、qBittorrent、媒体库扫描、异常阈值和 NFS。网页值保存在 PostgreSQL，优先于上表的环境变量；敏感值加密后存储且不会通过 API 回显。`appdata` 卷中的 Data Protection 密钥环必须保留，否则重启后的应用无法解密已保存的密钥。插件平台也必须位于持久卷中；随附 Compose 将 `PluginPlatform__RootPath` 设为 `/app/data/plugins`，因此重建容器不会丢失已安装包、catalog、配置或插件隔离数据。 如果运行多个应用副本并让它们连接同一个 PostgreSQL 数据库，必须把 `DataProtection__KeyRingPath` 指向所有副本共享的同一持久化密钥环（且都使用内置 application name `SecondDimensionWatcherReDive`）。实例各自使用本地密钥环会导致其他副本无法解密数据库中的运行时密钥和密码。 diff --git a/docs/plugin-platform.md b/docs/plugin-platform.md new file mode 100644 index 0000000..474ee29 --- /dev/null +++ b/docs/plugin-platform.md @@ -0,0 +1,40 @@ +# Controlled plugin platform + +The plugin platform extends notification and file-storage providers without loading third-party code into the web process. API version 1.0 uses a fresh worker process and a constrained V8 runtime for every handler invocation. The host enforces wall-clock, CPU, working-set, V8 heap, array-buffer, response-size, and cancellation limits. A worker crash or V8 fatal resource violation therefore ends only that invocation. Three consecutive failures open a per-plugin circuit breaker; health and the last failure are visible through `GET /api/plugins` and Settings → Plugins. + +## Trust and installation + +Only authenticated application operators can manage plugins. Packages are uploaded locally and inspected before any code can run. Inspection rejects path traversal, symbolic links, duplicate or oversized archive content, invalid manifests, and any missing, unlisted, or hash-mismatched file. `integrity.files` must enumerate every regular archive file except `manifest.json`; that exact path/digest set is covered by the publisher signature and is checked again while extracting and before every invocation. Inspection reports compatibility, signature trust, the immutable package checksum, and every requested capability. Installation succeeds only when the caller echoes both the checksum and the exact capability set. A changed byte requires another preview and approval. + +Preview staging is bounded by both package count and aggregate bytes (`MaximumStagedPackages` and `MaximumStagedPackageBytes`); expired previews are removed before admitting another upload. The installation boundary validates the echoed checksum shape before reading the staged package. + +Plugin and dependency versions use bounded, strict SemVer (`major.minor.patch` with optional legal prerelease/build identifiers); path separators, control/non-ASCII characters, empty identifiers, and ambiguous numeric forms are rejected before extraction. API versions use the same bounded grammar while permitting the API's `major.minor` form. Extraction also canonicalizes the version and temporary destinations and requires both to remain under that plugin's package directory. Publisher, provider, provider-operation, and handler names are bounded ASCII identifiers; display text is length/control-character checked. + +Unsigned local packages are rejected by default. Configure trusted publisher PEM public keys under `PluginPlatform:TrustedPublisherPublicKeys`. `AllowUnsignedLocalPackages` exists for local development and compatibility tests only; it does not accept invalid or untrusted signatures. The trusted public-key fingerprint is persisted as plugin ownership, so upgrades and reinstalls that would inherit retained configuration/data must use the same key. The remote-install endpoint is intentionally hard-disabled: the service never downloads or evaluates arbitrary JavaScript from a URL. + +## Isolation and capabilities + +Workers receive only JSON input, read-only JSON configuration, and a restricted host interface with one `Request` method. They cannot obtain `IServiceProvider`, CLR types, host objects, `fetch`, `require`, or the host filesystem. The parent validates every request: + +- `network.request` accepts HTTP(S), disables redirects, cookies, and proxies, requires an exact or `*.` domain approval, resolves once, pins the connection to the validated result, and requires every DNS answer to be public. It rejects loopback, private, link-local, metadata, multicast, unspecified, transition/special-use ranges, and fail-closes IPv6 to ordinary global-unicast space. Private-network access is not available in API 1.0. Host or container egress ACLs remain the final boundary for organization-specific NAT64, 6rd, ISATAP, or other custom translation prefixes that cannot be identified from an address alone. +- `file.read` and `file.list` require an approved absolute root. Linux/macOS paths are traversed with directory-relative `openat` plus `O_NOFOLLOW`, closing path/symbolic-link replacement races. Windows file reads validate the opened handle; directory listing, metadata, and data writes fail closed until equivalent handle-relative operations are available. +- `data.*` requires `storageAccess` and remains inside `PluginPlatform:RootPath/data/`. +- notification publication, download control, and background tasks are represented in the manifest capability model but have no generic broker operation in API 1.0; unknown operations are denied. + +Network and file responses are bounded. Plugin-scoped data also has configurable aggregate byte, file-count, and path-depth quotas; writes reserve quota under a per-plugin gate and atomically replace the destination. Plugins do not receive arbitrary request headers, credentials, database access, or a service container. + +`MaximumConcurrentWorkers` and `MaximumConcurrentWorkersPerPlugin` bound aggregate worker amplification and reject excess work without degrading plugin health. `MaximumPluginDataBytes`, `MaximumPluginDataFiles`, and `MaximumPluginDataPathDepth` bound persistent storage. Lifecycle operations close the invocation gate, cancel and drain active workers, and only then move package or data directories. + +## Compatibility and lifecycle + +A plugin can be installed while incompatible so an administrator can inspect it, but it cannot be enabled. Enable checks the API version, OS/architecture, minimum dependency versions, and whether dependencies are enabled. Disabling or uninstalling a dependency automatically disables dependents. Installs and upgrades start disabled. + +Configuration and plugin-scoped data survive upgrades. If `dataVersion` changes, the new manifest must explicitly declare `dataMigration.strategy` as `reset`; the host moves old data to a rollback directory until the catalog update commits. Other version changes preserve data. Uninstall preserves configuration and data by default for a future reinstall; `DELETE /api/plugins/{id}?deleteData=true` is the explicit irreversible removal path. + +Reset upgrades and uninstalls use an fsync'd lifecycle journal stored separately from their payload directory. Startup rolls back every prepared operation, finalizes committed cleanup, and removes unreferenced package versions. Journal deletion is the last cleanup step, so repeated termination during a large payload deletion remains recoverable and idempotent. Cascaded dependency disables acquire the same per-plugin lifecycle lease as direct disables, canceling and draining active workers before the disabled state is committed. + +Each event handler is invoked with an independent timeout and exception boundary. A failure or timeout is contained and does not prevent later handlers from running; caller cancellation still propagates immediately. + +## Provider contract and compatibility suite + +Manifest `providers` entries declare a `kind`, display `name`, and operation-to-handler map. Handler names are validated before installation. Notification providers must request the `notifications` capability, and storage providers must request `storageAccess`, so the approval screen never understates the host data delivered to a provider. Runtime provider identities are stable global keys in the form `plugin::`; they cannot collide with built-in stores or silently rebind to another plugin. API 1.0 wires `notification` providers through `INotificationProvider` and `storage` providers through `IFileStore`; manifests declaring unimplemented download or metadata adapters are rejected rather than appearing healthy but unusable. The example webhook notification and plugin-scoped storage packages are installed, enabled, and exercised end-to-end by `PluginPlatformIntegrationTests`. Package validation, missing dependencies, incompatible APIs, capability denial, worker timeout/crash/aggregate-concurrency containment, circuit breaking, event isolation, upgrade rollback/reset, quota enforcement, and retained uninstall state are covered by the same test suite. diff --git a/examples/plugins/scoped-storage/index.js b/examples/plugins/scoped-storage/index.js new file mode 100644 index 0000000..19766de --- /dev/null +++ b/examples/plugins/scoped-storage/index.js @@ -0,0 +1,35 @@ +'use strict'; + +function relative(path) { + return String(path || '').replace(/^\/+/, ''); +} + +globalThis.sdwPlugin = { + handlers: { + exists(input) { + return sdw.request('data.exists', { path: relative(input.path) }); + }, + info(input) { + return sdw.request('data.info', { path: relative(input.path) }); + }, + read(input) { + return sdw.request('data.read', { path: relative(input.path) }); + }, + list(input) { + const base = relative(input.path); + return sdw.request('data.list', { path: base }).map((entry) => ({ + isDirectory: entry.isDirectory, + path: base ? `${base}/${entry.name}` : entry.name, + fileName: entry.name, + length: entry.length, + lastModifiedUtc: entry.lastModifiedUtc, + })); + }, + seed(input) { + return sdw.request('data.write', { + path: relative(input.path), + base64: input.base64, + }); + }, + }, +}; diff --git a/examples/plugins/scoped-storage/manifest.json b/examples/plugins/scoped-storage/manifest.json new file mode 100644 index 0000000..dfa0b3b --- /dev/null +++ b/examples/plugins/scoped-storage/manifest.json @@ -0,0 +1,36 @@ +{ + "id": "example.scoped-storage", + "name": "Plugin-scoped storage", + "description": "A read-only FileStore provider backed only by this plugin's isolated data directory.", + "version": "1.0.0", + "apiVersion": "1.0", + "entryPoint": "index.js", + "dependencies": [], + "capabilities": { + "networkDomains": [], + "fileRoots": [], + "notifications": false, + "downloadControl": false, + "storageAccess": true, + "backgroundTasks": false + }, + "platforms": ["any"], + "integrity": { + "files": { + "index.js": "8daa9c1860964f8e33684744d827d720d1da42415755f0affcac5cce62457a9e" + } + }, + "providers": [ + { + "kind": "storage", + "name": "example-scoped", + "handlers": { + "exists": "exists", + "info": "info", + "read": "read", + "list": "list" + } + } + ], + "dataVersion": 1 +} diff --git a/examples/plugins/webhook/index.js b/examples/plugins/webhook/index.js new file mode 100644 index 0000000..1789c30 --- /dev/null +++ b/examples/plugins/webhook/index.js @@ -0,0 +1,16 @@ +'use strict'; + +globalThis.sdwPlugin = { + handlers: { + sendNotification(notification, configuration) { + if (!configuration.url) throw new Error('Webhook configuration requires url.'); + const response = sdw.request('network.request', { + method: 'POST', + url: configuration.url, + contentType: 'application/json', + body: JSON.stringify(notification), + }); + return { success: response.status >= 200 && response.status < 300 }; + }, + }, +}; diff --git a/examples/plugins/webhook/manifest.json b/examples/plugins/webhook/manifest.json new file mode 100644 index 0000000..7589d4c --- /dev/null +++ b/examples/plugins/webhook/manifest.json @@ -0,0 +1,31 @@ +{ + "id": "example.webhook", + "name": "Webhook notifications", + "description": "Sends SDW notifications to one explicitly approved webhook domain.", + "version": "1.0.0", + "apiVersion": "1.0", + "entryPoint": "index.js", + "dependencies": [], + "capabilities": { + "networkDomains": ["hooks.example.com"], + "fileRoots": [], + "notifications": true, + "downloadControl": false, + "storageAccess": false, + "backgroundTasks": false + }, + "platforms": ["any"], + "integrity": { + "files": { + "index.js": "cb04e27dacbadf8de122b491b2c2d32cb553564fcc9c390a3ab4d922a2cd0e1b" + } + }, + "providers": [ + { + "kind": "notification", + "name": "webhook", + "handlers": { "send": "sendNotification" } + } + ], + "dataVersion": 1 +} diff --git a/packaging/appsettings.yml b/packaging/appsettings.yml index 4cf660f..4cba653 100644 --- a/packaging/appsettings.yml +++ b/packaging/appsettings.yml @@ -20,6 +20,29 @@ Torrent: FileStore: Local: /var/lib/sdw-redive/downloads +# 受控 JavaScript 插件。生产环境默认拒绝未签名包；公钥值为 PEM 编码 RSA 公钥。 +PluginPlatform: + RootPath: /var/lib/sdw-redive/plugins + AllowUnsignedLocalPackages: false + MaximumPackageBytes: 4194304 + MaximumExpandedBytes: 16777216 + MaximumPackageFiles: 128 + MaximumStagedPackages: 32 + MaximumStagedPackageBytes: 67108864 + InvocationTimeoutMilliseconds: 5000 + MaximumWorkerCpuMilliseconds: 4000 + MaximumWorkerMemoryMegabytes: 256 + MaximumConcurrentWorkers: 4 + MaximumConcurrentWorkersPerPlugin: 1 + MaximumResponseBytes: 2097152 + MaximumPluginDataBytes: 67108864 + MaximumPluginDataFiles: 1000 + MaximumPluginDataPathDepth: 8 + CircuitBreakerFailures: 3 + CircuitBreakerSeconds: 60 + PreviewLifetimeMinutes: 30 + TrustedPublisherPublicKeys: {} + # 现有媒体库扫描（目录在网页「设置」中添加，不会移动或删除原文件） # 缺失条目先撤下虚拟映射，超过 MissingGracePeriod 后才清理数据库记录 MediaLibrary: diff --git a/sdk/javascript/README.md b/sdk/javascript/README.md new file mode 100644 index 0000000..566edee --- /dev/null +++ b/sdk/javascript/README.md @@ -0,0 +1,13 @@ +# SDW JavaScript plugin SDK 1.0 + +A plugin package is a ZIP archive (normally named `.sdwpkg`) containing `manifest.json` at its root and one JavaScript entry point. The entry point assigns synchronous handlers to `globalThis.sdwPlugin.handlers`. There is no `require`, `fetch`, filesystem API, .NET reflection, or service container. Privileged work goes through `sdw.request`, whose broker checks the installed manifest's approved capabilities on every call. + +Use `plugin-api.d.ts` while authoring. Compute SHA-256 after the final edit of every regular archive file except `manifest.json`, and place the exact path-to-digest map in `integrity.files`. Unlisted and missing files are rejected. Production installations require an RSA-SHA256 signature from a publisher configured in `PluginPlatform:TrustedPublisherPublicKeys`. Generate the bytes with the public .NET helper `PluginSignaturePayload.Create(manifest)`, then sign them with RSA PKCS#1 v1.5 and SHA-256. The versioned, base64-tagged canonical payload covers identity, description, API and plugin versions, the complete file list and digests, every capability, platform, dependency, provider/handler declaration, and data-migration field; changing any execution-relevant manifest value or package file invalidates the signature. + +Use strict `major.minor.patch` SemVer for `version` and dependency minimum versions. Optional prerelease/build identifiers follow the SemVer ASCII grammar; path separators, empty identifiers, leading-zero numeric identifiers, controls, and non-ASCII forms are rejected. Publisher, provider, provider-operation, and handler names are 1-64 character ASCII identifiers beginning with a letter. + +Publisher ownership is continuous across upgrades and retained-data reinstalls. The host persists the trusted public-key fingerprint, not merely the publisher label, and rejects a package signed by a different key. Transferring ownership requires first uninstalling with retained data deletion. + +The local upload API is deliberately two-stage: `POST /api/plugins/preview` returns the checksum, signature status, compatibility result, and required capabilities; `POST /api/plugins/install` must echo that checksum and the exact capability object. New installs and upgrades remain disabled until explicitly enabled. URL-based remote installation always returns 403. + +Compatibility rules and lifecycle guarantees are documented in [plugin-platform.md](../../docs/plugin-platform.md). The examples under `examples/plugins` are executable fixtures for the compatibility suite. They are intentionally unsigned; production operators should copy and sign them under their own trusted publisher key. diff --git a/sdk/javascript/plugin-api.d.ts b/sdk/javascript/plugin-api.d.ts new file mode 100644 index 0000000..0d4c52b --- /dev/null +++ b/sdk/javascript/plugin-api.d.ts @@ -0,0 +1,39 @@ +export interface SdwPluginHost { + request(capability: "network.request", payload: { + method: "GET" | "POST" | "PUT" | "PATCH" | "DELETE"; + url: string; + body?: string; + contentType?: string; + }): { status: number; contentType?: string; body: string }; + request(capability: "file.read", payload: { path: string }): { base64: string }; + request(capability: "file.list", payload: { path: string }): Array<{ name: string; isDirectory: boolean }>; + request(capability: "data.read", payload: { path: string }): { base64: string }; + request(capability: "data.write", payload: { path: string; base64: string }): { written: number }; + request(capability: "data.exists", payload: { path: string }): { exists: boolean; isDirectory: boolean }; + request(capability: "data.info", payload: { path: string }): PluginDataInfo; + request(capability: "data.list", payload: { path: string }): PluginDataEntry[]; +} + +export interface PluginDataEntry { + name: string; + isDirectory: boolean; + length?: number; + lastModifiedUtc?: string; +} + +export interface PluginDataInfo { + path: string; + fileName: string; + isDirectory: boolean; + length?: number; + lastModifiedUtc?: string; +} + +export interface SdwPlugin { + handlers: Record>) => unknown>; +} + +declare global { + const sdw: Readonly; + var sdwPlugin: SdwPlugin; +} From 876afca0731791f655a2a79837c44cf6c95aead6 Mon Sep 17 00:00:00 2001 From: mahoshojoHCG Date: Sun, 30 Aug 2026 01:32:14 +0800 Subject: [PATCH 07/45] fix: isolate plugin worker runtime environment --- .../PluginPlatformIntegrationTests.cs | 28 +++++++++++++++++++ .../PluginPlatform/PluginProcessExecutor.cs | 24 ++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/SecondDimensionWatcherReDive.Test/PluginPlatformIntegrationTests.cs b/SecondDimensionWatcherReDive.Test/PluginPlatformIntegrationTests.cs index 029b682..8bf8ff8 100644 --- a/SecondDimensionWatcherReDive.Test/PluginPlatformIntegrationTests.cs +++ b/SecondDimensionWatcherReDive.Test/PluginPlatformIntegrationTests.cs @@ -447,6 +447,34 @@ await Assert.ThrowsExactlyAsync(() => Assert.IsFalse(File.Exists(Path.Combine(dataRoot, "extra.bin"))); } + [TestMethod] + public void WorkerEnvironment_RemovesProfilerAndStartupHookInjection() + { + var startInfo = new ProcessStartInfo(); + string[] injectedVariables = + [ + "CORECLR_ENABLE_PROFILING", + "CORECLR_PROFILER", + "CORECLR_PROFILER_PATH", + "CORECLR_PROFILER_PATH_32", + "CORECLR_PROFILER_PATH_64", + "COR_ENABLE_PROFILING", + "COR_PROFILER", + "COR_PROFILER_PATH", + "COR_PROFILER_PATH_32", + "COR_PROFILER_PATH_64", + "DOTNET_STARTUP_HOOKS", + "DOTNET_ADDITIONAL_DEPS", + "DOTNET_SHARED_STORE" + ]; + foreach (var variable in injectedVariables) startInfo.Environment[variable] = "injected"; + + PluginProcessExecutor.RemoveRuntimeInjectionEnvironmentVariables(startInfo); + + foreach (var variable in injectedVariables) + Assert.IsFalse(startInfo.Environment.ContainsKey(variable), $"{variable} must not reach the worker."); + } + [TestMethod] public async Task Worker_ContainsTimeoutCrashAndResourceExhaustion_ThenOpensCircuit() { diff --git a/SecondDimensionWatcherReDive/PluginPlatform/PluginProcessExecutor.cs b/SecondDimensionWatcherReDive/PluginPlatform/PluginProcessExecutor.cs index 578c0d0..900c6bd 100644 --- a/SecondDimensionWatcherReDive/PluginPlatform/PluginProcessExecutor.cs +++ b/SecondDimensionWatcherReDive/PluginPlatform/PluginProcessExecutor.cs @@ -23,6 +23,23 @@ internal sealed class PluginProcessExecutor( IPluginCapabilityBroker capabilityBroker, IOptions options) : IPluginProcessExecutor { + private static readonly string[] RuntimeInjectionEnvironmentVariables = + [ + "CORECLR_ENABLE_PROFILING", + "CORECLR_PROFILER", + "CORECLR_PROFILER_PATH", + "CORECLR_PROFILER_PATH_32", + "CORECLR_PROFILER_PATH_64", + "COR_ENABLE_PROFILING", + "COR_PROFILER", + "COR_PROFILER_PATH", + "COR_PROFILER_PATH_32", + "COR_PROFILER_PATH_64", + "DOTNET_STARTUP_HOOKS", + "DOTNET_ADDITIONAL_DEPS", + "DOTNET_SHARED_STORE" + ]; + private readonly PluginPlatformOptions _options = options.Value; private readonly SemaphoreSlim _globalGate = new( options.Value.MaximumConcurrentWorkers, @@ -217,9 +234,16 @@ private static ProcessStartInfo CreateStartInfo() string.Equals(Path.GetFileNameWithoutExtension(processPath), "dotnet", StringComparison.OrdinalIgnoreCase)) startInfo.ArgumentList.Add(hostAssembly.Location); startInfo.ArgumentList.Add(PluginWorkerHost.WorkerArgument); + RemoveRuntimeInjectionEnvironmentVariables(startInfo); return startInfo; } + internal static void RemoveRuntimeInjectionEnvironmentVariables(ProcessStartInfo startInfo) + { + foreach (var variable in RuntimeInjectionEnvironmentVariables) + startInfo.Environment.Remove(variable); + } + private static async Task ReadErrorAsync(Process process) { var value = await process.StandardError.ReadToEndAsync(); From 2338e84c11789e4cfa6f98b7e63f9fcd5643f94d Mon Sep 17 00:00:00 2001 From: mahoshojoHCG Date: Sun, 30 Aug 2026 01:40:25 +0800 Subject: [PATCH 08/45] fix: isolate plugin process tests from coverage --- .github/workflows/build.yml | 12 ++- .../PluginPlatformIntegrationTests.cs | 101 ++++++++++-------- .../PluginPlatform/PluginProcessExecutor.cs | 32 ++---- 3 files changed, 80 insertions(+), 65 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 98cb3e1..139ad00 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,9 +26,19 @@ jobs: with: dotnet-version: '10.0.x' - - name: Run tests + # These tests intentionally start and kill child processes that load the application assembly. + # Coverlet statically instruments that assembly with one shared hits file, so collecting the + # class in-process can corrupt coverage when a hostile worker is terminated by design. + - name: Run plugin platform process tests without coverage + run: >- + dotnet test SecondDimensionWatcherReDive.Test/SecondDimensionWatcherReDive.Test.csproj -c Release + --filter "FullyQualifiedName~SecondDimensionWatcherReDive.Test.PluginPlatformIntegrationTests" + --logger "trx;LogFileName=plugin-platform-test-results.trx" + + - name: Run remaining tests with coverage run: >- dotnet test SecondDimensionWatcherReDive.slnx -c Release + --filter "FullyQualifiedName!~SecondDimensionWatcherReDive.Test.PluginPlatformIntegrationTests" --logger "trx;LogFileName=test-results.trx" --collect "XPlat Code Coverage" -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=cobertura diff --git a/SecondDimensionWatcherReDive.Test/PluginPlatformIntegrationTests.cs b/SecondDimensionWatcherReDive.Test/PluginPlatformIntegrationTests.cs index 8bf8ff8..d981d5e 100644 --- a/SecondDimensionWatcherReDive.Test/PluginPlatformIntegrationTests.cs +++ b/SecondDimensionWatcherReDive.Test/PluginPlatformIntegrationTests.cs @@ -448,61 +448,53 @@ await Assert.ThrowsExactlyAsync(() => } [TestMethod] - public void WorkerEnvironment_RemovesProfilerAndStartupHookInjection() + public async Task Worker_ReportsScriptErrorsThroughProtocol_AndLeavesHealthyPluginAvailable() { - var startInfo = new ProcessStartInfo(); - string[] injectedVariables = - [ - "CORECLR_ENABLE_PROFILING", - "CORECLR_PROFILER", - "CORECLR_PROFILER_PATH", - "CORECLR_PROFILER_PATH_32", - "CORECLR_PROFILER_PATH_64", - "COR_ENABLE_PROFILING", - "COR_PROFILER", - "COR_PROFILER_PATH", - "COR_PROFILER_PATH_32", - "COR_PROFILER_PATH_64", - "DOTNET_STARTUP_HOOKS", - "DOTNET_ADDITIONAL_DEPS", - "DOTNET_SHARED_STORE" - ]; - foreach (var variable in injectedVariables) startInfo.Environment[variable] = "injected"; - - PluginProcessExecutor.RemoveRuntimeInjectionEnvironmentVariables(startInfo); - - foreach (var variable in injectedVariables) - Assert.IsFalse(startInfo.Environment.ContainsKey(variable), $"{variable} must not reach the worker."); + const string crashingScript = """ + 'use strict'; + globalThis.sdwPlugin = { handlers: { + crash() { throw new Error('intentional crash'); } + }}; + """; + await using var fixture = new PluginPlatformFixture(); + var crashing = Manifest("test.crash-protocol"); + await fixture.InstallAndEnableAsync(crashing, crashingScript); + var healthy = Manifest("test.crash-healthy"); + await fixture.InstallAndEnableAsync(healthy, PingScript); + + var failure = await Assert.ThrowsExactlyAsync(() => + fixture.InvokeAsync(crashing.Id, "crash")); + StringAssert.Contains(failure.Message, "intentional crash"); + + var result = await fixture.InvokeAsync(healthy.Id, "ping", new { value = 7 }); + Assert.AreEqual(7, result.GetProperty("value").GetInt32()); } [TestMethod] - public async Task Worker_ContainsTimeoutCrashAndResourceExhaustion_ThenOpensCircuit() + public async Task WorkerCapacity_RejectionsDoNotDegradePluginHealth() { - const string hostileScript = """ + const string timeoutScript = """ 'use strict'; globalThis.sdwPlugin = { handlers: { - timeout() { while (true) {} }, - crash() { throw new Error('intentional crash'); }, - memory() { const values = []; while (true) values.push(new ArrayBuffer(1048576)); } + timeout() { while (true) {} } }}; """; await using var fixture = new PluginPlatformFixture(options => { - options.InvocationTimeoutMilliseconds = 400; - options.MaximumWorkerCpuMilliseconds = 300; - options.MaximumWorkerMemoryMegabytes = 64; + options.InvocationTimeoutMilliseconds = 2_000; + options.MaximumWorkerCpuMilliseconds = 1_500; + options.MaximumWorkerMemoryMegabytes = 256; options.MaximumConcurrentWorkers = 1; options.MaximumConcurrentWorkersPerPlugin = 1; options.CircuitBreakerFailures = 3; }); - var hostile = Manifest("test.hostile"); - await fixture.InstallAndEnableAsync(hostile, hostileScript); - var healthy = Manifest("test.healthy"); + var hostile = Manifest("test.capacity-hostile"); + await fixture.InstallAndEnableAsync(hostile, timeoutScript); + var healthy = Manifest("test.capacity-healthy"); await fixture.InstallAndEnableAsync(healthy, PingScript); - var stopwatch = Stopwatch.StartNew(); var firstTimeout = fixture.InvokeAsync(hostile.Id, "timeout"); - await Task.Delay(100); + await Task.Delay(25); var rejected = await Task.WhenAll(Enumerable.Range(0, 20).Select(async _ => { try @@ -527,16 +519,39 @@ await Assert.ThrowsExactlyAsync(() => .Single(plugin => plugin.Manifest.Id == healthy.Id).Health; Assert.AreEqual(0, healthyHealth.ConsecutiveFailures, "A global capacity rejection must not be attributed to another plugin."); - await Assert.ThrowsExactlyAsync(() => fixture.InvokeAsync(hostile.Id, "crash")); - await Assert.ThrowsAsync(() => fixture.InvokeAsync(hostile.Id, "memory")); + } + + [TestMethod] + public async Task Worker_ContainsTimeoutAndResourceExhaustion_ThenOpensCircuit() + { + const string hostileScript = """ + 'use strict'; + globalThis.sdwPlugin = { handlers: { + timeout() { while (true) {} }, + memory() { const values = []; while (true) values.push(new ArrayBuffer(1048576)); } + }}; + """; + await using var fixture = new PluginPlatformFixture(options => + { + options.InvocationTimeoutMilliseconds = 400; + options.MaximumWorkerCpuMilliseconds = 300; + options.MaximumWorkerMemoryMegabytes = 64; + options.CircuitBreakerFailures = 3; + }); + var hostile = Manifest("test.resource-hostile"); + await fixture.InstallAndEnableAsync(hostile, hostileScript); + var stopwatch = Stopwatch.StartNew(); + + await Assert.ThrowsExactlyAsync(() => fixture.InvokeAsync(hostile.Id, "timeout")); + await Assert.ThrowsExactlyAsync(() => fixture.InvokeAsync(hostile.Id, "timeout")); + var memoryFailure = await Assert.ThrowsAsync(() => fixture.InvokeAsync(hostile.Id, "memory")); + Assert.IsTrue(memoryFailure is TimeoutException or InvalidOperationException, + $"Unexpected resource failure type: {memoryFailure.GetType().Name}"); Assert.IsLessThan(TimeSpan.FromSeconds(8), stopwatch.Elapsed); var circuit = await Assert.ThrowsExactlyAsync(() => - fixture.InvokeAsync(hostile.Id, "crash")); + fixture.InvokeAsync(hostile.Id, "timeout")); StringAssert.Contains(circuit.Message, "circuit is open"); - - var result = await fixture.InvokeAsync(healthy.Id, "ping", new { value = 7 }); - Assert.AreEqual(7, result.GetProperty("value").GetInt32()); } [TestMethod] diff --git a/SecondDimensionWatcherReDive/PluginPlatform/PluginProcessExecutor.cs b/SecondDimensionWatcherReDive/PluginPlatform/PluginProcessExecutor.cs index 900c6bd..d9d3a35 100644 --- a/SecondDimensionWatcherReDive/PluginPlatform/PluginProcessExecutor.cs +++ b/SecondDimensionWatcherReDive/PluginPlatform/PluginProcessExecutor.cs @@ -23,23 +23,6 @@ internal sealed class PluginProcessExecutor( IPluginCapabilityBroker capabilityBroker, IOptions options) : IPluginProcessExecutor { - private static readonly string[] RuntimeInjectionEnvironmentVariables = - [ - "CORECLR_ENABLE_PROFILING", - "CORECLR_PROFILER", - "CORECLR_PROFILER_PATH", - "CORECLR_PROFILER_PATH_32", - "CORECLR_PROFILER_PATH_64", - "COR_ENABLE_PROFILING", - "COR_PROFILER", - "COR_PROFILER_PATH", - "COR_PROFILER_PATH_32", - "COR_PROFILER_PATH_64", - "DOTNET_STARTUP_HOOKS", - "DOTNET_ADDITIONAL_DEPS", - "DOTNET_SHARED_STORE" - ]; - private readonly PluginPlatformOptions _options = options.Value; private readonly SemaphoreSlim _globalGate = new( options.Value.MaximumConcurrentWorkers, @@ -132,8 +115,10 @@ private async Task InvokeCoreAsync( await HandleCapabilityAsync(process, plugin, message, timeout.Token); break; case "result" when message.Result is not null: + await WaitForWorkerExitAfterTerminalMessageAsync(process, timeout, monitor); return message.Result.Value.Clone(); case "error": + await WaitForWorkerExitAfterTerminalMessageAsync(process, timeout, monitor); throw new InvalidOperationException(message.Error ?? "Plugin execution failed."); default: throw new InvalidDataException($"Unexpected plugin worker message '{message.Type}'."); @@ -234,14 +219,19 @@ private static ProcessStartInfo CreateStartInfo() string.Equals(Path.GetFileNameWithoutExtension(processPath), "dotnet", StringComparison.OrdinalIgnoreCase)) startInfo.ArgumentList.Add(hostAssembly.Location); startInfo.ArgumentList.Add(PluginWorkerHost.WorkerArgument); - RemoveRuntimeInjectionEnvironmentVariables(startInfo); return startInfo; } - internal static void RemoveRuntimeInjectionEnvironmentVariables(ProcessStartInfo startInfo) + private static async Task WaitForWorkerExitAfterTerminalMessageAsync( + Process process, + CancellationTokenSource timeout, + Task monitor) { - foreach (var variable in RuntimeInjectionEnvironmentVariables) - startInfo.Environment.Remove(variable); + timeout.Cancel(); + try { await monitor; } catch (OperationCanceledException) { } + + using var exitGrace = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + try { await process.WaitForExitAsync(exitGrace.Token); } catch (OperationCanceledException) { } } private static async Task ReadErrorAsync(Process process) From cd08fc75adeefbecf5eb7d158d2e4fbe5477a49c Mon Sep 17 00:00:00 2001 From: mahoshojoHCG Date: Sun, 30 Aug 2026 10:44:30 +0800 Subject: [PATCH 09/45] fix: honor plugin storage and upload limits --- .../Plugins/PluginApiTests.cs | 30 +++++++++- .../WebDavWebApplicationFactory.cs | 1 + .../PluginControllerTests.cs | 59 +++++++++++++++++++ .../Controllers/PluginController.cs | 3 +- .../PluginPlatform/PluginPlatformOptions.cs | 7 ++- .../PluginPlatformServiceExtensions.cs | 13 +++- SecondDimensionWatcherReDive/Program.cs | 4 +- docs/plugin-platform.md | 2 + 8 files changed, 112 insertions(+), 7 deletions(-) diff --git a/SecondDimensionWatcherReDive.IntegrationTest/Plugins/PluginApiTests.cs b/SecondDimensionWatcherReDive.IntegrationTest/Plugins/PluginApiTests.cs index 04449a9..c5a219c 100644 --- a/SecondDimensionWatcherReDive.IntegrationTest/Plugins/PluginApiTests.cs +++ b/SecondDimensionWatcherReDive.IntegrationTest/Plugins/PluginApiTests.cs @@ -10,6 +10,23 @@ namespace SecondDimensionWatcherReDive.IntegrationTest.Plugins; [TestClass] public sealed class PluginApiTests { + [TestMethod] + public async Task ManagementApi_AcceptsConfiguredPackagesAboveTheLegacyEightMiBLimit() + { + await using var factory = new WebDavWebApplicationFactory(); + using var client = factory.CreateJwtClient(); + var id = $"test.large-{Guid.NewGuid():N}"; + var payload = RandomNumberGenerator.GetBytes(8 * 1024 * 1024 + 64 * 1024); + await using var package = CreatePackage(id, "1.0", payload); + Assert.IsGreaterThan(8L * 1024 * 1024, package.Length); + using var form = new MultipartFormDataContent(); + form.Add(new StreamContent(package), "package", $"{id}.sdwpkg"); + + using var response = await client.PostAsync("/api/plugins/preview", form); + + Assert.AreEqual(HttpStatusCode.OK, response.StatusCode, await response.Content.ReadAsStringAsync()); + } + [TestMethod] public async Task ManagementApi_RequiresPreviewApproval_AndSupportsLifecycle() { @@ -104,10 +121,13 @@ public async Task ManagementApi_ReturnsBadRequestForMalformedPluginId() StringAssert.Contains(await response.Content.ReadAsStringAsync(), "invalid_plugin_request"); } - private static MemoryStream CreatePackage(string id, string apiVersion) + private static MemoryStream CreatePackage(string id, string apiVersion, byte[]? payload = null) { const string script = "globalThis.sdwPlugin={handlers:{ping:()=>({ok:true})}};"; var digest = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(script))).ToLowerInvariant(); + var integrityFiles = new Dictionary { ["index.js"] = digest }; + if (payload is not null) + integrityFiles["payload.bin"] = Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant(); var manifest = JsonSerializer.Serialize(new { id, @@ -126,7 +146,7 @@ private static MemoryStream CreatePackage(string id, string apiVersion) backgroundTasks = false }, platforms = new[] { "any" }, - integrity = new { files = new Dictionary { ["index.js"] = digest } }, + integrity = new { files = integrityFiles }, providers = Array.Empty(), dataVersion = 1 }); @@ -135,6 +155,12 @@ private static MemoryStream CreatePackage(string id, string apiVersion) { WriteEntry(archive, "manifest.json", manifest); WriteEntry(archive, "index.js", script); + if (payload is not null) + { + var payloadEntry = archive.CreateEntry("payload.bin", CompressionLevel.NoCompression); + using var payloadStream = payloadEntry.Open(); + payloadStream.Write(payload); + } } stream.Position = 0; return stream; diff --git a/SecondDimensionWatcherReDive.IntegrationTest/WebDavWebApplicationFactory.cs b/SecondDimensionWatcherReDive.IntegrationTest/WebDavWebApplicationFactory.cs index fffafb2..0b17b94 100644 --- a/SecondDimensionWatcherReDive.IntegrationTest/WebDavWebApplicationFactory.cs +++ b/SecondDimensionWatcherReDive.IntegrationTest/WebDavWebApplicationFactory.cs @@ -49,6 +49,7 @@ static WebDavWebApplicationFactory() Environment.SetEnvironmentVariable("PluginPlatform__RootPath", Path.Combine(Path.GetTempPath(), $"sdw-plugin-api-tests-{Environment.ProcessId}")); Environment.SetEnvironmentVariable("PluginPlatform__AllowUnsignedLocalPackages", "true"); + Environment.SetEnvironmentVariable("PluginPlatform__MaximumPackageBytes", "10485760"); } public List Mappings { get; } = new(); diff --git a/SecondDimensionWatcherReDive.Test/PluginControllerTests.cs b/SecondDimensionWatcherReDive.Test/PluginControllerTests.cs index eaa3eab..71ca2cc 100644 --- a/SecondDimensionWatcherReDive.Test/PluginControllerTests.cs +++ b/SecondDimensionWatcherReDive.Test/PluginControllerTests.cs @@ -1,4 +1,9 @@ +using System.Reflection; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Http.Metadata; using Microsoft.AspNetCore.Mvc; using Moq; using SecondDimensionWatcherReDive.Controllers; @@ -10,6 +15,60 @@ namespace SecondDimensionWatcherReDive.Test; [TestClass] public sealed class PluginControllerTests { + [TestMethod] + public void Preview_AllowsTheConfiguredPackageRangeAtTheMultipartBoundary() + { + var method = typeof(PluginController).GetMethod(nameof(PluginController.Preview)); + Assert.IsNotNull(method); + var requestLimit = method.GetCustomAttribute(); + var formLimit = method.GetCustomAttribute(); + + Assert.IsNotNull(requestLimit); + Assert.IsNotNull(formLimit); + Assert.AreEqual( + PluginPlatformOptions.MaximumUploadRequestBytes, + ((IRequestSizeLimitMetadata)requestLimit).MaxRequestBodySize); + Assert.AreEqual(PluginPlatformOptions.MaximumAllowedPackageBytes, formLimit.MultipartBodyLengthLimit); + Assert.IsGreaterThan( + PluginPlatformOptions.MaximumAllowedPackageBytes, + PluginPlatformOptions.MaximumUploadRequestBytes, + "The request envelope must leave room for multipart framing."); + } + + [TestMethod] + public void PluginPlatform_WhenRootIsMissing_FallsBackBesideThePasswordFile() + { + var passwordFile = Path.Combine(Path.GetTempPath(), "sdw-app-data", "password.json"); + var defaultRoot = PluginPlatformOptions.GetDefaultRootPath(passwordFile); + var configuration = new ConfigurationBuilder().AddInMemoryCollection().Build(); + var services = new ServiceCollection(); + services.AddPluginPlatform(configuration, defaultRoot); + using var provider = services.BuildServiceProvider(); + + var options = provider.GetRequiredService>().Value; + + Assert.AreEqual(Path.GetFullPath(defaultRoot), options.RootPath); + } + + [TestMethod] + public void PluginPlatform_WhenRootIsConfigured_PreservesTheConfiguredPath() + { + var configuredRoot = Path.Combine(Path.GetTempPath(), "sdw-configured-plugins"); + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + [$"{PluginPlatformOptions.SectionName}:RootPath"] = configuredRoot + }) + .Build(); + var services = new ServiceCollection(); + services.AddPluginPlatform(configuration, Path.Combine(Path.GetTempPath(), "unused-default")); + using var provider = services.BuildServiceProvider(); + + var options = provider.GetRequiredService>().Value; + + Assert.AreEqual(configuredRoot, options.RootPath); + } + [TestMethod] public async Task Preview_WhenStagingCapacityIsReached_ReturnsConflict() { diff --git a/SecondDimensionWatcherReDive/Controllers/PluginController.cs b/SecondDimensionWatcherReDive/Controllers/PluginController.cs index a00e74a..96fb150 100644 --- a/SecondDimensionWatcherReDive/Controllers/PluginController.cs +++ b/SecondDimensionWatcherReDive/Controllers/PluginController.cs @@ -21,7 +21,8 @@ internal sealed class PluginController( [HttpPost("preview")] [Consumes("multipart/form-data")] - [RequestSizeLimit(8 * 1024 * 1024)] + [RequestSizeLimit(PluginPlatformOptions.MaximumUploadRequestBytes)] + [RequestFormLimits(MultipartBodyLengthLimit = PluginPlatformOptions.MaximumAllowedPackageBytes)] public async Task> Preview( [FromForm] IFormFile package, CancellationToken cancellationToken) diff --git a/SecondDimensionWatcherReDive/PluginPlatform/PluginPlatformOptions.cs b/SecondDimensionWatcherReDive/PluginPlatform/PluginPlatformOptions.cs index 06882a7..8ee30fa 100644 --- a/SecondDimensionWatcherReDive/PluginPlatform/PluginPlatformOptions.cs +++ b/SecondDimensionWatcherReDive/PluginPlatform/PluginPlatformOptions.cs @@ -3,8 +3,13 @@ namespace SecondDimensionWatcherReDive.PluginPlatform; public sealed class PluginPlatformOptions { public const string SectionName = "PluginPlatform"; + public const long MaximumAllowedPackageBytes = 64L * 1024 * 1024; + public const long MaximumUploadRequestBytes = MaximumAllowedPackageBytes + 1024 * 1024; - public string RootPath { get; set; } = "./plugin-data"; + internal static string GetDefaultRootPath(string passwordFile) + => Path.Combine(Path.GetDirectoryName(Path.GetFullPath(passwordFile))!, "plugins"); + + public string RootPath { get; set; } = string.Empty; public bool AllowUnsignedLocalPackages { get; set; } public long MaximumPackageBytes { get; set; } = 4 * 1024 * 1024; public long MaximumExpandedBytes { get; set; } = 16 * 1024 * 1024; diff --git a/SecondDimensionWatcherReDive/PluginPlatform/PluginPlatformServiceExtensions.cs b/SecondDimensionWatcherReDive/PluginPlatform/PluginPlatformServiceExtensions.cs index dda4b9c..7d349a5 100644 --- a/SecondDimensionWatcherReDive/PluginPlatform/PluginPlatformServiceExtensions.cs +++ b/SecondDimensionWatcherReDive/PluginPlatform/PluginPlatformServiceExtensions.cs @@ -9,11 +9,20 @@ internal static class PluginPlatformServiceExtensions { public static IServiceCollection AddPluginPlatform( this IServiceCollection services, - IConfiguration configuration) + IConfiguration configuration, + string defaultRootPath) { services.AddOptions() .Bind(configuration.GetSection(PluginPlatformOptions.SectionName)) - .Validate(options => options.MaximumPackageBytes is >= 1_024 and <= 64 * 1024 * 1024, + .PostConfigure(options => + { + if (string.IsNullOrWhiteSpace(options.RootPath)) + options.RootPath = Path.GetFullPath(defaultRootPath); + }) + .Validate(options => !string.IsNullOrWhiteSpace(options.RootPath), + "RootPath must not be empty.") + .Validate(options => options.MaximumPackageBytes is >= 1_024 and + <= PluginPlatformOptions.MaximumAllowedPackageBytes, "MaximumPackageBytes must be between 1 KiB and 64 MiB.") .Validate(options => options.MaximumExpandedBytes >= options.MaximumPackageBytes, "MaximumExpandedBytes must be at least MaximumPackageBytes.") diff --git a/SecondDimensionWatcherReDive/Program.cs b/SecondDimensionWatcherReDive/Program.cs index b187dda..5408b14 100644 --- a/SecondDimensionWatcherReDive/Program.cs +++ b/SecondDimensionWatcherReDive/Program.cs @@ -91,7 +91,9 @@ .SetApplicationName("SecondDimensionWatcherReDive") .PersistKeysToFileSystem(new DirectoryInfo(dataProtectionKeyRingPath)); builder.Services.AddApplicationRuntimeSettings(runtimeSettingsProvider); -builder.Services.AddPluginPlatform(builder.Configuration); +builder.Services.AddPluginPlatform( + builder.Configuration, + PluginPlatformOptions.GetDefaultRootPath(passwordFile)); builder.Services.Configure( builder.Configuration.GetSection(MediaLibraryOptions.SectionName)); diff --git a/docs/plugin-platform.md b/docs/plugin-platform.md index 474ee29..88b50c8 100644 --- a/docs/plugin-platform.md +++ b/docs/plugin-platform.md @@ -8,6 +8,8 @@ Only authenticated application operators can manage plugins. Packages are upload Preview staging is bounded by both package count and aggregate bytes (`MaximumStagedPackages` and `MaximumStagedPackageBytes`); expired previews are removed before admitting another upload. The installation boundary validates the echoed checksum shape before reading the staged package. +The HTTP upload envelope accepts the platform's full configurable package range (up to 64 MiB, plus multipart framing), while `MaximumPackageBytes` remains the authoritative per-deployment package limit enforced during inspection. If `PluginPlatform:RootPath` is absent during an upgrade, the platform stores plugins in a `plugins` directory beside `PasswordFile`; packaged installations therefore fall back to `/var/lib/sdw-redive/plugins` instead of the read-only application directory. + Plugin and dependency versions use bounded, strict SemVer (`major.minor.patch` with optional legal prerelease/build identifiers); path separators, control/non-ASCII characters, empty identifiers, and ambiguous numeric forms are rejected before extraction. API versions use the same bounded grammar while permitting the API's `major.minor` form. Extraction also canonicalizes the version and temporary destinations and requires both to remain under that plugin's package directory. Publisher, provider, provider-operation, and handler names are bounded ASCII identifiers; display text is length/control-character checked. Unsigned local packages are rejected by default. Configure trusted publisher PEM public keys under `PluginPlatform:TrustedPublisherPublicKeys`. `AllowUnsignedLocalPackages` exists for local development and compatibility tests only; it does not accept invalid or untrusted signatures. The trusted public-key fingerprint is persisted as plugin ownership, so upgrades and reinstalls that would inherit retained configuration/data must use the same key. The remote-install endpoint is intentionally hard-disabled: the service never downloads or evaluates arbitrary JavaScript from a URL. From 282220991f9f2abcc136a232ca5bea614ed41377 Mon Sep 17 00:00:00 2001 From: mahoshojoHCG Date: Sun, 30 Aug 2026 10:50:29 +0800 Subject: [PATCH 10/45] fix: reauthenticate destructive download cancellation --- .../src/components/AnimationInfo.tsx | 45 +++++++++++++------ 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/SecondDimensionWatcherReDive.Client/src/components/AnimationInfo.tsx b/SecondDimensionWatcherReDive.Client/src/components/AnimationInfo.tsx index a0a750f..f097db0 100644 --- a/SecondDimensionWatcherReDive.Client/src/components/AnimationInfo.tsx +++ b/SecondDimensionWatcherReDive.Client/src/components/AnimationInfo.tsx @@ -31,6 +31,7 @@ import { submitDownload, } from "../animation/utils"; import { useAccess } from "../auth/hooks"; +import { retryAfterReauthentication } from "../auth/utils"; import { setPlaybackWatched } from "../playback/api"; import { usePlaybackStates } from "../playback/hooks"; import { formatBytes, formatFileSize } from "../utils/formatBytes"; @@ -127,7 +128,7 @@ const AutomationDispositionBadge: React.FC<{ }; const ActionButtons: React.FC<{ value: IAnimationInfo }> = ({ value }) => { - const { t } = useTranslation("animation"); + const { t } = useTranslation(["animation", "settings"]); const { canContentWrite, isAdministrator } = useAccess(); const { data: status } = useAnimationDownloadStatus( value.isDownloadTracked && !value.isDownloadFinished ? value.id : null, @@ -136,6 +137,7 @@ const ActionButtons: React.FC<{ value: IAnimationInfo }> = ({ value }) => { const [isSheetOpen, setIsSheetOpen] = React.useState(false); const [isRetrying, setIsRetrying] = React.useState(false); const [isReidentifyingFiles, setIsReidentifyingFiles] = React.useState(false); + const [isCancelling, setIsCancelling] = React.useState(false); const [isUpdatingWatched, setIsUpdatingWatched] = React.useState(false); const { data: playbackStates, mutate: mutatePlaybackStates } = usePlaybackStates(value.isDownloadFinished ? value.id : undefined); @@ -214,13 +216,35 @@ const ActionButtons: React.FC<{ value: IAnimationInfo }> = ({ value }) => { } }, [value.id, addToast, t]); + const onCancelDownload = React.useCallback( + async (removeFile: boolean) => { + if (isCancelling) return; + + setIsCancelling(true); + try { + const operation = () => cancelDownload(value.id, removeFile); + if (removeFile) { + await retryAfterReauthentication( + operation, + t("settings:system.reauthenticatePrompt"), + ); + } else { + await operation(); + } + } catch { + addToast({ title: t("toast.deleteFailed"), color: "danger" }); + } finally { + setIsCancelling(false); + } + }, + [addToast, isCancelling, t, value.id], + ); + const onDelete = React.useCallback(() => { if (window.confirm(t("confirm.deleteFile"))) { - cancelDownload(value.id, true).catch(() => - addToast({ title: t("toast.deleteFailed"), color: "danger" }), - ); + void onCancelDownload(true); } - }, [value.id, addToast, t]); + }, [onCancelDownload, t]); const onToggleAllWatched = React.useCallback(async () => { if (!playbackStates || playbackStates.length === 0) return; @@ -415,7 +439,7 @@ const ActionButtons: React.FC<{ value: IAnimationInfo }> = ({ value }) => { canContentWrite ? ( { const removeFile = isAdministrator; if ( @@ -427,12 +451,7 @@ const ActionButtons: React.FC<{ value: IAnimationInfo }> = ({ value }) => { ), ) ) { - cancelDownload(value.id, removeFile).catch(() => - addToast({ - title: t("toast.deleteFailed"), - color: "danger", - }), - ); + void onCancelDownload(removeFile); } }} > @@ -447,7 +466,7 @@ const ActionButtons: React.FC<{ value: IAnimationInfo }> = ({ value }) => { isAdministrator ? ( From ba8b3149cd20b48253860a9df71cfe05d1e39555 Mon Sep 17 00:00:00 2001 From: mahoshojoHCG Date: Sun, 30 Aug 2026 10:50:56 +0800 Subject: [PATCH 11/45] fix: make data restore transitions complete --- .github/workflows/backup-restore.yml | 2 + .../LogicalDataTransferPostgreSqlTests.cs | 251 +++++++++++++++++- .../LogicalDataTransferRepository.cs | 149 ++++++++++- deployments/sdw-backup | 73 ++++- deployments/tests/backup-restore-smoke.sh | 5 + docs/backup-restore.md | 9 +- packaging/backup.env | 4 + 7 files changed, 467 insertions(+), 26 deletions(-) diff --git a/.github/workflows/backup-restore.yml b/.github/workflows/backup-restore.yml index f0c664c..e219226 100644 --- a/.github/workflows/backup-restore.yml +++ b/.github/workflows/backup-restore.yml @@ -141,6 +141,8 @@ jobs: --key-ring-destination "$WORK_DIR/restored/keys" \ --plugin-destination "$WORK_DIR/restored/plugins" \ --safety-directory "$WORK_DIR/backups" + test "$(psql --no-psqlrc --tuples-only --no-align --command \ + "SELECT to_regclass('public.restore_guard') IS NULL")" = "t" test "$(psql --no-psqlrc --tuples-only --no-align --command 'SELECT count(*) FROM "Feeds"')" = "1" cmp "$WORK_DIR/source/password.json" "$WORK_DIR/restored/password.json" cmp "$WORK_DIR/source/appsettings.yml" "$WORK_DIR/restored/appsettings.yml" diff --git a/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/LogicalDataTransferPostgreSqlTests.cs b/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/LogicalDataTransferPostgreSqlTests.cs index d39c863..94437f9 100644 --- a/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/LogicalDataTransferPostgreSqlTests.cs +++ b/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/LogicalDataTransferPostgreSqlTests.cs @@ -1,6 +1,8 @@ using Microsoft.EntityFrameworkCore; +using Moq; using SecondDimensionWatcherReDive.Framework.DataRepository; using SecondDimensionWatcherReDive.Repositories; +using SecondDimensionWatcherReDive.Utils.FileStore; using Testcontainers.PostgreSql; using Models = SecondDimensionWatcherReDive.Models; @@ -83,7 +85,7 @@ public async Task FeedsPoliciesAndRulesRoundTripWithExplicitConflictStrategies() CreatedAt = DateTimeOffset.UtcNow }); await source.SaveChangesAsync(); - bundle = await new LogicalDataTransferRepository(source).ExportAsync( + bundle = await Repository(source).ExportAsync( LogicalDataCategory.Feeds | LogicalDataCategory.AutomationPolicies | LogicalDataCategory.FileNameRules, @@ -98,7 +100,7 @@ public async Task FeedsPoliciesAndRulesRoundTripWithExplicitConflictStrategies() } await using var target = new Models.ApplicationContext(Options); - var repository = new LogicalDataTransferRepository(target); + var repository = Repository(target); var first = await repository.ImportAsync( bundle, LogicalImportConflictStrategy.Skip, @@ -219,7 +221,7 @@ INSERT INTO "PlaybackPreferences" """); Assert.AreEqual(1, await source.PlaybackPreferences.AsNoTracking().CountAsync()); - bundle = await new LogicalDataTransferRepository(source).ExportAsync( + bundle = await Repository(source).ExportAsync( LogicalDataCategory.MetadataCorrections | LogicalDataCategory.Playback, Guid.Empty, "1.0.0", @@ -244,7 +246,7 @@ INSERT INTO "PlaybackPreferences" target.AddRange(targetInfo, Mapping(targetInfoId, VirtualPath)); await target.SaveChangesAsync(); - var result = await new LogicalDataTransferRepository(target).ImportAsync( + var result = await Repository(target).ImportAsync( bundle, LogicalImportConflictStrategy.Skip, Guid.Empty, @@ -268,6 +270,240 @@ INSERT INTO "PlaybackPreferences" Assert.IsFalse(preferences.AutoPlayNext); } + [TestMethod] + public async Task MetadataImportTransitionsMappingsPlaybackAndRemainsUndoable() + { + var publishedAt = DateTimeOffset.UtcNow.AddDays(-1); + var operationId = Guid.NewGuid(); + var infoId = Guid.NewGuid(); + const string DownloadUrl = "https://example.com/remapped-release.torrent"; + const string PreviousPath = "/Old Show/Old Group/Old Show S01E02.mkv"; + const string ProposedPath = "/Correct Show/Correct Group/Correct Show S02E05.mkv"; + const string PhysicalPath = "/target-media/release.mkv"; + + await using (var seed = new Models.ApplicationContext(Options)) + { + var oldAnimation = Animation("tv:old", "Old Show"); + var oldGroup = new Models.AnimationGroup { Id = Guid.NewGuid(), Name = "Old Group" }; + var info = Release(infoId, DownloadUrl, publishedAt); + info.Animation = oldAnimation; + info.Group = oldGroup; + info.Season = 1; + info.Episode = 2; + info.IsDownloadFinished = true; + info.FileStore = "local"; + info.StorePath = "/target-media"; + info.StateVersion = 7; + seed.AddRange( + oldAnimation, + oldGroup, + info, + new Models.FileMapping + { + Id = Guid.NewGuid(), + AnimationInfoId = infoId, + VirtualPath = PreviousPath, + PhysicalPath = PhysicalPath, + FileStore = "local" + }, + new Models.PlaybackProgress + { + Id = Guid.NewGuid(), + UserId = Guid.Empty, + AnimationInfoId = infoId, + VirtualPath = PreviousPath, + PositionSeconds = 120, + DurationSeconds = 1_440, + UpdatedAt = DateTimeOffset.UtcNow.AddHours(-1) + }); + await seed.SaveChangesAsync(); + } + + var bundle = new LogicalDataBundle( + 1, + DateTimeOffset.UtcNow, + "1.0.0", + LogicalDataCategory.MetadataCorrections | LogicalDataCategory.Playback, + [], + [], + [], + [ + new LogicalMetadataCorrection( + operationId, + DownloadUrl, + "[Group] Example - 02", + publishedAt, + "tv:correct", + "Correct Show", + "Correct Show", + "/correct.jpg", + "corrected description", + 2, + 5, + "Correct Group", + DateTimeOffset.UtcNow) + ], + [ + new LogicalPlaybackProgress( + ProposedPath, + 600, + 1_440, + false, + DateTimeOffset.UtcNow, + null) + ], + null); + var mapper = new Mock(MockBehavior.Strict); + mapper.Setup(candidate => candidate.PreviewDownloadAsync( + It.Is(info => + info.Id == infoId && + info.Animation != null && info.Animation.TmdbId == "tv:correct" && + info.Group != null && info.Group.Name == "Correct Group" && + info.Season == 2 && info.Episode == 5), + CancellationToken.None)) + .ReturnsAsync(new FileMappingPreview( + [ + new FileMapping( + Guid.NewGuid(), + infoId, + ProposedPath, + PhysicalPath, + "local") + ], + [])); + + await using (var importing = new Models.ApplicationContext(Options)) + { + var result = await Repository(importing, mapper.Object).ImportAsync( + bundle, + LogicalImportConflictStrategy.Overwrite, + Guid.Empty, + CancellationToken.None); + Assert.AreEqual(1, result.Added); + Assert.AreEqual(1, result.Updated); + } + + await using (var verification = new Models.ApplicationContext(Options)) + { + var info = await verification.AnimationInfo + .Include(candidate => candidate.Animation) + .Include(candidate => candidate.Group) + .SingleAsync(); + Assert.AreEqual(8, info.StateVersion); + Assert.AreEqual(operationId, info.CurrentMetadataReviewOperationId); + Assert.AreEqual("tv:correct", info.Animation!.TmdbId); + Assert.AreEqual("Correct Group", info.Group!.Name); + + var mapping = await verification.FileMappings.SingleAsync(); + Assert.AreEqual(ProposedPath, mapping.VirtualPath); + Assert.AreEqual(PhysicalPath, mapping.PhysicalPath); + var snapshots = await verification.MetadataReviewMappingSnapshots + .OrderBy(snapshot => snapshot.Kind) + .ToListAsync(); + Assert.AreEqual(2, snapshots.Count); + Assert.AreEqual(PreviousPath, + snapshots.Single(snapshot => snapshot.Kind == MetadataReviewMappingKind.Previous).VirtualPath); + Assert.AreEqual(ProposedPath, + snapshots.Single(snapshot => snapshot.Kind == MetadataReviewMappingKind.Proposed).VirtualPath); + Assert.IsTrue(snapshots.All(snapshot => snapshot.PhysicalPath == PhysicalPath)); + var progress = await verification.PlaybackProgresses.SingleAsync(); + Assert.AreEqual(ProposedPath, progress.VirtualPath); + Assert.AreEqual(600, progress.PositionSeconds); + } + + await using (var undoContext = new Models.ApplicationContext(Options)) + { + var undone = await new MetadataReviewRepository(undoContext, Options).UndoAsync( + operationId, + 8, + CancellationToken.None); + Assert.AreEqual(MetadataReviewMutationOutcome.Success, undone.Outcome); + } + + await using (var verification = new Models.ApplicationContext(Options)) + { + var info = await verification.AnimationInfo + .Include(candidate => candidate.Animation) + .Include(candidate => candidate.Group) + .SingleAsync(); + Assert.AreEqual(9, info.StateVersion); + Assert.AreEqual("tv:old", info.Animation!.TmdbId); + Assert.AreEqual("Old Group", info.Group!.Name); + Assert.AreEqual(PreviousPath, (await verification.FileMappings.SingleAsync()).VirtualPath); + var progress = await verification.PlaybackProgresses.SingleAsync(); + Assert.AreEqual(PreviousPath, progress.VirtualPath); + Assert.AreEqual(600, progress.PositionSeconds); + } + } + + [TestMethod] + public async Task UnavailableMetadataMappingPlanRollsBackWholeImport() + { + var publishedAt = DateTimeOffset.UtcNow.AddDays(-1); + var infoId = Guid.NewGuid(); + const string DownloadUrl = "https://example.com/unavailable-mapping.torrent"; + const string PreviousPath = "/unknown/original.mkv"; + await using (var seed = new Models.ApplicationContext(Options)) + { + var info = Release(infoId, DownloadUrl, publishedAt); + info.IsDownloadFinished = true; + info.FileStore = "local"; + info.StorePath = "/missing-media"; + seed.AddRange(info, Mapping(infoId, PreviousPath)); + await seed.SaveChangesAsync(); + } + + var bundle = new LogicalDataBundle( + 1, + DateTimeOffset.UtcNow, + "1.0.0", + LogicalDataCategory.Feeds | LogicalDataCategory.MetadataCorrections, + [new LogicalFeed(Guid.NewGuid(), "https://example.com/new.xml", "New", DateTimeOffset.UtcNow)], + [], + [], + [ + new LogicalMetadataCorrection( + Guid.NewGuid(), + DownloadUrl, + "[Group] Example - 02", + publishedAt, + "tv:correct", + "Correct Show", + "Correct Show", + null, + "corrected description", + 2, + 5, + "Correct Group", + DateTimeOffset.UtcNow) + ], + [], + null); + var mapper = new Mock(MockBehavior.Strict); + mapper.Setup(candidate => candidate.PreviewDownloadAsync( + It.IsAny(), + CancellationToken.None)) + .ReturnsAsync((FileMappingPreview?)null); + + await using (var importing = new Models.ApplicationContext(Options)) + { + await Assert.ThrowsAsync(() => + Repository(importing, mapper.Object).ImportAsync( + bundle, + LogicalImportConflictStrategy.Overwrite, + Guid.Empty, + CancellationToken.None)); + } + + await using var verification = new Models.ApplicationContext(Options); + Assert.AreEqual(0, await verification.Feeds.CountAsync()); + Assert.AreEqual(0, await verification.MetadataReviewOperations.CountAsync()); + var infoAfter = await verification.AnimationInfo.SingleAsync(); + Assert.AreEqual("uncorrected", infoAfter.Description); + Assert.AreEqual(0, infoAfter.StateVersion); + Assert.AreEqual(PreviousPath, (await verification.FileMappings.SingleAsync()).VirtualPath); + } + [TestMethod] public async Task FailConflictStrategyRollsBackEarlierItems() { @@ -301,7 +537,7 @@ public async Task FailConflictStrategyRollsBackEarlierItems() await using (var importing = new Models.ApplicationContext(Options)) { await Assert.ThrowsAsync(() => - new LogicalDataTransferRepository(importing).ImportAsync( + Repository(importing).ImportAsync( bundle, LogicalImportConflictStrategy.Fail, Guid.Empty, @@ -348,4 +584,9 @@ private static Models.FileMapping Mapping(Guid animationInfoId, string virtualPa PhysicalPath = "/media/example.mkv", FileStore = "local" }; + + private static LogicalDataTransferRepository Repository( + Models.ApplicationContext context, + IFileMapper? fileMapper = null) => + new(context, fileMapper ?? Mock.Of()); } diff --git a/SecondDimensionWatcherReDive/Repositories/LogicalDataTransferRepository.cs b/SecondDimensionWatcherReDive/Repositories/LogicalDataTransferRepository.cs index d9ef62b..fff9139 100644 --- a/SecondDimensionWatcherReDive/Repositories/LogicalDataTransferRepository.cs +++ b/SecondDimensionWatcherReDive/Repositories/LogicalDataTransferRepository.cs @@ -2,10 +2,14 @@ using System.Security.Cryptography; using System.Text; using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Utils.FileStore; +using DataFileMapping = SecondDimensionWatcherReDive.Framework.DataRepository.FileMapping; namespace SecondDimensionWatcherReDive.Repositories; -public sealed class LogicalDataTransferRepository(Models.ApplicationContext context) +public sealed class LogicalDataTransferRepository( + Models.ApplicationContext context, + IFileMapper fileMapper) : ILogicalDataTransferRepository { private const int FormatVersion = 1; @@ -43,18 +47,18 @@ public async Task ExportAsync( var rules = categories.HasFlag(LogicalDataCategory.FileNameRules) ? await (from rule in context.FileNameRegexRules.AsNoTracking() - join animation in context.Animations.AsNoTracking() - on rule.AnimationId equals animation.Id - orderby animation.TmdbId, rule.CreatedAt - select new LogicalFileNameRule( - rule.Id, - animation.TmdbId, - animation.Name, - animation.OriginalName, - animation.PosterPath, - rule.Pattern, - rule.Description, - rule.CreatedAt)) + join animation in context.Animations.AsNoTracking() + on rule.AnimationId equals animation.Id + orderby animation.TmdbId, rule.CreatedAt + select new LogicalFileNameRule( + rule.Id, + animation.TmdbId, + animation.Name, + animation.OriginalName, + animation.PosterPath, + rule.Pattern, + rule.Description, + rule.CreatedAt)) .ToListAsync(cancellationToken) : []; @@ -143,7 +147,14 @@ public async Task ImportAsync( await ImportFeedsAsync(bundle, conflictStrategy, feedsByUrl, usedFeedIds, statistics, cancellationToken); await ImportPoliciesAsync(bundle, conflictStrategy, feedsByUrl, statistics, cancellationToken); await ImportRulesAsync(bundle, conflictStrategy, statistics, cancellationToken); + // Mapping previews can consume filename rules and animation rows imported in + // this bundle. Flush them inside the transaction before planning corrections. + await context.SaveChangesAsync(cancellationToken); await ImportMetadataCorrectionsAsync(bundle, conflictStrategy, statistics, cancellationToken); + // Metadata correction imports replace FileMappings in the same transaction. + // Flush them before playback import queries by virtual path; a later failure + // still rolls the entire import back. + await context.SaveChangesAsync(cancellationToken); await ImportPlaybackAsync(bundle, conflictStrategy, userId, statistics, cancellationToken); await context.SaveChangesAsync(cancellationToken); @@ -302,7 +313,19 @@ private async Task ImportMetadataCorrectionsAsync( bundle.MetadataCorrections.Count == 0) return; + // Use the same lock order as metadata review and FileMappingRepository so a + // correction cannot race another virtual-path transition. + await MappingTransactionLock.AcquireAsync(context, cancellationToken); + var downloadUrls = bundle.MetadataCorrections.Select(item => item.ReleaseDownloadUrl).Distinct().ToArray(); + var candidateIds = await context.AnimationInfo.AsNoTracking() + .Where(info => downloadUrls.Contains(info.DownloadUrl)) + .Select(info => info.Id) + .ToListAsync(cancellationToken); + await MappingTransactionLock.LockAnimationInfosAsync( + context, + candidateIds, + cancellationToken); var candidates = await context.AnimationInfo .Include(info => info.Animation) .Include(info => info.Group) @@ -365,6 +388,60 @@ private async Task ImportMetadataCorrectionsAsync( groups.Add(group.Name, group); } + var existingMappings = await context.FileMappings + .AsNoTracking() + .Where(mapping => mapping.AnimationInfoId == info.Id) + .OrderBy(mapping => mapping.VirtualPath) + .ToListAsync(cancellationToken); + IReadOnlyList proposedMappings; + if (info.IsDownloadFinished) + { + var proposedInfo = info.ToRecord() with + { + Animation = animation.ToRecord(), + Group = group?.ToRecord(), + Description = imported.Description, + Season = imported.Season, + Episode = imported.Episode, + MetadataStatus = MetadataReviewStatus.Reviewed, + MetadataConfidence = 1, + MetadataLastError = null, + MetadataReviewedAt = imported.AppliedAt, + IsAiProcessed = true, + AiRetryCount = 0 + }; + var preview = await fileMapper.PreviewDownloadAsync( + proposedInfo, + cancellationToken); + if (preview is null) + throw new LogicalDataImportConflictException( + $"Cannot rebuild mappings for imported metadata:{imported.ReleaseTitle}."); + proposedMappings = preview.Mappings; + } + else + { + // Match the normal review workflow: metadata-only corrections do not + // delete an inconsistent legacy mapping before download completion. + proposedMappings = existingMappings + .Select(mapping => mapping.ToRecord()) + .ToList(); + } + + if (proposedMappings.Any(mapping => mapping.AnimationInfoId != info.Id) || + proposedMappings.Select(mapping => mapping.VirtualPath) + .Distinct(StringComparer.Ordinal).Count() != proposedMappings.Count) + throw new LogicalDataImportConflictException( + $"Invalid mapping plan for imported metadata:{imported.ReleaseTitle}."); + + var proposedPaths = proposedMappings.Select(mapping => mapping.VirtualPath).ToArray(); + if (proposedPaths.Length > 0 && + await context.FileMappings.AsNoTracking().AnyAsync( + mapping => mapping.AnimationInfoId != info.Id && + proposedPaths.Contains(mapping.VirtualPath), + cancellationToken)) + throw new LogicalDataImportConflictException( + $"Mapping conflict for imported metadata:{imported.ReleaseTitle}."); + var nextVersion = checked(info.StateVersion + 1); var operation = new Models.MetadataReviewOperation { @@ -399,7 +476,26 @@ private async Task ImportMetadataCorrectionsAsync( PreviousIsAiProcessed = info.IsAiProcessed, PreviousAiRetryCount = info.AiRetryCount, PreviousReviewedAt = info.MetadataReviewedAt, - PreviousCurrentOperationId = info.CurrentMetadataReviewOperationId + PreviousCurrentOperationId = info.CurrentMetadataReviewOperationId, + MappingSnapshots = existingMappings.Select(mapping => + new Models.MetadataReviewMappingSnapshot + { + Id = Guid.NewGuid(), + OperationId = imported.OperationId, + Kind = MetadataReviewMappingKind.Previous, + VirtualPath = mapping.VirtualPath, + PhysicalPath = mapping.PhysicalPath, + FileStore = mapping.FileStore + }).Concat(proposedMappings.Select(mapping => + new Models.MetadataReviewMappingSnapshot + { + Id = Guid.NewGuid(), + OperationId = imported.OperationId, + Kind = MetadataReviewMappingKind.Proposed, + VirtualPath = mapping.VirtualPath, + PhysicalPath = mapping.PhysicalPath, + FileStore = mapping.FileStore + })).ToList() }; context.MetadataReviewOperations.Add(operation); info.Animation = animation; @@ -415,6 +511,31 @@ private async Task ImportMetadataCorrectionsAsync( info.AiRetryCount = 0; info.StateVersion = nextVersion; info.CurrentMetadataReviewOperationId = operation.Id; + + var replacementMappings = proposedMappings.Select(mapping => + new Models.FileMapping + { + Id = Guid.NewGuid(), + AnimationInfoId = info.Id, + VirtualPath = mapping.VirtualPath, + PhysicalPath = mapping.PhysicalPath, + FileStore = mapping.FileStore + }).ToList(); + await PlaybackProgressMappingMigrator.MigrateAsync( + context, + info.Id, + existingMappings, + replacementMappings, + cancellationToken); + await context.FileMappings + .Where(mapping => mapping.AnimationInfoId == info.Id) + .ExecuteDeleteAsync(cancellationToken); + if (replacementMappings.Count > 0) + await context.FileMappings.AddRangeAsync(replacementMappings, cancellationToken); + + // Make this plan visible to collision checks for subsequent corrections + // in the same bundle. The enclosing transaction still provides atomicity. + await context.SaveChangesAsync(cancellationToken); existingOperationIds.Add(operation.Id); statistics.Add(); } diff --git a/deployments/sdw-backup b/deployments/sdw-backup index 1670cad..1303ae8 100755 --- a/deployments/sdw-backup +++ b/deployments/sdw-backup @@ -66,7 +66,9 @@ Restore options: PostgreSQL is read from PGHOST/PGPORT/PGUSER/PGPASSWORD/PGDATABASE. If those are absent, ConnectionStrings__sdw may contain an ASP.NET semicolon connection -string. Secrets are never printed. +string. PGMAINTENANCEDATABASE defaults to postgres and is used only to replace +the target database during restore. Restore requires a target-owning role with +CREATEDB (or a superuser). Secrets are never printed. EOF } @@ -478,6 +480,71 @@ major_version() { printf '%s' "${value%%.*}" } +prepare_database_replacement() { + local maintenance_database=${PGMAINTENANCEDATABASE:-postgres} + [[ -n "${PGDATABASE}" && -n "${maintenance_database}" ]] || + die "target and maintenance database names are required" + case "${PGDATABASE}" in + postgres|template0|template1) + die "refusing to replace a PostgreSQL system database" + ;; + esac + [[ "${PGDATABASE}" != "${maintenance_database}" ]] || + die "maintenance database must differ from the restore target" + + require_command createdb + require_command dropdb + psql --no-psqlrc --dbname "${maintenance_database}" --tuples-only --no-align \ + --set=ON_ERROR_STOP=1 --command 'SELECT 1' >/dev/null || + die "PostgreSQL maintenance database is unavailable" + + local can_replace + can_replace=$(psql --no-psqlrc --dbname "${PGDATABASE}" \ + --tuples-only --no-align --set=ON_ERROR_STOP=1 --command \ + "SELECT CASE WHEN role.rolsuper OR + (role.rolcreatedb AND pg_has_role(current_user, db.datdba, 'MEMBER')) + THEN 'yes' ELSE 'no' END + FROM pg_database AS db CROSS JOIN pg_roles AS role + WHERE db.datname = current_database() AND role.rolname = current_user" \ + 2>/dev/null) || die "cannot inspect target database ownership" + [[ "${can_replace}" == yes ]] || + die "PostgreSQL role must own the target and have CREATEDB, or be superuser" +} + +replace_database_from_archive() { + local archive=$1 maintenance_database=${PGMAINTENANCEDATABASE:-postgres} + local database_owner database_encoding database_collation database_ctype + database_owner=$(psql --no-psqlrc --dbname "${PGDATABASE}" \ + --tuples-only --no-align --set=ON_ERROR_STOP=1 --command \ + "SELECT pg_get_userbyid(datdba) FROM pg_database WHERE datname = current_database()") + database_encoding=$(psql --no-psqlrc --dbname "${PGDATABASE}" \ + --tuples-only --no-align --set=ON_ERROR_STOP=1 --command \ + "SELECT pg_encoding_to_char(encoding) FROM pg_database WHERE datname = current_database()") + database_collation=$(psql --no-psqlrc --dbname "${PGDATABASE}" \ + --tuples-only --no-align --set=ON_ERROR_STOP=1 --command \ + "SELECT datcollate FROM pg_database WHERE datname = current_database()") + database_ctype=$(psql --no-psqlrc --dbname "${PGDATABASE}" \ + --tuples-only --no-align --set=ON_ERROR_STOP=1 --command \ + "SELECT datctype FROM pg_database WHERE datname = current_database()") + [[ -n "${database_owner}" && -n "${database_encoding}" && + -n "${database_collation}" && -n "${database_ctype}" ]] || + die "target database properties are unavailable" + + # pg_restore --clean only drops objects present in the archive. Recreate the + # database so objects introduced after the backup cannot survive a replacement + # restore and conflict with the restored EF migration history. + dropdb --force --maintenance-db="${maintenance_database}" -- "${PGDATABASE}" + createdb --maintenance-db="${maintenance_database}" \ + --template=template0 \ + --owner="${database_owner}" \ + --encoding="${database_encoding}" \ + --lc-collate="${database_collation}" \ + --lc-ctype="${database_ctype}" \ + -- "${PGDATABASE}" + pg_restore --exit-on-error --no-owner --no-acl \ + --dbname "${PGDATABASE}" "${archive}" +} + restore_backup() { [[ $# -ge 1 ]] || die "restore requires an archive" local archive=$1 @@ -516,6 +583,7 @@ restore_backup() { parse_connection_string require_command psql require_command pg_dump + prepare_database_replacement prepare_archive "${archive}" "${identity}" local extracted="${backup_temp_dir}/extracted" @@ -560,8 +628,7 @@ restore_backup() { chmod 0600 "${safety_dump}.partial" mv "${safety_dump}.partial" "${safety_dump}" - pg_restore --clean --if-exists --exit-on-error --no-owner --no-acl \ - --dbname "${PGDATABASE}" "${extracted}/database.dump" + replace_database_from_archive "${extracted}/database.dump" [[ "$(database_schema_version)" == "${backup_schema}" ]] || die "restored database schema does not match the backup" diff --git a/deployments/tests/backup-restore-smoke.sh b/deployments/tests/backup-restore-smoke.sh index 38f4875..34d7059 100755 --- a/deployments/tests/backup-restore-smoke.sh +++ b/deployments/tests/backup-restore-smoke.sh @@ -65,6 +65,9 @@ fi createdb sdw_restore export PGDATABASE=sdw_restore +psql --no-psqlrc --set=ON_ERROR_STOP=1 --command \ + "CREATE TABLE destination_only (id integer); INSERT INTO destination_only VALUES (99);" \ + >/dev/null "${repo_root}/deployments/sdw-backup" restore "${archive}" \ --confirm-replace \ --expected-version "${app_version}" \ @@ -76,5 +79,7 @@ export PGDATABASE=sdw_restore test "$(psql --no-psqlrc --tuples-only --no-align --command \ 'SELECT value FROM drill_items WHERE id = 1')" = round-trip +test "$(psql --no-psqlrc --tuples-only --no-align --command \ + "SELECT to_regclass('public.destination_only') IS NULL")" = t cmp "${drill_root}/password.json" "${drill_root}/restored/password.json" printf 'backup restore smoke test passed\n' diff --git a/docs/backup-restore.md b/docs/backup-restore.md index 49e2952..3858501 100644 --- a/docs/backup-restore.md +++ b/docs/backup-restore.md @@ -84,14 +84,16 @@ podman-compose exec sdw-redive sdw-backup verify /app/backups/sdw-backup-....tar 恢复是替换操作。先停止所有应用副本和后台任务；仅停止应用，不要停止 PostgreSQL。 -1. 把目标应用安装为与备份相同的 major 版本，并准备空数据库。 +1. 把目标应用安装为与备份相同的 major 版本。恢复脚本会在生成 safety dump 后删除并重建目标数据库，数据库登录角色必须拥有目标数据库并具有 `CREATEDB`（或使用 PostgreSQL 超级用户）；不要把 `PGDATABASE` 指向 `postgres`、`template0` 或 `template1`。默认的低权限应用账号不应长期获得 `CREATEDB`，恢复时请临时提供独立的数据库管理员凭据。 2. 预先挂载足够空间；脚本在任何数据库写入前验证路径、链接、所有 SHA-256、`pg_restore --list`、格式、major 版本、可选 schema 版本、临时空间和目标目录可写性。 3. 执行恢复。命令先在 safety directory 创建现有数据库 dump，旧配置、密码和密钥环也会重命名为 `.pre-restore-*`，可人工回退。 4. 修正文件所有者与权限，启动应用，检查 `/api/auth/allowRegister`、登录、订阅和文件浏览。 ```bash sudo systemctl stop sdw-redive -sudo -u sdw-redive --preserve-env=PGHOST,PGPORT,PGUSER,PGPASSWORD,PGDATABASE \ +# /etc/sdw-redive 由 root 管理，因此完整恢复必须以 root 写入配置； +# 下列 PostgreSQL 变量应由 root 可读的凭据文件或当前管理会话提供。 +sudo --preserve-env=PGHOST,PGPORT,PGUSER,PGPASSWORD,PGDATABASE,PGMAINTENANCEDATABASE \ sdw-backup restore /var/lib/sdw-redive/backups/sdw-backup-....tar.gz \ --confirm-replace \ --expected-version 2.3.0 \ @@ -101,8 +103,7 @@ sudo -u sdw-redive --preserve-env=PGHOST,PGPORT,PGUSER,PGPASSWORD,PGDATABASE \ --key-ring-destination /var/lib/sdw-redive/data-protection-keys sudo chown root:sdw-redive /etc/sdw-redive/appsettings.yml sudo chmod 0640 /etc/sdw-redive/appsettings.yml -sudo chown -R sdw-redive:sdw-redive /var/lib/sdw-redive/data-protection-keys \ - /var/lib/sdw-redive/password.json +sudo chown -R sdw-redive:sdw-redive /var/lib/sdw-redive sudo systemctl start sdw-redive curl --fail http://127.0.0.1:5097/api/auth/allowRegister ``` diff --git a/packaging/backup.env b/packaging/backup.env index 4b8684b..400ebdf 100644 --- a/packaging/backup.env +++ b/packaging/backup.env @@ -4,6 +4,10 @@ PGPORT=5432 PGUSER=sdw PGDATABASE=sdw PGPASSWORD=CHANGE_ME +# Replacement restore additionally requires a temporary database administrator +# that owns PGDATABASE and has CREATEDB. Do not grant that privilege permanently +# to this runtime/backup account; override PGUSER/PGPASSWORD for the manual restore. +# PGMAINTENANCEDATABASE=postgres SDW_BACKUP_DIRECTORY=/var/lib/sdw-redive/backups SDW_BACKUP_RETENTION_DAYS=14 From e0fc4c24bfe3d5af013f9f3daf511eb93ccf54e8 Mon Sep 17 00:00:00 2001 From: mahoshojoHCG Date: Sun, 30 Aug 2026 11:13:56 +0800 Subject: [PATCH 12/45] fix: harden progressive transcoding --- .../FfmpegProcessRunnerTests.cs | 15 +- .../HlsTranscodingServiceTests.cs | 211 ++++++++++++++++-- .../TranscodingPlannerTests.cs | 38 +++- .../Transcoding/FfmpegProcessRunner.cs | 72 +++--- .../Transcoding/HlsTranscodingService.cs | 172 ++++++++++---- .../Services/Transcoding/TranscodingModels.cs | 4 +- .../Transcoding/TranscodingPlanner.cs | 34 ++- 7 files changed, 430 insertions(+), 116 deletions(-) diff --git a/SecondDimensionWatcherReDive.Test/FfmpegProcessRunnerTests.cs b/SecondDimensionWatcherReDive.Test/FfmpegProcessRunnerTests.cs index c1a5114..6aaaee2 100644 --- a/SecondDimensionWatcherReDive.Test/FfmpegProcessRunnerTests.cs +++ b/SecondDimensionWatcherReDive.Test/FfmpegProcessRunnerTests.cs @@ -32,6 +32,8 @@ public async Task ProbeAndGenerateHlsAsync_ProducesProgressivePlaylistFromPipeIn MediaProbe probe; await using (var source = File.OpenRead(sourcePath)) probe = await runner.ProbeAsync(source, CancellationToken.None); + Assert.IsTrue(probe.Video?.Profile is "Baseline" or "Constrained Baseline" or "Main" or "High"); + Assert.AreEqual("yuv420p", probe.Video?.PixelFormat); var sourceInfo = new FileInfo(sourcePath); var sourceModel = new TranscodingSource( Guid.NewGuid(), @@ -67,13 +69,18 @@ public async Task ProbeAndGenerateHlsAsync_ProducesProgressivePlaylistFromPipeIn StringAssert.Contains( await File.ReadAllTextAsync(Path.Combine(output, "media.m3u8")), "#EXT-X-ENDLIST"); - IReadOnlyList subtitles; - await using (var source = File.OpenRead(sourcePath)) - subtitles = await runner.ExtractTextSubtitlesAsync( + var subtitles = new List(); + for (var index = 0; index < plan.TextSubtitles.Count; index++) + { + await using var source = File.OpenRead(sourcePath); + var subtitle = await runner.ExtractTextSubtitleAsync( source, - plan, + plan.TextSubtitles[index], + index + 1, output, CancellationToken.None); + if (subtitle is not null) subtitles.Add(subtitle); + } Assert.AreEqual(1, subtitles.Count); StringAssert.StartsWith( await File.ReadAllTextAsync(Path.Combine(output, subtitles[0].FileName)), diff --git a/SecondDimensionWatcherReDive.Test/HlsTranscodingServiceTests.cs b/SecondDimensionWatcherReDive.Test/HlsTranscodingServiceTests.cs index e77c747..f46da09 100644 --- a/SecondDimensionWatcherReDive.Test/HlsTranscodingServiceTests.cs +++ b/SecondDimensionWatcherReDive.Test/HlsTranscodingServiceTests.cs @@ -1,3 +1,4 @@ +using System.Reflection; using Microsoft.AspNetCore.StaticFiles; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging.Abstractions; @@ -171,6 +172,99 @@ public async Task FailedJobDeletesPartialOutputAndReportsFailureRate() Assert.AreEqual(1, metrics.FailureRate); } + [TestMethod] + public async Task EmbeddedSubtitleIsPublishedWhileHlsGenerationIsStillRunning() + { + var runner = new ProgressiveSubtitleRunner(failFirstSubtitle: false); + await using var fixture = await TranscodingFixture.CreateAsync(runner); + var initial = await fixture.Service.PrepareAsync( + fixture.AnimationInfoId, + "episode.mkv", + TranscodingSelection.Create("auto", null, null, null, null), + CancellationToken.None); + + try + { + await runner.GenerationStarted.Task.WaitAsync(TimeSpan.FromSeconds(3)); + await runner.ExtractionCompleted.Task.WaitAsync(TimeSpan.FromSeconds(3)); + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(3)); + TranscodingSessionStatus status; + do + { + status = await fixture.Service.GetStatusAsync( + initial.SessionId, + initial.AccessToken, + timeout.Token) + ?? throw new AssertFailedException("The transcoding session disappeared."); + if (status.Subtitles.Count == 1) break; + await Task.Delay(10, timeout.Token); + } while (true); + + Assert.AreEqual(TranscodingJobState.Transcoding, status.State); + Assert.IsTrue(status.IsPlayable); + Assert.AreEqual("subtitle-2.vtt", status.Subtitles[0].FileName); + } + finally + { + runner.ReleaseGeneration.TrySetResult(); + } + + await WaitForStateAsync(fixture.Service, initial, TranscodingJobState.Ready); + } + + [TestMethod] + public async Task FailedEmbeddedSubtitleDoesNotSuppressValidTrack() + { + var runner = new ProgressiveSubtitleRunner(failFirstSubtitle: true); + runner.ReleaseGeneration.TrySetResult(); + await using var fixture = await TranscodingFixture.CreateAsync(runner); + var initial = await fixture.Service.PrepareAsync( + fixture.AnimationInfoId, + "episode.mkv", + TranscodingSelection.Create("auto", null, null, null, null), + CancellationToken.None); + + var ready = await WaitForStateAsync(fixture.Service, initial, TranscodingJobState.Ready); + + CollectionAssert.AreEqual(new[] { 2, 3 }, runner.ExtractedStreamIndices.ToArray()); + Assert.AreEqual(1, ready.Subtitles.Count); + Assert.AreEqual("subtitle-3.vtt", ready.Subtitles[0].FileName); + } + + [TestMethod] + public async Task CacheCleanup_WaitsForSessionCreationCriticalSection() + { + await using var fixture = await TranscodingFixture.CreateAsync(new CompletingRunner()); + var serviceType = typeof(HlsTranscodingService); + var creationGate = (SemaphoreSlim)(serviceType.GetField( + "_creationGate", + BindingFlags.Instance | BindingFlags.NonPublic)?.GetValue(fixture.Service) + ?? throw new AssertFailedException("The cache creation gate was not found.")); + var cleanupMethod = serviceType.GetMethod( + "CleanupCacheAsync", + BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new AssertFailedException("The cache cleanup method was not found."); + + await creationGate.WaitAsync(CancellationToken.None); + Task cleanupTask; + try + { + cleanupTask = (Task)(cleanupMethod.Invoke( + fixture.Service, + [false, CancellationToken.None]) + ?? throw new AssertFailedException("Cache cleanup did not return a task.")); + Assert.IsFalse( + cleanupTask.IsCompleted, + "Cleanup must not inspect or evict cache entries while PrepareAsync can attach a session."); + } + finally + { + creationGate.Release(); + } + + await cleanupTask.WaitAsync(TimeSpan.FromSeconds(3)); + } + private static async Task WaitForStateAsync( IHlsTranscodingService service, TranscodingSessionStatus session, @@ -201,9 +295,9 @@ public Task ProbeAsync(Stream source, CancellationToken cancellation "matroska", TimeSpan.FromSeconds(30), [ - new MediaStreamProbe(0, "video", "h264", null, null, true, false, false), - new MediaStreamProbe(1, "audio", "aac", "jpn", "Japanese", true, false, false), - new MediaStreamProbe(2, "subtitle", "ass", "eng", "English", true, false, false) + new MediaStreamProbe(0, "video", "h264", null, null, true, false, false, "High", "yuv420p"), + new MediaStreamProbe(1, "audio", "aac", "jpn", "Japanese", true, false, false, null, null), + new MediaStreamProbe(2, "subtitle", "ass", "eng", "English", true, false, false, null, null) ])); public async Task GenerateHlsAsync( @@ -228,18 +322,19 @@ await File.WriteAllTextAsync( return new FfmpegRunResult(0, string.Empty); } - public async Task> ExtractTextSubtitlesAsync( + public async Task ExtractTextSubtitleAsync( Stream source, - TranscodingPlan plan, + MediaStreamProbe subtitle, + int ordinal, string outputDirectory, CancellationToken cancellationToken) { - const string name = "subtitle-2.vtt"; + var name = $"subtitle-{subtitle.Index}.vtt"; await File.WriteAllTextAsync( Path.Combine(outputDirectory, name), "WEBVTT\n", cancellationToken); - return [new TranscodingSubtitle(name, "English", "eng", "vtt")]; + return new TranscodingSubtitle(name, "English", "eng", "vtt"); } } @@ -252,8 +347,8 @@ public Task ProbeAsync(Stream source, CancellationToken cancellation "matroska", TimeSpan.FromSeconds(30), [ - new MediaStreamProbe(0, "video", "h264", null, null, true, false, false), - new MediaStreamProbe(1, "audio", "aac", null, null, true, false, false) + new MediaStreamProbe(0, "video", "h264", null, null, true, false, false, "High", "yuv420p"), + new MediaStreamProbe(1, "audio", "aac", null, null, true, false, false, null, null) ])); public async Task GenerateHlsAsync( @@ -274,12 +369,13 @@ await File.WriteAllTextAsync( return new FfmpegRunResult(0, string.Empty); } - public Task> ExtractTextSubtitlesAsync( + public Task ExtractTextSubtitleAsync( Stream source, - TranscodingPlan plan, + MediaStreamProbe subtitle, + int ordinal, string outputDirectory, CancellationToken cancellationToken) - => Task.FromResult>([]); + => Task.FromResult(null); } private sealed class FailingRunner : IFfmpegProcessRunner @@ -289,8 +385,8 @@ public Task ProbeAsync(Stream source, CancellationToken cancellation "matroska", TimeSpan.FromSeconds(30), [ - new MediaStreamProbe(0, "video", "h264", null, null, true, false, false), - new MediaStreamProbe(1, "audio", "aac", null, null, true, false, false) + new MediaStreamProbe(0, "video", "h264", null, null, true, false, false, "High", "yuv420p"), + new MediaStreamProbe(1, "audio", "aac", null, null, true, false, false, null, null) ])); public async Task GenerateHlsAsync( @@ -309,12 +405,95 @@ await File.WriteAllTextAsync( return new FfmpegRunResult(1, "fixture FFmpeg failure"); } - public Task> ExtractTextSubtitlesAsync( + public Task ExtractTextSubtitleAsync( + Stream source, + MediaStreamProbe subtitle, + int ordinal, + string outputDirectory, + CancellationToken cancellationToken) + => Task.FromResult(null); + } + + private sealed class ProgressiveSubtitleRunner(bool failFirstSubtitle) : IFfmpegProcessRunner + { + private readonly object _gate = new(); + private readonly List _extractedStreamIndices = []; + + public TaskCompletionSource GenerationStarted { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource ReleaseGeneration { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource ExtractionCompleted { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + public IReadOnlyList ExtractedStreamIndices + { + get { lock (_gate) return _extractedStreamIndices.ToArray(); } + } + + public Task ProbeAsync(Stream source, CancellationToken cancellationToken) + { + var streams = new List + { + new(0, "video", "h264", null, null, true, false, false, "High", "yuv420p"), + new(1, "audio", "aac", null, null, true, false, false, null, null), + new(2, "subtitle", "ass", "eng", failFirstSubtitle ? "Broken" : "English", false, false, false, null, null) + }; + if (failFirstSubtitle) + streams.Add(new MediaStreamProbe( + 3, + "subtitle", + "subrip", + "jpn", + "Japanese", + true, + false, + false, + null, + null)); + return Task.FromResult(new MediaProbe("matroska", TimeSpan.FromSeconds(30), streams)); + } + + public async Task GenerateHlsAsync( Stream source, TranscodingPlan plan, + TranscodingSelection selection, string outputDirectory, + bool useHardwareEncoder, + Action onProgress, CancellationToken cancellationToken) - => Task.FromResult>([]); + { + await File.WriteAllBytesAsync( + Path.Combine(outputDirectory, "segment-000000.ts"), + [1, 2, 3], + cancellationToken); + await File.WriteAllTextAsync( + Path.Combine(outputDirectory, "media.m3u8"), + "#EXTM3U\n#EXTINF:6,\nsegment-000000.ts\n#EXT-X-ENDLIST\n", + cancellationToken); + onProgress(new FfmpegProgress(1, 1, true)); + GenerationStarted.TrySetResult(); + await ReleaseGeneration.Task.WaitAsync(cancellationToken); + return new FfmpegRunResult(0, string.Empty); + } + + public async Task ExtractTextSubtitleAsync( + Stream source, + MediaStreamProbe subtitle, + int ordinal, + string outputDirectory, + CancellationToken cancellationToken) + { + lock (_gate) _extractedStreamIndices.Add(subtitle.Index); + if (failFirstSubtitle && subtitle.Index == 2) return null; + + var name = $"subtitle-{subtitle.Index}.vtt"; + await File.WriteAllTextAsync( + Path.Combine(outputDirectory, name), + "WEBVTT\n", + cancellationToken); + if (subtitle.Index == 3 || !failFirstSubtitle) ExtractionCompleted.TrySetResult(); + return new TranscodingSubtitle(name, subtitle.Title ?? $"Subtitle {ordinal}", subtitle.Language, "vtt"); + } } private sealed class TranscodingFixture : IAsyncDisposable diff --git a/SecondDimensionWatcherReDive.Test/TranscodingPlannerTests.cs b/SecondDimensionWatcherReDive.Test/TranscodingPlannerTests.cs index a696c89..d14d145 100644 --- a/SecondDimensionWatcherReDive.Test/TranscodingPlannerTests.cs +++ b/SecondDimensionWatcherReDive.Test/TranscodingPlannerTests.cs @@ -33,6 +33,33 @@ public void CreatePlan_CompatibleTracksInMkv_UsesLosslessRemux() Assert.IsTrue(plan.CopyAudio); } + [TestMethod] + public void CreatePlan_Hi10pH264_TranscodesToBrowserCompatibleVideo() + { + var plan = TranscodingPlanner.CreatePlan( + CreateSource("episode.mkv"), + CreateProbe(Video("h264", "High 10", "yuv420p10le"), Audio("aac")), + TranscodingSelection.Create("auto", null, null, null, null), + burnBitmapSubtitles: false); + + Assert.AreEqual(TranscodingStrategy.Transcode, plan.Strategy); + Assert.IsFalse(plan.CopyVideo); + Assert.IsTrue(plan.CopyAudio); + } + + [TestMethod] + public void CreatePlan_H264WithoutCompatibilityMetadata_FailsClosedToTranscode() + { + var plan = TranscodingPlanner.CreatePlan( + CreateSource("episode.mp4"), + CreateProbe(Video("h264", null, null), Audio("aac")), + TranscodingSelection.Create("auto", null, null, null, null), + burnBitmapSubtitles: false); + + Assert.AreEqual(TranscodingStrategy.Transcode, plan.Strategy); + Assert.IsFalse(plan.CopyVideo); + } + [TestMethod] public void CreatePlan_UnsupportedCodecs_TranscodesAndSelectsPreferredAudio() { @@ -125,8 +152,11 @@ private static TranscodingSource CreateSource(string fileName) private static MediaProbe CreateProbe(params MediaStreamProbe[] streams) => new("matroska", TimeSpan.FromMinutes(24), streams); - private static MediaStreamProbe Video(string codec) - => new(0, "video", codec, null, null, true, false, false); + private static MediaStreamProbe Video( + string codec, + string? profile = "High", + string? pixelFormat = "yuv420p") + => new(0, "video", codec, null, null, true, false, false, profile, pixelFormat); private static MediaStreamProbe Audio( string codec, @@ -134,12 +164,12 @@ private static MediaStreamProbe Audio( string? language = null, string? title = null, bool isDefault = false) - => new(index, "audio", codec, language, title, isDefault, false, false); + => new(index, "audio", codec, language, title, isDefault, false, false, null, null); private static MediaStreamProbe Subtitle( string codec, int index, string? language, string? title) - => new(index, "subtitle", codec, language, title, false, false, false); + => new(index, "subtitle", codec, language, title, false, false, false, null, null); } diff --git a/SecondDimensionWatcherReDive/Services/Transcoding/FfmpegProcessRunner.cs b/SecondDimensionWatcherReDive/Services/Transcoding/FfmpegProcessRunner.cs index c5189a2..9d46544 100644 --- a/SecondDimensionWatcherReDive/Services/Transcoding/FfmpegProcessRunner.cs +++ b/SecondDimensionWatcherReDive/Services/Transcoding/FfmpegProcessRunner.cs @@ -24,9 +24,10 @@ Task GenerateHlsAsync( Action onProgress, CancellationToken cancellationToken); - Task> ExtractTextSubtitlesAsync( + Task ExtractTextSubtitleAsync( Stream source, - TranscodingPlan plan, + MediaStreamProbe subtitle, + int ordinal, string outputDirectory, CancellationToken cancellationToken); } @@ -89,7 +90,9 @@ public async Task ProbeAsync(Stream source, CancellationToken cancel stream.Tags?.Title, stream.Disposition?.Default == 1, stream.Disposition?.Forced == 1, - stream.Disposition?.AttachedPic == 1)) + stream.Disposition?.AttachedPic == 1, + stream.Profile, + stream.PixelFormat)) .ToArray(); var duration = ParseDuration(document.Format?.Duration) ?? (document.Streams ?? []).Select(stream => ParseDuration(stream.Duration)).FirstOrDefault(value => value is not null); @@ -185,32 +188,26 @@ public async Task GenerateHlsAsync( cancellationToken); } - public async Task> ExtractTextSubtitlesAsync( + public async Task ExtractTextSubtitleAsync( Stream source, - TranscodingPlan plan, + MediaStreamProbe subtitle, + int ordinal, string outputDirectory, CancellationToken cancellationToken) { - if (plan.TextSubtitles.Count == 0) return []; - + var finalPath = Path.Combine(outputDirectory, $"subtitle-{subtitle.Index}.vtt"); + var temporaryPath = $"{finalPath}.tmp"; + TryDelete(temporaryPath); var startInfo = CreateStartInfo(_options.FfmpegPath, redirectOutput: false); AddArguments(startInfo, "-hide_banner", "-y", "-i", "pipe:0", "-threads", _options.MaxThreadsPerJob.ToString(CultureInfo.InvariantCulture), - "-nostats"); - var pending = new List<(MediaStreamProbe Stream, string TemporaryPath, string FinalPath)>(); - foreach (var stream in plan.TextSubtitles) - { - var finalPath = Path.Combine(outputDirectory, $"subtitle-{stream.Index}.vtt"); - var temporaryPath = $"{finalPath}.tmp"; - pending.Add((stream, temporaryPath, finalPath)); - AddArguments(startInfo, - "-map", $"0:{stream.Index}", - "-c:s", "webvtt", - "-f", "webvtt", - temporaryPath); - } + "-nostats", + "-map", $"0:{subtitle.Index}", + "-c:s", "webvtt", + "-f", "webvtt", + temporaryPath); var result = await RunFfmpegAsync( startInfo, source, @@ -220,23 +217,18 @@ public async Task> ExtractTextSubtitlesAsync( detectFirstSegment: false); if (result.ExitCode != 0) { - LogSubtitleExtractionFailed(logger, result.ExitCode, result.ErrorOutput); - foreach (var item in pending) TryDelete(item.TemporaryPath); - return []; + LogSubtitleExtractionFailed(logger, subtitle.Index, result.ExitCode, result.ErrorOutput); + TryDelete(temporaryPath); + return null; } - var subtitles = new List(); - foreach (var item in pending) - { - if (!File.Exists(item.TemporaryPath)) continue; - File.Move(item.TemporaryPath, item.FinalPath, overwrite: true); - subtitles.Add(new TranscodingSubtitle( - Path.GetFileName(item.FinalPath), - BuildSubtitleLabel(item.Stream, subtitles.Count + 1), - item.Stream.Language, - "vtt")); - } - return subtitles; + if (!File.Exists(temporaryPath)) return null; + File.Move(temporaryPath, finalPath, overwrite: true); + return new TranscodingSubtitle( + Path.GetFileName(finalPath), + BuildSubtitleLabel(subtitle, ordinal), + subtitle.Language, + "vtt"); } private async Task RunFfmpegAsync( @@ -515,8 +507,12 @@ private static void TryDelete(string path) catch (IOException) { } } - [LoggerMessage(Level = LogLevel.Warning, Message = "FFmpeg subtitle extraction exited with code {ExitCode}: {Error}")] - private static partial void LogSubtitleExtractionFailed(ILogger logger, int exitCode, string error); + [LoggerMessage(Level = LogLevel.Warning, Message = "FFmpeg subtitle stream {StreamIndex} extraction exited with code {ExitCode}: {Error}")] + private static partial void LogSubtitleExtractionFailed( + ILogger logger, + int streamIndex, + int exitCode, + string error); private sealed record FfprobeDocument( [property: JsonPropertyName("streams")] FfprobeStream[]? Streams, @@ -526,6 +522,8 @@ private sealed record FfprobeStream( [property: JsonPropertyName("index")] int? Index, [property: JsonPropertyName("codec_name")] string? CodecName, [property: JsonPropertyName("codec_type")] string? CodecType, + [property: JsonPropertyName("profile")] string? Profile, + [property: JsonPropertyName("pix_fmt")] string? PixelFormat, [property: JsonPropertyName("duration")] string? Duration, [property: JsonPropertyName("disposition")] FfprobeDisposition? Disposition, [property: JsonPropertyName("tags")] FfprobeTags? Tags); diff --git a/SecondDimensionWatcherReDive/Services/Transcoding/HlsTranscodingService.cs b/SecondDimensionWatcherReDive/Services/Transcoding/HlsTranscodingService.cs index dcf6537..6837408 100644 --- a/SecondDimensionWatcherReDive/Services/Transcoding/HlsTranscodingService.cs +++ b/SecondDimensionWatcherReDive/Services/Transcoding/HlsTranscodingService.cs @@ -13,7 +13,7 @@ namespace SecondDimensionWatcherReDive.Services.Transcoding; internal sealed partial class HlsTranscodingService : BackgroundService, IHlsTranscodingService { - private const int CacheManifestVersion = 1; + private const int CacheManifestVersion = 2; private const string CacheOwnershipMarker = ".sdw-transcode-cache"; private readonly IServiceScopeFactory _scopeFactory; private readonly IFfmpegProcessRunner _processRunner; @@ -305,6 +305,9 @@ private async Task ProcessJobAsync(TranscodingJob job, CancellationToken stoppin RecreateJobDirectory(job.CacheDirectory); job.SetState(TranscodingJobState.Transcoding); UpdateJobGauges(); + using var subtitleCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var subtitleTask = ExtractTextSubtitlesAsync(job, plan, subtitleCancellation.Token); + var subtitleTaskObserved = false; var firstSegmentRecorded = 0; void OnProgress(FfmpegProgress update) { @@ -318,49 +321,59 @@ void OnProgress(FfmpegProgress update) } } - var useHardware = !plan.CopyVideo && !string.IsNullOrWhiteSpace(_options.HardwareVideoEncoder); - FfmpegRunResult result; - await using (var source = await OpenSourceStreamAsync(job.Source, cancellationToken)) - result = await _processRunner.GenerateHlsAsync( - source, - plan, - job.Selection, - job.CacheDirectory, - useHardware, - OnProgress, - cancellationToken); - if (result.ExitCode != 0 && useHardware) + try { - LogHardwareFallback(_logger, _options.HardwareVideoEncoder!, result.ErrorOutput); - DeleteGeneratedFiles(job.CacheDirectory); - await using var source = await OpenSourceStreamAsync(job.Source, cancellationToken); - result = await _processRunner.GenerateHlsAsync( - source, - plan, - job.Selection, - job.CacheDirectory, - useHardwareEncoder: false, - OnProgress, - cancellationToken); - } - if (result.ExitCode != 0) - throw new InvalidOperationException($"FFmpeg exited with code {result.ExitCode}: {result.ErrorOutput}"); + var useHardware = !plan.CopyVideo && !string.IsNullOrWhiteSpace(_options.HardwareVideoEncoder); + FfmpegRunResult result; + await using (var source = await OpenSourceStreamAsync(job.Source, cancellationToken)) + result = await _processRunner.GenerateHlsAsync( + source, + plan, + job.Selection, + job.CacheDirectory, + useHardware, + OnProgress, + cancellationToken); + if (result.ExitCode != 0 && useHardware) + { + LogHardwareFallback(_logger, _options.HardwareVideoEncoder!, result.ErrorOutput); + DeleteGeneratedHlsFiles(job.CacheDirectory); + await using var source = await OpenSourceStreamAsync(job.Source, cancellationToken); + result = await _processRunner.GenerateHlsAsync( + source, + plan, + job.Selection, + job.CacheDirectory, + useHardwareEncoder: false, + OnProgress, + cancellationToken); + } + if (result.ExitCode != 0) + throw new InvalidOperationException($"FFmpeg exited with code {result.ExitCode}: {result.ErrorOutput}"); - IReadOnlyList subtitles; - await using (var source = await OpenSourceStreamAsync(job.Source, cancellationToken)) - subtitles = await _processRunner.ExtractTextSubtitlesAsync( - source, - plan, - job.CacheDirectory, - cancellationToken); - job.SetSubtitles(subtitles); - await WriteManifestAsync(job, cancellationToken); - _metrics.RecordCompleted(); - if (job.GetSpeed() is { } speed) _metrics.RecordSpeed(speed); - UpdateCacheBytes(); - job.SetReady(subtitles); - UpdateJobGauges(); - await CleanupCacheAsync(removeIncomplete: false, cancellationToken); + var subtitles = await subtitleTask; + subtitleTaskObserved = true; + await WriteManifestAsync(job, cancellationToken); + _metrics.RecordCompleted(); + if (job.GetSpeed() is { } speed) _metrics.RecordSpeed(speed); + UpdateCacheBytes(); + job.SetReady(subtitles); + UpdateJobGauges(); + await CleanupCacheAsync(removeIncomplete: false, cancellationToken); + } + finally + { + if (!subtitleTaskObserved) + { + try + { + await subtitleCancellation.CancelAsync(); + await subtitleTask; + } + catch (OperationCanceledException) when (subtitleCancellation.IsCancellationRequested) { } + catch (Exception exception) { LogSubtitleCleanupFailed(_logger, exception); } + } + } } catch (OperationCanceledException) when (job.Cancellation.IsCancellationRequested) { @@ -441,6 +454,32 @@ private async Task OpenSourceStreamAsync( } } + private async Task> ExtractTextSubtitlesAsync( + TranscodingJob job, + TranscodingPlan plan, + CancellationToken cancellationToken) + { + var subtitles = new List(); + for (var index = 0; index < plan.TextSubtitles.Count; index++) + { + cancellationToken.ThrowIfCancellationRequested(); + var track = plan.TextSubtitles[index]; + TranscodingSubtitle? subtitle; + await using (var source = await OpenSourceStreamAsync(job.Source, cancellationToken)) + subtitle = await _processRunner.ExtractTextSubtitleAsync( + source, + track, + index + 1, + job.CacheDirectory, + cancellationToken); + if (subtitle is null) continue; + + subtitles.Add(subtitle); + job.SetSubtitles(subtitles.ToArray()); + } + return subtitles; + } + private TranscodingSession? FindSession(Guid id, string token, bool touch = true) { if (!_sessions.TryGetValue(id, out var session) || !TokensEqual(session.AccessToken, token)) @@ -506,6 +545,19 @@ private async Task WriteManifestAsync(TranscodingJob job, CancellationToken canc } private async Task CleanupCacheAsync(bool removeIncomplete, CancellationToken cancellationToken) + { + await _creationGate.WaitAsync(cancellationToken); + try + { + CleanupCacheCore(removeIncomplete, cancellationToken); + } + finally + { + _creationGate.Release(); + } + } + + private void CleanupCacheCore(bool removeIncomplete, CancellationToken cancellationToken) { CleanupExpiredSessions(); if (!Directory.Exists(_options.CachePath)) return; @@ -549,7 +601,6 @@ private async Task CleanupCacheAsync(bool removeIncomplete, CancellationToken ca total -= candidate.Size; } _metrics.SetCacheBytes(Math.Max(0, total)); - await Task.CompletedTask; } private void CleanupExpiredSessions() @@ -677,12 +728,16 @@ private static void RecreateJobDirectory(string path) File.WriteAllText(Path.Combine(path, CacheOwnershipMarker), string.Empty); } - private static void DeleteGeneratedFiles(string directory) + private static void DeleteGeneratedHlsFiles(string directory) { foreach (var path in Directory.EnumerateFiles(directory)) - if (Path.GetFileName(path) is not (".access" or CacheOwnershipMarker)) + { + var fileName = Path.GetFileName(path); + if (fileName is "media.m3u8" or "media.m3u8.tmp" + || fileName.StartsWith("segment-", StringComparison.Ordinal)) try { File.Delete(path); } catch (IOException) { } + } } private static void TryDeleteDirectory(string path) @@ -741,6 +796,9 @@ private static bool IsCacheKey(string name) [LoggerMessage(Level = LogLevel.Warning, Message = "Ignoring invalid transcoding cache manifest {Path}")] private static partial void LogInvalidCacheManifest(ILogger logger, string path, Exception exception); + [LoggerMessage(Level = LogLevel.Warning, Message = "Failed while stopping background subtitle extraction")] + private static partial void LogSubtitleCleanupFailed(ILogger logger, Exception exception); + private sealed record CacheDirectory(string Key, string Path, DateTimeOffset LastAccess, long Size); private sealed record CacheManifest( @@ -828,10 +886,30 @@ public static TranscodingJob FromManifest( _progress = 1, _subtitles = manifest.Subtitles }; - var video = new MediaStreamProbe(0, "video", manifest.VideoCodec, null, null, true, false, false); + var video = new MediaStreamProbe( + 0, + "video", + manifest.VideoCodec, + null, + null, + true, + false, + false, + null, + null); var audio = manifest.AudioCodec is null ? null - : new MediaStreamProbe(1, "audio", manifest.AudioCodec, null, null, true, false, false); + : new MediaStreamProbe( + 1, + "audio", + manifest.AudioCodec, + null, + null, + true, + false, + false, + null, + null); job._plan = new TranscodingPlan( manifest.Strategy, video, diff --git a/SecondDimensionWatcherReDive/Services/Transcoding/TranscodingModels.cs b/SecondDimensionWatcherReDive/Services/Transcoding/TranscodingModels.cs index 860130d..ca0e599 100644 --- a/SecondDimensionWatcherReDive/Services/Transcoding/TranscodingModels.cs +++ b/SecondDimensionWatcherReDive/Services/Transcoding/TranscodingModels.cs @@ -89,7 +89,9 @@ internal sealed record MediaStreamProbe( string? Title, bool IsDefault, bool IsForced, - bool IsAttachedPicture); + bool IsAttachedPicture, + string? Profile, + string? PixelFormat); internal sealed record MediaProbe( string Container, diff --git a/SecondDimensionWatcherReDive/Services/Transcoding/TranscodingPlanner.cs b/SecondDimensionWatcherReDive/Services/Transcoding/TranscodingPlanner.cs index ef4a2db..9d28b58 100644 --- a/SecondDimensionWatcherReDive/Services/Transcoding/TranscodingPlanner.cs +++ b/SecondDimensionWatcherReDive/Services/Transcoding/TranscodingPlanner.cs @@ -40,11 +40,11 @@ public static TranscodingPlan CreatePlan( : null; var extension = Path.GetExtension(source.FileName); - var copyVideo = video.CodecName.Equals("h264", StringComparison.OrdinalIgnoreCase) + var copyVideo = IsBrowserCompatibleH264(video) && selection.Quality == "auto" && bitmapToBurn is null; var copyAudio = audio is null || HlsAudioCodecs.Contains(audio.CodecName); - var direct = IsDirectPlayContainer(extension, video.CodecName, audio?.CodecName) + var direct = IsDirectPlayContainer(extension, video, audio?.CodecName) && selection.Quality == "auto" && bitmapToBurn is null; var strategy = direct @@ -107,20 +107,40 @@ private static string NormalizeLanguage(string language) return separator < 0 ? normalized : normalized[..separator]; } - private static bool IsDirectPlayContainer(string extension, string videoCodec, string? audioCodec) + private static bool IsDirectPlayContainer( + string extension, + MediaStreamProbe video, + string? audioCodec) { if (extension.Equals(".mp4", StringComparison.OrdinalIgnoreCase) || extension.Equals(".m4v", StringComparison.OrdinalIgnoreCase)) - return videoCodec.Equals("h264", StringComparison.OrdinalIgnoreCase) + return IsBrowserCompatibleH264(video) && (audioCodec is null || audioCodec.Equals("aac", StringComparison.OrdinalIgnoreCase)); if (!extension.Equals(".webm", StringComparison.OrdinalIgnoreCase)) return false; - var supportedVideo = videoCodec.Equals("vp8", StringComparison.OrdinalIgnoreCase) - || videoCodec.Equals("vp9", StringComparison.OrdinalIgnoreCase) - || videoCodec.Equals("av1", StringComparison.OrdinalIgnoreCase); + var supportedVideo = video.CodecName.Equals("vp8", StringComparison.OrdinalIgnoreCase) + || video.CodecName.Equals("vp9", StringComparison.OrdinalIgnoreCase) + || video.CodecName.Equals("av1", StringComparison.OrdinalIgnoreCase); var supportedAudio = audioCodec is null || audioCodec.Equals("opus", StringComparison.OrdinalIgnoreCase) || audioCodec.Equals("vorbis", StringComparison.OrdinalIgnoreCase); return supportedVideo && supportedAudio; } + + private static bool IsBrowserCompatibleH264(MediaStreamProbe video) + { + if (!video.CodecName.Equals("h264", StringComparison.OrdinalIgnoreCase)) return false; + + var profile = video.Profile?.Trim(); + var compatibleProfile = profile is not null + && (profile.Equals("Baseline", StringComparison.OrdinalIgnoreCase) + || profile.Equals("Constrained Baseline", StringComparison.OrdinalIgnoreCase) + || profile.Equals("Main", StringComparison.OrdinalIgnoreCase) + || profile.Equals("High", StringComparison.OrdinalIgnoreCase)); + var pixelFormat = video.PixelFormat?.Trim(); + var compatiblePixelFormat = pixelFormat is not null + && (pixelFormat.Equals("yuv420p", StringComparison.OrdinalIgnoreCase) + || pixelFormat.Equals("yuvj420p", StringComparison.OrdinalIgnoreCase)); + return compatibleProfile && compatiblePixelFormat; + } } From f32939d62f5c8267273f05000448aeba6ba5ebf5 Mon Sep 17 00:00:00 2001 From: mahoshojoHCG Date: Sun, 30 Aug 2026 11:15:29 +0800 Subject: [PATCH 13/45] fix: harden streaming and distributed task state --- .../ChatController.cs | 22 +-- .../Tools/ManageTasksTool.cs | 25 +++- .../IScheduledTaskLeaseRepository.cs | 4 + .../DataRepository/ScheduledTaskLeaseState.cs | 8 ++ .../Tasks/IScheduledTaskLeaseManager.cs | 8 ++ .../Tasks/ScheduledTaskBase.cs | 94 +++++++++---- .../FileMappingRepositoryPostgreSqlTests.cs | 37 +++++ .../ChatControllerStreamingTests.cs | 128 ++++++++++++++++++ .../ManageTasksToolTests.cs | 42 ++++++ .../PostgresScheduledTaskLeaseManagerTests.cs | 70 ++++++++++ .../ScheduledTaskBaseTests.cs | 64 +++++++++ .../TasksControllerTests.cs | 39 ++++++ .../Controllers/TasksController.cs | 27 ++-- ...eMappingRepositoryPostgreSqlTestFixture.cs | 10 ++ .../ScheduledTaskLeaseRepository.cs | 19 +++ .../PostgresScheduledTaskLeaseManager.cs | 35 +++++ 16 files changed, 581 insertions(+), 51 deletions(-) create mode 100644 SecondDimensionWatcherReDive.Framework/DataRepository/ScheduledTaskLeaseState.cs create mode 100644 SecondDimensionWatcherReDive.Test/ChatControllerStreamingTests.cs create mode 100644 SecondDimensionWatcherReDive.Test/ManageTasksToolTests.cs create mode 100644 SecondDimensionWatcherReDive.Test/PostgresScheduledTaskLeaseManagerTests.cs create mode 100644 SecondDimensionWatcherReDive.Test/TasksControllerTests.cs diff --git a/Plugins/SecondDimensionWatcherReDive.Chat/ChatController.cs b/Plugins/SecondDimensionWatcherReDive.Chat/ChatController.cs index 7420380..48ef392 100644 --- a/Plugins/SecondDimensionWatcherReDive.Chat/ChatController.cs +++ b/Plugins/SecondDimensionWatcherReDive.Chat/ChatController.cs @@ -179,7 +179,7 @@ public async Task SendMessage( cancellationToken)); } - private async IAsyncEnumerable> StreamChatEvents( + internal async IAsyncEnumerable> StreamChatEvents( IAIEngine aiEngine, List messages, ChatOptions chatOptions, @@ -197,6 +197,8 @@ private async IAsyncEnumerable> StreamChatEvents( SingleWriter = true, FullMode = BoundedChannelFullMode.Wait }); + using var producerCancellation = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken); // Producer: runs AI chat streaming in background, writes SSE items to channel. // Keep the task and await it during iterator disposal so a disconnected request cannot @@ -204,7 +206,7 @@ private async IAsyncEnumerable> StreamChatEvents( var producer = ProduceChatEventsAsync( aiEngine, messages, chatOptions, conversationId, messageOrder, firstUserMessage, autoTitleEligible, model, - channel.Writer, cancellationToken); + channel.Writer, producerCancellation.Token); try { @@ -214,13 +216,15 @@ private async IAsyncEnumerable> StreamChatEvents( } finally { - // Do not use the canceled request token here. The producer receives it directly, - // performs bounded engine cleanup, and persists accumulated messages before returning. + // The reader can be disposed independently of RequestAborted (for example when + // response-body I/O fails). Explicitly stop the producer, then let it persist + // accumulated messages before this request scope is released. + await producerCancellation.CancelAsync(); await producer; } } - private async Task ProduceChatEventsAsync( + internal async Task ProduceChatEventsAsync( IAIEngine aiEngine, List messages, ChatOptions chatOptions, @@ -334,12 +338,14 @@ await writer.WriteAsync( catch (Exception ex) { LogStreamingError(ex, conversationId); - await writer.WriteAsync( + // A disconnected/failed reader may leave this bounded channel full. + // Terminal error delivery is best-effort so persistence and request + // scope disposal can never wait forever for a reader that is gone. + writer.TryWrite( new SseItem( JsonSerializer.Serialize(new SseError(ex.Message), ChatJsonSerializerContext.Default.SseError), - "error"), - CancellationToken.None); + "error")); } // Save all accumulated segments to DB diff --git a/Plugins/SecondDimensionWatcherReDive.Chat/Tools/ManageTasksTool.cs b/Plugins/SecondDimensionWatcherReDive.Chat/Tools/ManageTasksTool.cs index ae9875f..5fc6330 100644 --- a/Plugins/SecondDimensionWatcherReDive.Chat/Tools/ManageTasksTool.cs +++ b/Plugins/SecondDimensionWatcherReDive.Chat/Tools/ManageTasksTool.cs @@ -9,9 +9,10 @@ namespace SecondDimensionWatcherReDive.Chat.Tools; "manage_tasks", "Manage background scheduled tasks. List all task statuses or manually trigger a specific task to run.")] internal sealed partial class ManageTasksTool( - IEnumerable scheduledTasks) : ITool + IEnumerable scheduledTasks, + IScheduledTaskLeaseManager leaseManager) : ITool { - private Task ExecuteCoreAsync( + private async Task ExecuteCoreAsync( ManageTasksParams param, CancellationToken cancellationToken) { var taskList = scheduledTasks.ToList(); @@ -20,10 +21,24 @@ private Task ExecuteCoreAsync( switch (param.Action) { case ManageTasksAction.List: + { + var statuses = await leaseManager.GetStatusesAsync( + taskList.Select(task => task.Id).ToArray(), + cancellationToken); result = new ToolSuccessResult(new TaskListResult( - taskList.Select(t => new TaskSummary( - t.Id, t.Interval.ToString(), t.IsEnabled, t.LastRunAt, t.IsRunning)))); + taskList.Select(task => + { + var status = statuses.GetValueOrDefault(task.Id) + ?? new ScheduledTaskStatus(null, false); + return new TaskSummary( + task.Id, + task.Interval.ToString(), + task.IsEnabled, + status.LastRunAt, + status.IsRunning); + }))); break; + } case ManageTasksAction.Run: { @@ -52,7 +67,7 @@ private Task ExecuteCoreAsync( break; } - return Task.FromResult(result); + return result; } } diff --git a/SecondDimensionWatcherReDive.Framework/DataRepository/IScheduledTaskLeaseRepository.cs b/SecondDimensionWatcherReDive.Framework/DataRepository/IScheduledTaskLeaseRepository.cs index 34b6424..684afa3 100644 --- a/SecondDimensionWatcherReDive.Framework/DataRepository/IScheduledTaskLeaseRepository.cs +++ b/SecondDimensionWatcherReDive.Framework/DataRepository/IScheduledTaskLeaseRepository.cs @@ -25,4 +25,8 @@ Task CompleteAsync( bool succeeded, string? error, CancellationToken cancellationToken); + + Task> GetStatesAsync( + IReadOnlyCollection taskIds, + CancellationToken cancellationToken); } diff --git a/SecondDimensionWatcherReDive.Framework/DataRepository/ScheduledTaskLeaseState.cs b/SecondDimensionWatcherReDive.Framework/DataRepository/ScheduledTaskLeaseState.cs new file mode 100644 index 0000000..c2d9f65 --- /dev/null +++ b/SecondDimensionWatcherReDive.Framework/DataRepository/ScheduledTaskLeaseState.cs @@ -0,0 +1,8 @@ +namespace SecondDimensionWatcherReDive.Framework.DataRepository; + +public sealed record ScheduledTaskLeaseState( + string TaskId, + string? LeaseOwner, + DateTimeOffset? LeaseExpiresAt, + DateTimeOffset? LastStartedAt, + DateTimeOffset? LastCompletedAt); diff --git a/SecondDimensionWatcherReDive.Framework/Tasks/IScheduledTaskLeaseManager.cs b/SecondDimensionWatcherReDive.Framework/Tasks/IScheduledTaskLeaseManager.cs index 2a0f1bd..eede259 100644 --- a/SecondDimensionWatcherReDive.Framework/Tasks/IScheduledTaskLeaseManager.cs +++ b/SecondDimensionWatcherReDive.Framework/Tasks/IScheduledTaskLeaseManager.cs @@ -3,6 +3,10 @@ namespace SecondDimensionWatcherReDive.Framework.Tasks; public sealed class ScheduledTaskLeaseUnavailableException(Exception innerException) : Exception("The scheduled-task lease store is unavailable.", innerException); +public sealed record ScheduledTaskStatus( + DateTimeOffset? LastRunAt, + bool IsRunning); + public interface IScheduledTaskExecutionLease : IAsyncDisposable { CancellationToken LeaseLostToken { get; } @@ -20,4 +24,8 @@ public interface IScheduledTaskLeaseManager TimeSpan interval, bool force, CancellationToken cancellationToken); + + Task> GetStatusesAsync( + IReadOnlyCollection taskIds, + CancellationToken cancellationToken); } diff --git a/SecondDimensionWatcherReDive.Framework/Tasks/ScheduledTaskBase.cs b/SecondDimensionWatcherReDive.Framework/Tasks/ScheduledTaskBase.cs index fa025c7..e8945e9 100644 --- a/SecondDimensionWatcherReDive.Framework/Tasks/ScheduledTaskBase.cs +++ b/SecondDimensionWatcherReDive.Framework/Tasks/ScheduledTaskBase.cs @@ -56,45 +56,49 @@ public async Task ProcessQueueAsync( await foreach (var _ in _runQueue.Reader.ReadAllAsync(cancellationToken)) { TaskCompletionSource? completion; - bool force; lock (_sync) { completion = _pendingRun; - force = _pendingForce; } if (completion is null) continue; - IScheduledTaskExecutionLease? lease; - try + IScheduledTaskExecutionLease? lease = null; + while (lease is null) { - lease = await leaseManager.TryAcquireAsync( - Id, - Interval, - force, - cancellationToken); - } - catch (OperationCanceledException exception) when (cancellationToken.IsCancellationRequested) - { - completion.TrySetCanceled(exception.CancellationToken); - FinishRun(completion); - throw; - } - catch (Exception exception) - { - completion.TrySetException( - new ScheduledTaskLeaseUnavailableException(exception)); - FinishRun(completion); - continue; - } + var force = TakePendingForce(completion); + try + { + lease = await leaseManager.TryAcquireAsync( + Id, + Interval, + force, + cancellationToken); + } + catch (OperationCanceledException exception) when (cancellationToken.IsCancellationRequested) + { + completion.TrySetCanceled(exception.CancellationToken); + FinishRun(completion); + throw; + } + catch (Exception exception) + { + completion.TrySetException( + new ScheduledTaskLeaseUnavailableException(exception)); + FinishRun(completion); + break; + } - if (lease is null) - { - // Another instance owns the same periodic task. Its local timer - // will drive the execution; this duplicate signal is complete. - completion.TrySetResult(false); - FinishRun(completion); - continue; + if (lease is not null) + break; + + // A manual request can upgrade a periodic acquisition while the + // database call is in flight. Retry that upgrade before completing + // the shared signal so a completed cooldown cannot swallow it. + if (CompleteLeaseDenialOrRetryForce(completion, force)) + continue; + break; } + if (lease is null) continue; await using (lease) { @@ -167,6 +171,36 @@ private Task QueueRun(bool force) } } + private bool TakePendingForce(TaskCompletionSource completion) + { + lock (_sync) + { + if (!ReferenceEquals(_pendingRun, completion)) + return false; + var force = _pendingForce; + _pendingForce = false; + return force; + } + } + + private bool CompleteLeaseDenialOrRetryForce( + TaskCompletionSource completion, + bool attemptedForce) + { + lock (_sync) + { + if (!ReferenceEquals(_pendingRun, completion)) + return false; + if (!attemptedForce && _pendingForce) + return true; + + _pendingRun = null; + _pendingForce = false; + completion.TrySetResult(false); + return false; + } + } + private void FinishRun(TaskCompletionSource completion) { lock (_sync) diff --git a/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/FileMappingRepositoryPostgreSqlTests.cs b/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/FileMappingRepositoryPostgreSqlTests.cs index 3e53a88..2611336 100644 --- a/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/FileMappingRepositoryPostgreSqlTests.cs +++ b/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/FileMappingRepositoryPostgreSqlTests.cs @@ -240,6 +240,43 @@ await Fixture.CompleteTaskLeaseAsync( CancellationToken.None)); } + [TestMethod] + public async Task ScheduledTaskLease_StatusProjectionReadsSharedState() + { + var now = DateTimeOffset.FromUnixTimeMilliseconds( + DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); + var leaseUntil = now.AddSeconds(30); + Assert.IsTrue(await Fixture.TryAcquireTaskLeaseAsync( + "StatusTask", + "instance-a", + now, + leaseUntil, + false, + CancellationToken.None)); + + var running = await Fixture.GetTaskLeaseStatesAsync( + ["StatusTask", "missing"], + CancellationToken.None); + + Assert.HasCount(1, running); + Assert.AreEqual("instance-a", running[0].LeaseOwner); + Assert.AreEqual(now, running[0].LastStartedAt); + Assert.AreEqual(leaseUntil, running[0].LeaseExpiresAt); + Assert.IsNull(running[0].LastCompletedAt); + + var completedAt = now.AddSeconds(5); + await Fixture.CompleteTaskLeaseAsync( + "StatusTask", + "instance-a", + completedAt, + now.AddMinutes(10), + CancellationToken.None); + var completed = await Fixture.GetTaskLeaseStatesAsync( + ["StatusTask"], + CancellationToken.None); + Assert.AreEqual(completedAt, completed.Single().LastCompletedAt); + } + [TestMethod] public async Task DeadLetterJobs_CanBeRetriedOrMarkedHandled() { diff --git a/SecondDimensionWatcherReDive.Test/ChatControllerStreamingTests.cs b/SecondDimensionWatcherReDive.Test/ChatControllerStreamingTests.cs new file mode 100644 index 0000000..a5def04 --- /dev/null +++ b/SecondDimensionWatcherReDive.Test/ChatControllerStreamingTests.cs @@ -0,0 +1,128 @@ +using System.Net.ServerSentEvents; +using System.Runtime.CompilerServices; +using System.Threading.Channels; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using SecondDimensionWatcherReDive.AI.Abstractions; +using SecondDimensionWatcherReDive.AI.Models; +using SecondDimensionWatcherReDive.Chat; +using SecondDimensionWatcherReDive.Framework.DataRepository; + +namespace SecondDimensionWatcherReDive.Test; + +[TestClass] +public sealed class ChatControllerStreamingTests +{ + [TestMethod] + public async Task ProduceChatEventsAsync_ErrorWithFullChannel_DoesNotBlock() + { + var channel = Channel.CreateBounded>(1); + Assert.IsTrue(channel.Writer.TryWrite(new SseItem("buffered", "text_delta"))); + var controller = CreateController(new Mock(MockBehavior.Strict).Object); + + await controller.ProduceChatEventsAsync( + new ThrowingEngine(), + [], + new ChatOptions(), + Guid.NewGuid(), + 0, + "question", + false, + null, + channel.Writer, + CancellationToken.None) + .WaitAsync(TimeSpan.FromSeconds(2)); + + Assert.IsTrue(channel.Reader.TryRead(out var buffered)); + Assert.AreEqual("buffered", buffered.Data); + await channel.Reader.Completion.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.IsFalse(channel.Reader.TryRead(out _)); + } + + [TestMethod] + public async Task StreamChatEvents_EarlyReaderExit_CancelsAndJoinsProducer() + { + var conversationId = Guid.NewGuid(); + var repository = new Mock(MockBehavior.Strict); + repository.Setup(candidate => candidate.AddMessagesAsync( + conversationId, + It.Is>(messages => + messages.Any(message => message.Role == "assistant" + && message.Content == "partial")), + CancellationToken.None)) + .Returns(Task.CompletedTask); + var engine = new CancellationAwareEngine(); + var controller = CreateController(repository.Object); + var enumerator = controller.StreamChatEvents( + engine, + [], + new ChatOptions(), + conversationId, + 0, + "question", + false, + null, + CancellationToken.None) + .GetAsyncEnumerator(); + + Assert.IsTrue(await enumerator.MoveNextAsync()); + await enumerator.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(2)); + + await engine.CancellationObserved.Task.WaitAsync(TimeSpan.FromSeconds(2)); + repository.VerifyAll(); + } + + private static ChatController CreateController(IChatRepository repository) => + new( + repository, + Mock.Of(), + Mock.Of(), + NullLogger.Instance); + + private sealed class ThrowingEngine : IAIEngine + { + public Task> GetAvailableModelsAsync( + CancellationToken cancellationToken) => + Task.FromResult>([]); + + public async IAsyncEnumerable ChatAsync( + IReadOnlyList messages, + ChatOptions? options, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + await Task.Yield(); + if (!cancellationToken.IsCancellationRequested) + throw new InvalidOperationException("provider failed"); + cancellationToken.ThrowIfCancellationRequested(); + yield break; + } + } + + private sealed class CancellationAwareEngine : IAIEngine + { + public TaskCompletionSource CancellationObserved { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + + public Task> GetAvailableModelsAsync( + CancellationToken cancellationToken) => + Task.FromResult>([]); + + public async IAsyncEnumerable ChatAsync( + IReadOnlyList messages, + ChatOptions? options, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + yield return new TextDelta("partial"); + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + CancellationObserved.TrySetResult(); + throw; + } + } + } +} diff --git a/SecondDimensionWatcherReDive.Test/ManageTasksToolTests.cs b/SecondDimensionWatcherReDive.Test/ManageTasksToolTests.cs new file mode 100644 index 0000000..fc6702c --- /dev/null +++ b/SecondDimensionWatcherReDive.Test/ManageTasksToolTests.cs @@ -0,0 +1,42 @@ +using System.Text.Json; +using Moq; +using SecondDimensionWatcherReDive.AI.Models; +using SecondDimensionWatcherReDive.Chat.Tools; +using SecondDimensionWatcherReDive.Framework.Tasks; + +namespace SecondDimensionWatcherReDive.Test; + +[TestClass] +public sealed class ManageTasksToolTests +{ + [TestMethod] + public async Task List_UsesSharedLeaseStatuses() + { + var task = new Mock(MockBehavior.Strict); + task.SetupGet(candidate => candidate.Id).Returns("remote-task"); + task.SetupGet(candidate => candidate.Interval).Returns(TimeSpan.FromMinutes(10)); + task.SetupGet(candidate => candidate.IsEnabled).Returns(true); + var lastRunAt = DateTimeOffset.UtcNow.AddMinutes(-4); + var leaseManager = new Mock(MockBehavior.Strict); + leaseManager.Setup(candidate => candidate.GetStatusesAsync( + It.Is>(ids => ids.SequenceEqual(new[] { "remote-task" })), + CancellationToken.None)) + .ReturnsAsync(new Dictionary + { + ["remote-task"] = new(lastRunAt, true) + }); + var tool = new ManageTasksTool([task.Object], leaseManager.Object); + + var result = await tool.ExecuteAsync( + JsonSerializer.SerializeToElement( + new ManageTasksParams(ManageTasksAction.List), + ToolJsonOptions.Options), + CancellationToken.None); + + var success = result as ToolSuccessResult; + Assert.IsNotNull(success); + var status = success.Result.Tasks.Single(); + Assert.AreEqual(lastRunAt, status.LastRunAt); + Assert.IsTrue(status.IsRunning); + } +} diff --git a/SecondDimensionWatcherReDive.Test/PostgresScheduledTaskLeaseManagerTests.cs b/SecondDimensionWatcherReDive.Test/PostgresScheduledTaskLeaseManagerTests.cs new file mode 100644 index 0000000..b269f9f --- /dev/null +++ b/SecondDimensionWatcherReDive.Test/PostgresScheduledTaskLeaseManagerTests.cs @@ -0,0 +1,70 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Framework.Tasks; +using SecondDimensionWatcherReDive.Services; + +namespace SecondDimensionWatcherReDive.Test; + +[TestClass] +public sealed class PostgresScheduledTaskLeaseManagerTests +{ + [TestMethod] + public async Task GetStatusesAsync_DerivesCrossReplicaStateFromPersistedLease() + { + var now = DateTimeOffset.UtcNow; + var lastCompleted = now.AddMinutes(-2); + var repository = new Mock(MockBehavior.Strict); + repository.Setup(candidate => candidate.GetStatesAsync( + It.IsAny>(), + CancellationToken.None)) + .ReturnsAsync( + [ + new ScheduledTaskLeaseState( + "remote-running", + "instance-b", + now.AddMinutes(5), + now.AddMinutes(-1), + lastCompleted), + new ScheduledTaskLeaseState( + "cooldown", + "instance-b", + now.AddMinutes(5), + now.AddMinutes(-2), + now.AddMinutes(-1)), + new ScheduledTaskLeaseState( + "expired-incomplete", + "instance-b", + now.AddMinutes(-1), + now.AddMinutes(-2), + null), + new ScheduledTaskLeaseState( + "ownerless", + null, + now.AddMinutes(5), + now.AddMinutes(-1), + null) + ]); + using var services = new ServiceCollection() + .AddScoped(_ => repository.Object) + .BuildServiceProvider(); + var manager = new PostgresScheduledTaskLeaseManager( + services.GetRequiredService(), + NullLogger.Instance); + + var statuses = await manager.GetStatusesAsync( + ["remote-running", "cooldown", "expired-incomplete", "ownerless", "not-started"], + CancellationToken.None); + + Assert.IsTrue(statuses["remote-running"].IsRunning); + Assert.AreEqual(lastCompleted, statuses["remote-running"].LastRunAt); + Assert.IsFalse(statuses["cooldown"].IsRunning); + Assert.AreEqual(now.AddMinutes(-1), statuses["cooldown"].LastRunAt); + Assert.IsFalse(statuses["expired-incomplete"].IsRunning); + Assert.IsNull(statuses["expired-incomplete"].LastRunAt); + Assert.IsFalse(statuses["ownerless"].IsRunning); + Assert.IsFalse(statuses["not-started"].IsRunning); + Assert.IsNull(statuses["not-started"].LastRunAt); + } +} diff --git a/SecondDimensionWatcherReDive.Test/ScheduledTaskBaseTests.cs b/SecondDimensionWatcherReDive.Test/ScheduledTaskBaseTests.cs index 94b5105..9f4c715 100644 --- a/SecondDimensionWatcherReDive.Test/ScheduledTaskBaseTests.cs +++ b/SecondDimensionWatcherReDive.Test/ScheduledTaskBaseTests.cs @@ -63,6 +63,29 @@ public async Task RunScheduledAsync_ContentionReportsSkippedWithoutForcingCooldo await AssertCanceledAsync(processor); } + [TestMethod] + public async Task RunScheduledAsync_ForceArrivingDuringAcquisitionRetriesAsForced() + { + var task = new BlockingTask(); + var leaseManager = new ForceUpgradeLeaseManager(); + using var cancellation = new CancellationTokenSource(); + var processor = task.ProcessQueueAsync(leaseManager, cancellation.Token); + + var scheduled = task.RunScheduledAsync(CancellationToken.None); + await leaseManager.FirstAcquireStarted.Task.WaitAsync(TimeSpan.FromSeconds(2)); + task.Enqueue(); + leaseManager.ReleaseFirstAcquire.TrySetResult(); + await task.Started.Task.WaitAsync(TimeSpan.FromSeconds(2)); + task.Release.TrySetResult(); + + Assert.IsTrue(await scheduled.WaitAsync(TimeSpan.FromSeconds(2))); + CollectionAssert.AreEqual(new[] { false, true }, leaseManager.Forces); + Assert.AreEqual(1, task.ExecutionCount); + + await cancellation.CancelAsync(); + await AssertCanceledAsync(processor); + } + [TestMethod] public async Task RunScheduledAsync_TemporaryLeaseStoreFailureDoesNotStopQueue() { @@ -147,6 +170,47 @@ private sealed class FakeLeaseManager : IScheduledTaskLeaseManager return Task.FromException(AcquireException); return Task.FromResult(Deny ? null : Lease); } + + public Task> GetStatusesAsync( + IReadOnlyCollection taskIds, + CancellationToken cancellationToken) => + Task.FromResult>( + new Dictionary()); + } + + private sealed class ForceUpgradeLeaseManager : IScheduledTaskLeaseManager + { + public TaskCompletionSource FirstAcquireStarted { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource ReleaseFirstAcquire { get; } = new( + TaskCreationOptions.RunContinuationsAsynchronously); + public bool[] Forces => _forces.ToArray(); + + private readonly List _forces = []; + private readonly FakeLease _lease = new(); + + public async Task TryAcquireAsync( + string taskId, + TimeSpan interval, + bool force, + CancellationToken cancellationToken) + { + _forces.Add(force); + if (_forces.Count == 1) + { + FirstAcquireStarted.TrySetResult(); + await ReleaseFirstAcquire.Task.WaitAsync(cancellationToken); + return null; + } + + return _lease; + } + + public Task> GetStatusesAsync( + IReadOnlyCollection taskIds, + CancellationToken cancellationToken) => + Task.FromResult>( + new Dictionary()); } private sealed class FakeLease : IScheduledTaskExecutionLease diff --git a/SecondDimensionWatcherReDive.Test/TasksControllerTests.cs b/SecondDimensionWatcherReDive.Test/TasksControllerTests.cs new file mode 100644 index 0000000..89cb9c3 --- /dev/null +++ b/SecondDimensionWatcherReDive.Test/TasksControllerTests.cs @@ -0,0 +1,39 @@ +using Microsoft.AspNetCore.Mvc; +using Moq; +using SecondDimensionWatcherReDive.Controllers; +using SecondDimensionWatcherReDive.Framework.Tasks; + +namespace SecondDimensionWatcherReDive.Test; + +[TestClass] +public sealed class TasksControllerTests +{ + [TestMethod] + public async Task GetTasksAsync_UsesSharedLeaseStatuses() + { + var task = new Mock(MockBehavior.Strict); + task.SetupGet(candidate => candidate.Id).Returns("remote-task"); + task.SetupGet(candidate => candidate.Interval).Returns(TimeSpan.FromMinutes(10)); + task.SetupGet(candidate => candidate.IsEnabled).Returns(true); + var lastRunAt = DateTimeOffset.UtcNow.AddMinutes(-3); + var leaseManager = new Mock(MockBehavior.Strict); + leaseManager.Setup(candidate => candidate.GetStatusesAsync( + It.Is>(ids => ids.SequenceEqual(new[] { "remote-task" })), + CancellationToken.None)) + .ReturnsAsync(new Dictionary + { + ["remote-task"] = new(lastRunAt, true) + }); + var controller = new TasksController([task.Object], leaseManager.Object); + + var result = await controller.GetTasksAsync(CancellationToken.None); + + var ok = result as OkObjectResult; + Assert.IsNotNull(ok); + var response = ok.Value as IReadOnlyList; + Assert.IsNotNull(response); + Assert.HasCount(1, response); + Assert.AreEqual(lastRunAt, response[0].LastRunAt); + Assert.IsTrue(response[0].IsRunning); + } +} diff --git a/SecondDimensionWatcherReDive/Controllers/TasksController.cs b/SecondDimensionWatcherReDive/Controllers/TasksController.cs index 50dd947..4cec76d 100644 --- a/SecondDimensionWatcherReDive/Controllers/TasksController.cs +++ b/SecondDimensionWatcherReDive/Controllers/TasksController.cs @@ -8,17 +8,28 @@ namespace SecondDimensionWatcherReDive.Controllers; [ApiController] [Route("api/[controller]")] [Authorize(AuthenticationSchemes = JwtBearerDefaults.AuthenticationScheme)] -internal class TasksController(IEnumerable scheduledTasks) : ControllerBase +internal class TasksController( + IEnumerable scheduledTasks, + IScheduledTaskLeaseManager leaseManager) : ControllerBase { [HttpGet] - public IActionResult GetTasks() + public async Task GetTasksAsync(CancellationToken cancellationToken) { - var tasks = scheduledTasks.Select(t => new External.ScheduledTask( - t.Id, - t.Interval.ToString(), - t.IsEnabled, - t.LastRunAt, - t.IsRunning)).ToList(); + var taskList = scheduledTasks.ToList(); + var statuses = await leaseManager.GetStatusesAsync( + taskList.Select(task => task.Id).ToArray(), + cancellationToken); + var tasks = taskList.Select(task => + { + var status = statuses.GetValueOrDefault(task.Id) + ?? new ScheduledTaskStatus(null, false); + return new External.ScheduledTask( + task.Id, + task.Interval.ToString(), + task.IsEnabled, + status.LastRunAt, + status.IsRunning); + }).ToList(); return Ok(tasks); } diff --git a/SecondDimensionWatcherReDive/Repositories/FileMappingRepositoryPostgreSqlTestFixture.cs b/SecondDimensionWatcherReDive/Repositories/FileMappingRepositoryPostgreSqlTestFixture.cs index 4068035..b9b2987 100644 --- a/SecondDimensionWatcherReDive/Repositories/FileMappingRepositoryPostgreSqlTestFixture.cs +++ b/SecondDimensionWatcherReDive/Repositories/FileMappingRepositoryPostgreSqlTestFixture.cs @@ -249,6 +249,16 @@ public async Task CompleteTaskLeaseAsync( cancellationToken); } + public async Task> GetTaskLeaseStatesAsync( + IReadOnlyCollection taskIds, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + return await new ScheduledTaskLeaseRepository(context).GetStatesAsync( + taskIds, + cancellationToken); + } + public async Task RetryJobsAsync( IReadOnlyCollection ids, DateTimeOffset now, diff --git a/SecondDimensionWatcherReDive/Repositories/ScheduledTaskLeaseRepository.cs b/SecondDimensionWatcherReDive/Repositories/ScheduledTaskLeaseRepository.cs index 7e1d49f..32d6f5f 100644 --- a/SecondDimensionWatcherReDive/Repositories/ScheduledTaskLeaseRepository.cs +++ b/SecondDimensionWatcherReDive/Repositories/ScheduledTaskLeaseRepository.cs @@ -92,4 +92,23 @@ public Task CompleteAsync( state => succeeded ? completedAt : state.LastSucceededAt) .SetProperty(state => state.LastError, succeeded ? null : error), cancellationToken); + + public async Task> GetStatesAsync( + IReadOnlyCollection taskIds, + CancellationToken cancellationToken) + { + if (taskIds.Count == 0) return []; + + var ids = taskIds.Distinct(StringComparer.Ordinal).ToArray(); + return await context.ScheduledTaskStates + .AsNoTracking() + .Where(state => ids.Contains(state.TaskId)) + .Select(state => new ScheduledTaskLeaseState( + state.TaskId, + state.LeaseOwner, + state.LeaseExpiresAt, + state.LastStartedAt, + state.LastCompletedAt)) + .ToListAsync(cancellationToken); + } } diff --git a/SecondDimensionWatcherReDive/Services/PostgresScheduledTaskLeaseManager.cs b/SecondDimensionWatcherReDive/Services/PostgresScheduledTaskLeaseManager.cs index c16108d..ebafb6c 100644 --- a/SecondDimensionWatcherReDive/Services/PostgresScheduledTaskLeaseManager.cs +++ b/SecondDimensionWatcherReDive/Services/PostgresScheduledTaskLeaseManager.cs @@ -33,6 +33,41 @@ public sealed partial class PostgresScheduledTaskLeaseManager( : null; } + public async Task> GetStatusesAsync( + IReadOnlyCollection taskIds, + CancellationToken cancellationToken) + { + var ids = taskIds.Distinct(StringComparer.Ordinal).ToArray(); + if (ids.Length == 0) + return new Dictionary(StringComparer.Ordinal); + + await using var scope = scopeFactory.CreateAsyncScope(); + var repository = scope.ServiceProvider.GetRequiredService(); + var persistedStates = await repository.GetStatesAsync(ids, cancellationToken); + var statesById = persistedStates.ToDictionary( + state => state.TaskId, + StringComparer.Ordinal); + var now = DateTimeOffset.UtcNow; + return ids.ToDictionary( + taskId => taskId, + taskId => statesById.TryGetValue(taskId, out var state) + ? ToStatus(state, now) + : new ScheduledTaskStatus(null, false), + StringComparer.Ordinal); + } + + private static ScheduledTaskStatus ToStatus( + ScheduledTaskLeaseState state, + DateTimeOffset now) + { + var isRunning = state.LeaseOwner is not null + && state.LeaseExpiresAt > now + && state.LastStartedAt is { } startedAt + && (state.LastCompletedAt is null + || startedAt > state.LastCompletedAt); + return new ScheduledTaskStatus(state.LastCompletedAt, isRunning); + } + private sealed class ExecutionLease : IScheduledTaskExecutionLease { private readonly string _taskId; From 23198d0e139232fa5dd1ca714e7ebc07065325d0 Mon Sep 17 00:00:00 2001 From: mahoshojoHCG Date: Sun, 30 Aug 2026 11:21:22 +0800 Subject: [PATCH 14/45] ci: initialize backup drill workspace at runtime --- .github/workflows/backup-restore.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/backup-restore.yml b/.github/workflows/backup-restore.yml index e219226..67dc844 100644 --- a/.github/workflows/backup-restore.yml +++ b/.github/workflows/backup-restore.yml @@ -41,7 +41,6 @@ jobs: ConnectionStrings__sdw: Host=127.0.0.1;Port=5432;Username=postgres;Password=postgres;Database=sdw_source JwtSecret: backup-restore-drill-jwt-secret-with-at-least-32-bytes ASPNETCORE_ENVIRONMENT: Production - WORK_DIR: ${{ runner.temp }}/sdw-backup-drill steps: - uses: actions/checkout@v7 @@ -49,6 +48,9 @@ jobs: with: dotnet-version: "10.0.x" + - name: Configure drill workspace + run: printf 'WORK_DIR=%s\n' "$RUNNER_TEMP/sdw-backup-drill" >> "$GITHUB_ENV" + - name: Install PostgreSQL client run: sudo apt-get update && sudo apt-get install -y postgresql-client From f6fda4e09f6f74dcff987e88a0e26a5514e3450d Mon Sep 17 00:00:00 2001 From: mahoshojoHCG Date: Sun, 30 Aug 2026 11:47:06 +0800 Subject: [PATCH 15/45] fix: make release upgrades atomic and recoverable --- .../Tools/ManageDownloadsTool.cs | 1 + .../DataRepository/AnimationInfo.cs | 2 +- .../DataRepository/IFileMappingRepository.cs | 1 + .../DataRepository/ReleaseUpgrade.cs | 2 + .../Helpers/Fakes.cs | 1 + .../FileMappingRepositoryPostgreSqlTests.cs | 366 ++++- .../AnimationInfoControllerTests.cs | 6 + .../ReleaseUpgradeCoordinatorTests.cs | 421 +++++- .../Controllers/AnimationInfoController.cs | 1 + ...orceSingleActiveEpisodeRelease.Designer.cs | 1199 +++++++++++++++++ ...30724_EnforceSingleActiveEpisodeRelease.cs | 89 ++ .../ApplicationContextModelSnapshot.cs | 12 +- .../Models/AnimationInfo.cs | 2 +- .../Models/ApplicationContext.cs | 15 +- .../Repositories/AnimationInfoRepository.cs | 272 +++- .../Repositories/FileMappingRepository.cs | 27 +- ...eMappingRepositoryPostgreSqlTestFixture.cs | 542 +++++++- .../Repositories/MetadataReviewRepository.cs | 42 + .../Repositories/ReleaseUpgradeRepository.cs | 355 ++++- .../ReleaseUpgradeCoordinator.cs | 249 +++- 20 files changed, 3435 insertions(+), 170 deletions(-) create mode 100644 SecondDimensionWatcherReDive/Migrations/20260830030724_EnforceSingleActiveEpisodeRelease.Designer.cs create mode 100644 SecondDimensionWatcherReDive/Migrations/20260830030724_EnforceSingleActiveEpisodeRelease.cs diff --git a/Plugins/SecondDimensionWatcherReDive.Chat/Tools/ManageDownloadsTool.cs b/Plugins/SecondDimensionWatcherReDive.Chat/Tools/ManageDownloadsTool.cs index 831ec4b..f28acd2 100644 --- a/Plugins/SecondDimensionWatcherReDive.Chat/Tools/ManageDownloadsTool.cs +++ b/Plugins/SecondDimensionWatcherReDive.Chat/Tools/ManageDownloadsTool.cs @@ -141,6 +141,7 @@ private async Task CancelDownloadAsync( info.Id, info.DownloadAttemptId, cancellationAttemptId, + terminalDisposition: null, finalizeCancellation.Token); if (!cancelled) return new ToolFailureResult("Download state changed during cancellation"); diff --git a/SecondDimensionWatcherReDive.Framework/DataRepository/AnimationInfo.cs b/SecondDimensionWatcherReDive.Framework/DataRepository/AnimationInfo.cs index be05aaa..d15d21c 100644 --- a/SecondDimensionWatcherReDive.Framework/DataRepository/AnimationInfo.cs +++ b/SecondDimensionWatcherReDive.Framework/DataRepository/AnimationInfo.cs @@ -47,4 +47,4 @@ public sealed record AnimationInfo( string? ReleaseScoreReasonsJson = null, int? ExpectedEpisodeCount = null, DateTimeOffset? IngestedAt = null, - bool IsActiveRelease = true); + bool IsActiveRelease = false); diff --git a/SecondDimensionWatcherReDive.Framework/DataRepository/IFileMappingRepository.cs b/SecondDimensionWatcherReDive.Framework/DataRepository/IFileMappingRepository.cs index e3b800e..0d7cd2e 100644 --- a/SecondDimensionWatcherReDive.Framework/DataRepository/IFileMappingRepository.cs +++ b/SecondDimensionWatcherReDive.Framework/DataRepository/IFileMappingRepository.cs @@ -46,6 +46,7 @@ Task TryFinalizeDownloadCancellationAsync( Guid animationInfoId, Guid? downloadAttemptId, Guid cancellationAttemptId, + SubscriptionAutomationDisposition? terminalDisposition, CancellationToken cancellationToken); Task RemoveByAnimationInfoAsync(Guid animationInfoId, CancellationToken cancellationToken); diff --git a/SecondDimensionWatcherReDive.Framework/DataRepository/ReleaseUpgrade.cs b/SecondDimensionWatcherReDive.Framework/DataRepository/ReleaseUpgrade.cs index 547cd5d..0c2e280 100644 --- a/SecondDimensionWatcherReDive.Framework/DataRepository/ReleaseUpgrade.cs +++ b/SecondDimensionWatcherReDive.Framework/DataRepository/ReleaseUpgrade.cs @@ -76,6 +76,8 @@ Task> GetReadyCandidateIdsAsync( Task ActivateAsync( Guid operationId, + IReadOnlyList expectedPreviousMappings, + IReadOnlyList expectedCandidateMappings, DateTimeOffset verifiedAt, DateTimeOffset rollbackUntil, CancellationToken cancellationToken); diff --git a/SecondDimensionWatcherReDive.IntegrationTest/Helpers/Fakes.cs b/SecondDimensionWatcherReDive.IntegrationTest/Helpers/Fakes.cs index 62034f0..1be2230 100644 --- a/SecondDimensionWatcherReDive.IntegrationTest/Helpers/Fakes.cs +++ b/SecondDimensionWatcherReDive.IntegrationTest/Helpers/Fakes.cs @@ -79,6 +79,7 @@ public Task TryFinalizeDownloadCancellationAsync( Guid animationInfoId, Guid? downloadAttemptId, Guid cancellationAttemptId, + SubscriptionAutomationDisposition? terminalDisposition, CancellationToken cancellationToken) { _mappings.RemoveAll(mapping => mapping.AnimationInfoId == animationInfoId); diff --git a/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/FileMappingRepositoryPostgreSqlTests.cs b/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/FileMappingRepositoryPostgreSqlTests.cs index 91313ad..2bd4534 100644 --- a/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/FileMappingRepositoryPostgreSqlTests.cs +++ b/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/FileMappingRepositoryPostgreSqlTests.cs @@ -165,12 +165,36 @@ public async Task Migration_CreatesReleaseUniquenessAndSearchIndexes() "IX_Animations_OriginalName_Trgm", "IX_AnimationGroups_Name_Trgm", "IX_FileMappings_VirtualPath_Trgm", - "IX_AnimationInfo_ReleaseLanguages_Gin" + "IX_AnimationInfo_ReleaseLanguages_Gin", + "IX_AnimationInfo_AnimationId", + "UX_AnimationInfo_ActiveEpisodeRelease" }; Assert.IsTrue(expected.All(indexes.Contains), $"Missing indexes: {string.Join(", ", expected.Except(indexes))}"); } + [TestMethod] + public async Task Migration_DeduplicatesActiveReleasesBeforeCreatingUniqueIndex() + { + await using var migrationDatabase = new PostgreSqlBuilder("postgres:17-alpine") + .WithDatabase("sdw_migration_tests") + .WithUsername("postgres") + .WithPassword("postgres") + .Build(); + await migrationDatabase.StartAsync(); + var fixture = new FileMappingRepositoryPostgreSqlTestFixture( + migrationDatabase.GetConnectionString()); + + var result = await fixture.MigrateDuplicateActiveReleasesAsync(CancellationToken.None); + var indexes = await fixture.GetLibraryIndexNamesAsync(CancellationToken.None); + + Assert.HasCount(1, result.ActiveIds); + Assert.AreEqual(result.ExpectedActiveId, result.ActiveIds[0]); + Assert.AreEqual(1, result.DowngradedCandidateOperationCount); + CollectionAssert.Contains(indexes.ToArray(), "IX_AnimationInfo_AnimationId"); + CollectionAssert.Contains(indexes.ToArray(), "UX_AnimationInfo_ActiveEpisodeRelease"); + } + [TestMethod] public async Task UpgradeRace_ClaimsOnce_AtomicallySwapsMappings_AndRollsBack() { @@ -184,8 +208,13 @@ public async Task UpgradeRace_ClaimsOnce_AtomicallySwapsMappings_AndRollsBack() var beforeCurrent = await Fixture.GetMappingsAsync( scenario.Candidate.CurrentReleaseId, CancellationToken.None); - Assert.HasCount(1, beforeCurrent); - Assert.AreEqual(scenario.CanonicalPath, beforeCurrent[0].VirtualPath); + Assert.HasCount(2, beforeCurrent); + CollectionAssert.Contains( + beforeCurrent.Select(mapping => mapping.VirtualPath).ToArray(), + scenario.CanonicalPath); + CollectionAssert.Contains( + beforeCurrent.Select(mapping => mapping.VirtualPath).ToArray(), + scenario.CanonicalSubtitlePath); var applied = await Fixture.ActivateUpgradeAsync(operation.Id, CancellationToken.None); Assert.IsTrue(applied.IsSuccess); @@ -194,18 +223,341 @@ public async Task UpgradeRace_ClaimsOnce_AtomicallySwapsMappings_AndRollsBack() scenario.Candidate.CurrentReleaseId, CancellationToken.None)); var activeCandidate = await Fixture.GetMappingsAsync( scenario.Candidate.CandidateReleaseId, CancellationToken.None); - Assert.HasCount(1, activeCandidate); - Assert.AreEqual(scenario.CanonicalPath, activeCandidate[0].VirtualPath); + Assert.HasCount(2, activeCandidate); + var activeVideo = activeCandidate.Single(mapping => mapping.VirtualPath == scenario.CanonicalPath); + var activeSubtitle = activeCandidate.Single(mapping => + mapping.VirtualPath == scenario.CanonicalSubtitlePath); + Assert.AreEqual("/store/new.mkv", activeVideo.PhysicalPath); + Assert.AreEqual("/store/new.en.srt", activeSubtitle.PhysicalPath); + var activeProgress = await Fixture.GetPlaybackProgressesAsync( + scenario.UserId, + CancellationToken.None); + Assert.HasCount(1, activeProgress); + Assert.AreEqual(scenario.Candidate.CandidateReleaseId, activeProgress[0].AnimationInfoId); + Assert.AreEqual(scenario.CanonicalPath, activeProgress[0].VirtualPath); + Assert.AreEqual(321d, activeProgress[0].PositionSeconds); + var lateFailure = await Fixture.MarkUpgradeFailedAsync(operation.Id, CancellationToken.None); + Assert.IsFalse(lateFailure.IsSuccess); + Assert.AreEqual("invalid_state", lateFailure.Outcome); + Assert.AreEqual(ReleaseUpgradeStatus.Applied, lateFailure.Operation!.Status); var rolledBack = await Fixture.RollbackUpgradeAsync(operation.Id, CancellationToken.None); Assert.IsTrue(rolledBack.IsSuccess); Assert.AreEqual(ReleaseUpgradeStatus.RolledBack, rolledBack.Operation!.Status); var restored = await Fixture.GetMappingsAsync( scenario.Candidate.CurrentReleaseId, CancellationToken.None); - Assert.HasCount(1, restored); - Assert.AreEqual(scenario.CanonicalPath, restored[0].VirtualPath); + Assert.HasCount(2, restored); + CollectionAssert.Contains( + restored.Select(mapping => mapping.VirtualPath).ToArray(), + scenario.CanonicalPath); + CollectionAssert.Contains( + restored.Select(mapping => mapping.VirtualPath).ToArray(), + scenario.CanonicalSubtitlePath); Assert.IsEmpty(await Fixture.GetMappingsAsync( scenario.Candidate.CandidateReleaseId, CancellationToken.None)); + var restoredProgress = await Fixture.GetPlaybackProgressesAsync( + scenario.UserId, + CancellationToken.None); + Assert.HasCount(1, restoredProgress); + Assert.AreEqual(scenario.Candidate.CurrentReleaseId, restoredProgress[0].AnimationInfoId); + Assert.AreEqual(scenario.CanonicalPath, restoredProgress[0].VirtualPath); + Assert.AreEqual(321d, restoredProgress[0].PositionSeconds); + } + + [TestMethod] + public async Task IdentifiedAlternatives_KeepExactlyOneActiveRelease() + { + var activities = await Fixture.IdentifyCompetingReleasesAsync(CancellationToken.None); + + Assert.IsTrue(activities.FirstActive); + Assert.IsFalse(activities.SecondActive); + } + + [TestMethod] + public async Task MovingActiveRelease_PromotesPreviousEpisodeSuccessor() + { + var activities = await Fixture.IdentifyCompetingReleasesAsync( + CancellationToken.None, + moveFirst: true); + + Assert.IsTrue(activities.FirstActive); + Assert.AreEqual(2, activities.FirstEpisode); + Assert.IsTrue(activities.SecondActive); + } + + [TestMethod] + public async Task DeidentifyingActiveRelease_PromotesPreviousEpisodeSuccessor() + { + var activities = await Fixture.IdentifyCompetingReleasesAsync( + CancellationToken.None, + deidentifyFirst: true); + + Assert.IsFalse(activities.FirstActive); + Assert.IsTrue(activities.SecondActive); + } + + [TestMethod] + public async Task ConcurrentIdentification_ActivatesExactlyOneRelease() + { + var activities = await Fixture.IdentifyCompetingReleasesAsync( + CancellationToken.None, + concurrent: true); + + Assert.AreEqual(1, new[] { activities.FirstActive, activities.SecondActive }.Count(active => active)); + } + + [TestMethod] + public async Task ReleaseUpgrade_RejectsMetadataDriftAfterOperationBegins() + { + var scenario = await Fixture.SeedUpgradeScenarioAsync(CancellationToken.None); + var operation = await Fixture.BeginUpgradeAsync(scenario.Candidate, CancellationToken.None); + Assert.IsNotNull(operation); + await Fixture.ChangeReleaseEpisodeAsync( + scenario.Candidate.CandidateReleaseId, + 2, + CancellationToken.None); + + var applied = await Fixture.ActivateUpgradeAsync(operation.Id, CancellationToken.None); + + Assert.IsFalse(applied.IsSuccess); + Assert.AreEqual("release_changed", applied.Outcome); + Assert.HasCount(2, await Fixture.GetMappingsAsync( + scenario.Candidate.CurrentReleaseId, CancellationToken.None)); + Assert.HasCount(2, await Fixture.GetMappingsAsync( + scenario.Candidate.CandidateReleaseId, CancellationToken.None)); + } + + [TestMethod] + public async Task ReleaseUpgrade_RollbackRejectsInterveningMappingDrift() + { + var scenario = await Fixture.SeedUpgradeScenarioAsync(CancellationToken.None); + var operation = await Fixture.BeginUpgradeAsync(scenario.Candidate, CancellationToken.None); + Assert.IsNotNull(operation); + var applied = await Fixture.ActivateUpgradeAsync(operation.Id, CancellationToken.None); + Assert.IsTrue(applied.IsSuccess); + var remappedPath = "/Upgrade Show/Reviewed/Upgrade Show S01E01.MKV"; + await Fixture.RemapCandidatePlaybackAsync( + scenario.Candidate.CandidateReleaseId, + scenario.UserId, + scenario.CanonicalPath, + remappedPath, + CancellationToken.None); + + var rolledBack = await Fixture.RollbackUpgradeAsync(operation.Id, CancellationToken.None); + + Assert.IsFalse(rolledBack.IsSuccess); + Assert.AreEqual("mapping_changed", rolledBack.Outcome); + var candidateMappings = await Fixture.GetMappingsAsync( + scenario.Candidate.CandidateReleaseId, CancellationToken.None); + CollectionAssert.Contains( + candidateMappings.Select(mapping => mapping.VirtualPath).ToArray(), + remappedPath); + var progress = await Fixture.GetPlaybackProgressesAsync( + scenario.UserId, + CancellationToken.None); + Assert.HasCount(1, progress); + Assert.AreEqual(scenario.Candidate.CandidateReleaseId, progress[0].AnimationInfoId); + Assert.AreEqual(remappedPath, progress[0].VirtualPath); + } + + [TestMethod] + public async Task ReleaseUpgrade_RejectsMappingsChangedAfterFileValidation() + { + var scenario = await Fixture.SeedUpgradeScenarioAsync(CancellationToken.None); + var operation = await Fixture.BeginUpgradeAsync(scenario.Candidate, CancellationToken.None); + Assert.IsNotNull(operation); + var expected = await Fixture.GetUpgradeActivationAsync( + scenario.Candidate.CandidateReleaseId, + CancellationToken.None); + Assert.IsNotNull(expected); + var candidateVideo = expected.CandidateMappings.Single(mapping => + mapping.PhysicalPath == "/store/new.mkv"); + await Fixture.ChangeMappingPhysicalPathAsync( + scenario.Candidate.CandidateReleaseId, + candidateVideo.VirtualPath, + "/store/unvalidated.mkv", + CancellationToken.None); + + var applied = await Fixture.ActivateUpgradeAsync( + operation.Id, + expected, + CancellationToken.None); + + Assert.IsFalse(applied.IsSuccess); + Assert.AreEqual("mapping_changed", applied.Outcome); + Assert.HasCount(2, await Fixture.GetMappingsAsync( + scenario.Candidate.CurrentReleaseId, CancellationToken.None)); + var candidateMappings = await Fixture.GetMappingsAsync( + scenario.Candidate.CandidateReleaseId, CancellationToken.None); + CollectionAssert.Contains( + candidateMappings.Select(mapping => mapping.PhysicalPath).ToArray(), + "/store/unvalidated.mkv"); + } + + [TestMethod] + public async Task ReleaseUpgrade_DuplicateFileRolesRollbackDeterministically() + { + var scenario = await Fixture.SeedUpgradeScenarioAsync( + CancellationToken.None, + includeDuplicateVideoRole: true); + var operation = await Fixture.BeginUpgradeAsync(scenario.Candidate, CancellationToken.None); + Assert.IsNotNull(operation); + + var applied = await Fixture.ActivateUpgradeAsync(operation.Id, CancellationToken.None); + Assert.IsTrue(applied.IsSuccess); + var activeMappings = await Fixture.GetMappingsAsync( + scenario.Candidate.CandidateReleaseId, + CancellationToken.None); + Assert.AreEqual( + "/store/new.mkv", + activeMappings.Single(mapping => mapping.VirtualPath == scenario.CanonicalPath).PhysicalPath); + + var rolledBack = await Fixture.RollbackUpgradeAsync(operation.Id, CancellationToken.None); + Assert.IsTrue(rolledBack.IsSuccess); + Assert.HasCount(2, await Fixture.GetMappingsAsync( + scenario.Candidate.CurrentReleaseId, + CancellationToken.None)); + Assert.IsEmpty(await Fixture.GetMappingsAsync( + scenario.Candidate.CandidateReleaseId, + CancellationToken.None)); + } + + [TestMethod] + public async Task FailedReleaseUpgrade_CanBeClaimedAgain() + { + var scenario = await Fixture.SeedUpgradeScenarioAsync(CancellationToken.None); + var first = await Fixture.BeginUpgradeAsync(scenario.Candidate, CancellationToken.None); + Assert.IsNotNull(first); + var failed = await Fixture.MarkUpgradeFailedAsync(first.Id, CancellationToken.None); + Assert.IsTrue(failed.IsSuccess); + + var retry = await Fixture.BeginUpgradeAsync(scenario.Candidate, CancellationToken.None); + + Assert.IsNotNull(retry); + Assert.AreNotEqual(first.Id, retry.Id); + } + + [TestMethod] + public async Task CancellingTrackedUpgrade_TerminatesOperationAndAllowsRetry() + { + var scenario = await Fixture.SeedUpgradeScenarioAsync(CancellationToken.None); + var downloadAttemptId = await Fixture.SetCandidateDownloadInProgressAsync( + scenario.Candidate.CandidateReleaseId, + CancellationToken.None); + var first = await Fixture.BeginUpgradeAsync(scenario.Candidate, CancellationToken.None); + Assert.IsNotNull(first); + Assert.AreEqual(ReleaseUpgradeStatus.Downloading, first.Status); + + var cancelled = await Fixture.CancelUpgradeCandidateAsync( + scenario.Candidate.CandidateReleaseId, + downloadAttemptId, + CancellationToken.None); + Assert.IsNotNull(cancelled); + Assert.IsFalse(cancelled.IsDownloadTracked); + var retry = await Fixture.BeginUpgradeAsync(scenario.Candidate, CancellationToken.None); + + Assert.IsNotNull(retry); + Assert.AreNotEqual(first.Id, retry.Id); + } + + [TestMethod] + public async Task UpgradeCancellationIntent_PreventsActivationBeforeRemoteFinalize() + { + var scenario = await Fixture.SeedUpgradeScenarioAsync(CancellationToken.None); + var operation = await Fixture.BeginUpgradeAsync(scenario.Candidate, CancellationToken.None); + Assert.IsNotNull(operation); + Assert.AreEqual(ReleaseUpgradeStatus.Verifying, operation.Status); + var expected = await Fixture.GetUpgradeActivationAsync( + scenario.Candidate.CandidateReleaseId, + CancellationToken.None); + Assert.IsNotNull(expected); + var cancellationAttemptId = Guid.NewGuid(); + + var beganCancellation = await Fixture.BeginUpgradeCandidateCancellationAsync( + scenario.Candidate.CandidateReleaseId, + downloadAttemptId: null, + cancellationAttemptId, + CancellationToken.None); + var applied = await Fixture.ActivateUpgradeAsync( + operation.Id, + expected, + CancellationToken.None); + var finalized = await Fixture.FinalizeUpgradeCandidateCancellationAsync( + scenario.Candidate.CandidateReleaseId, + downloadAttemptId: null, + cancellationAttemptId, + CancellationToken.None); + var persisted = await Fixture.GetUpgradeOperationAsync( + operation.Id, + CancellationToken.None); + + Assert.IsTrue(beganCancellation); + Assert.IsFalse(applied.IsSuccess); + Assert.AreEqual("invalid_state", applied.Outcome); + Assert.IsTrue(finalized); + Assert.AreEqual(ReleaseUpgradeStatus.Failed, persisted.Status); + Assert.IsEmpty(await Fixture.GetMappingsAsync( + scenario.Candidate.CandidateReleaseId, + CancellationToken.None)); + Assert.HasCount(2, await Fixture.GetMappingsAsync( + scenario.Candidate.CurrentReleaseId, + CancellationToken.None)); + } + + [TestMethod] + public async Task AppliedUpgrade_RejectsStaleCancellationBeforeRemoteDeletion() + { + var scenario = await Fixture.SeedUpgradeScenarioAsync(CancellationToken.None); + var operation = await Fixture.BeginUpgradeAsync(scenario.Candidate, CancellationToken.None); + Assert.IsNotNull(operation); + var applied = await Fixture.ActivateUpgradeAsync(operation.Id, CancellationToken.None); + Assert.IsTrue(applied.IsSuccess); + + var beganCancellation = await Fixture.BeginUpgradeCandidateCancellationAsync( + scenario.Candidate.CandidateReleaseId, + downloadAttemptId: null, + cancellationAttemptId: Guid.NewGuid(), + CancellationToken.None); + + Assert.IsFalse(beganCancellation); + Assert.HasCount(2, await Fixture.GetMappingsAsync( + scenario.Candidate.CandidateReleaseId, + CancellationToken.None)); + Assert.IsEmpty(await Fixture.GetMappingsAsync( + scenario.Candidate.CurrentReleaseId, + CancellationToken.None)); + } + + [TestMethod] + public async Task ReleaseUpgrade_MergesNewestPlaybackStateAcrossActivationAndRollback() + { + var scenario = await Fixture.SeedUpgradeScenarioAsync( + CancellationToken.None, + includeCandidateProgress: true); + var operation = await Fixture.BeginUpgradeAsync(scenario.Candidate, CancellationToken.None); + Assert.IsNotNull(operation); + + var applied = await Fixture.ActivateUpgradeAsync(operation.Id, CancellationToken.None); + Assert.IsTrue(applied.IsSuccess); + var activeProgress = await Fixture.GetPlaybackProgressesAsync( + scenario.UserId, + CancellationToken.None); + Assert.HasCount(1, activeProgress); + Assert.AreEqual(scenario.Candidate.CandidateReleaseId, activeProgress[0].AnimationInfoId); + Assert.IsTrue(activeProgress[0].IsWatched); + Assert.IsNotNull(activeProgress[0].WatchedAt); + Assert.AreEqual(1200d, activeProgress[0].PositionSeconds); + + var rolledBack = await Fixture.RollbackUpgradeAsync(operation.Id, CancellationToken.None); + Assert.IsTrue(rolledBack.IsSuccess); + var restoredProgress = await Fixture.GetPlaybackProgressesAsync( + scenario.UserId, + CancellationToken.None); + Assert.HasCount(1, restoredProgress); + Assert.AreEqual(scenario.Candidate.CurrentReleaseId, restoredProgress[0].AnimationInfoId); + Assert.IsTrue(restoredProgress[0].IsWatched); + Assert.IsNotNull(restoredProgress[0].WatchedAt); + Assert.AreEqual(1200d, restoredProgress[0].PositionSeconds); } private static FileMapping Mapping(Guid animationInfoId, string virtualPath) => diff --git a/SecondDimensionWatcherReDive.Test/AnimationInfoControllerTests.cs b/SecondDimensionWatcherReDive.Test/AnimationInfoControllerTests.cs index 1cf8e25..f80d5e0 100644 --- a/SecondDimensionWatcherReDive.Test/AnimationInfoControllerTests.cs +++ b/SecondDimensionWatcherReDive.Test/AnimationInfoControllerTests.cs @@ -327,6 +327,7 @@ public async Task CancelDownload_Success_SetsIsDownloadTrackedFalseAndUpdates() id, info.DownloadAttemptId, It.IsAny(), + null, It.Is(token => token.CanBeCanceled && !token.IsCancellationRequested))) .ReturnsAsync(true); @@ -339,6 +340,7 @@ public async Task CancelDownload_Success_SetsIsDownloadTrackedFalseAndUpdates() id, info.DownloadAttemptId, cancellationAttemptId.Value, + null, It.Is(token => token.CanBeCanceled && !token.IsCancellationRequested)), Times.Once); } @@ -370,6 +372,7 @@ public async Task CancelDownload_AutomaticDownload_MarksDispositionCancelled() id, info.DownloadAttemptId, It.IsAny(), + null, It.IsAny())) .ReturnsAsync(true); @@ -380,6 +383,7 @@ public async Task CancelDownload_AutomaticDownload_MarksDispositionCancelled() id, info.DownloadAttemptId, It.IsAny(), + null, It.IsAny()), Times.Once); } @@ -413,6 +417,7 @@ public async Task CancelDownload_PendingCancellation_ReusesIdAndFinalizes() id, info.DownloadAttemptId, cancellationAttemptId, + null, It.IsAny())) .ReturnsAsync(true); @@ -428,6 +433,7 @@ public async Task CancelDownload_PendingCancellation_ReusesIdAndFinalizes() id, info.DownloadAttemptId, cancellationAttemptId, + null, It.IsAny()), Times.Once); } diff --git a/SecondDimensionWatcherReDive.Test/ReleaseUpgradeCoordinatorTests.cs b/SecondDimensionWatcherReDive.Test/ReleaseUpgradeCoordinatorTests.cs index e048ecb..ff75110 100644 --- a/SecondDimensionWatcherReDive.Test/ReleaseUpgradeCoordinatorTests.cs +++ b/SecondDimensionWatcherReDive.Test/ReleaseUpgradeCoordinatorTests.cs @@ -23,7 +23,8 @@ public async Task CandidateValidationFailure_DoesNotInvokeAtomicSwap_AndRecordsF Assert.IsNotNull(result); Assert.IsFalse(result.IsSuccess); fixture.UpgradeRepository.Verify(repository => repository.ActivateAsync( - It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny>(), + It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); fixture.UpgradeRepository.Verify(repository => repository.MarkFailedAsync( fixture.Operation.Id, @@ -50,26 +51,363 @@ public async Task ValidCandidate_InvokesAtomicSwapOnlyAfterEveryFilePasses() fixture.FileStore.Verify(store => store.FileInfoAsync( "/store/new.mkv", It.IsAny()), Times.Once); fixture.UpgradeRepository.Verify(repository => repository.ActivateAsync( - fixture.Operation.Id, + fixture.Operation.Id, It.IsAny>(), + It.IsAny>(), It.IsAny(), It.Is(until => until > DateTimeOffset.UtcNow.AddHours(71)), It.IsAny()), Times.Once); } + [TestMethod] + public async Task CandidateCompletedDuringClaim_ActivatesWithoutStartingAnotherDownload() + { + var fixture = new CoordinatorFixture( + fileExists: true, + candidateDownloaded: false, + operationStatus: ReleaseUpgradeStatus.Verifying); + + var result = await fixture.Coordinator.ExecuteAsync( + fixture.Candidate, + dryRun: false, + CancellationToken.None); + + Assert.IsTrue(result.IsSuccess); + fixture.AnimationRepository.Verify(repository => repository.TryStartDownloadAsync( + It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny()), Times.Never); + } + + [TestMethod] + public async Task CandidateCompletedBeforeMapping_KeepsVerifyingOperationPending() + { + var fixture = new CoordinatorFixture( + fileExists: true, + candidateDownloaded: false, + operationStatus: ReleaseUpgradeStatus.Verifying, + candidateMapped: false); + + var result = await fixture.Coordinator.ExecuteAsync( + fixture.Candidate, + dryRun: false, + CancellationToken.None); + + Assert.IsTrue(result.IsSuccess); + Assert.AreEqual("mapping_pending", result.Outcome); + fixture.UpgradeRepository.Verify(repository => repository.MarkFailedAsync( + It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + } + + [TestMethod] + public async Task ExistingTrackedCandidate_IsFollowedWithoutReplacingItsAttempt() + { + var fixture = new CoordinatorFixture( + fileExists: true, + candidateDownloaded: false, + candidateTracked: true); + + var result = await fixture.Coordinator.ExecuteAsync( + fixture.Candidate, + dryRun: false, + CancellationToken.None); + + Assert.IsTrue(result.IsSuccess); + Assert.AreEqual("download_in_progress", result.Outcome); + fixture.AnimationRepository.Verify(repository => repository.TryStartDownloadAsync( + It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny()), Times.Never); + fixture.UpgradeRepository.Verify(repository => repository.MarkFailedAsync( + It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + } + + [TestMethod] + public async Task DownloadClientResolutionFailure_RestoresClaimedLocalState() + { + var fixture = new CoordinatorFixture(fileExists: true, candidateDownloaded: false); + fixture.DownloadClientProvider + .Setup(provider => provider.GetRequiredClient(fixture.CandidateInfo.DownloadType)) + .Throws(new InvalidOperationException("client unavailable")); + + var result = await fixture.Coordinator.ExecuteAsync( + fixture.Candidate, + dryRun: false, + CancellationToken.None); + + Assert.IsFalse(result.IsSuccess); + fixture.AnimationRepository.Verify(repository => repository.TryBeginCancelDownloadAsync( + fixture.CandidateInfo.Id, + It.IsAny(), + It.IsAny(), + It.IsAny()), Times.Once); + fixture.FileMappingRepository.Verify(repository => repository.TryFinalizeDownloadCancellationAsync( + fixture.CandidateInfo.Id, + It.IsAny(), + It.IsAny(), + SubscriptionAutomationDisposition.AutoDownloadFailed, + It.IsAny()), Times.Once); + fixture.DownloadClient.Verify(client => client.CancelDownloadTaskAsync( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny()), Times.Never); + fixture.UpgradeRepository.Verify(repository => repository.MarkFailedAsync( + fixture.Operation.Id, + It.IsAny(), + It.IsAny()), Times.Once); + } + + [TestMethod] + public async Task DownloadSubmissionFailure_CancelsRemoteThenRestoresLocalState() + { + var fixture = new CoordinatorFixture(fileExists: true, candidateDownloaded: false); + var cancellationRegistered = false; + var remoteCancelled = false; + fixture.AnimationRepository.Setup(repository => repository.TryBeginCancelDownloadAsync( + fixture.CandidateInfo.Id, + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Callback(() => cancellationRegistered = true) + .ReturnsAsync(true); + fixture.DownloadClient.Setup(client => client.SubmitDownloadTaskAsync( + fixture.CandidateInfo.Id, + fixture.CandidateInfo.DownloadUrl, + fixture.CandidateInfo.CachedDownloadData, + fixture.CandidateInfo.AdditionalDownloadInfo, + It.IsAny())) + .ThrowsAsync(new IOException("submission interrupted")); + fixture.DownloadClient.Setup(client => client.CancelDownloadTaskAsync( + fixture.CandidateInfo.Id, + fixture.CandidateInfo.DownloadUrl, + fixture.CandidateInfo.CachedDownloadData, + fixture.CandidateInfo.AdditionalDownloadInfo, + false, + It.IsAny())) + .Callback(() => + { + Assert.IsTrue(cancellationRegistered); + remoteCancelled = true; + }) + .ReturnsAsync(new CancelDownloadResult(true, false)); + fixture.FileMappingRepository.Setup(repository => repository.TryFinalizeDownloadCancellationAsync( + fixture.CandidateInfo.Id, + It.IsAny(), + It.IsAny(), + SubscriptionAutomationDisposition.AutoDownloadFailed, + It.IsAny())) + .Callback(() => Assert.IsTrue(remoteCancelled)) + .ReturnsAsync(true); + + var result = await fixture.Coordinator.ExecuteAsync( + fixture.Candidate, + dryRun: false, + CancellationToken.None); + + Assert.IsFalse(result.IsSuccess); + fixture.DownloadClient.Verify(client => client.CancelDownloadTaskAsync( + fixture.CandidateInfo.Id, + fixture.CandidateInfo.DownloadUrl, + fixture.CandidateInfo.CachedDownloadData, + fixture.CandidateInfo.AdditionalDownloadInfo, + false, + It.IsAny()), Times.Once); + fixture.AnimationRepository.Verify(repository => repository.TryBeginCancelDownloadAsync( + fixture.CandidateInfo.Id, + It.IsAny(), + It.IsAny(), + It.IsAny()), Times.Once); + fixture.FileMappingRepository.Verify(repository => repository.TryFinalizeDownloadCancellationAsync( + fixture.CandidateInfo.Id, + It.IsAny(), + It.IsAny(), + SubscriptionAutomationDisposition.AutoDownloadFailed, + It.IsAny()), Times.Once); + fixture.UpgradeRepository.Verify(repository => repository.MarkFailedAsync( + fixture.Operation.Id, + It.IsAny(), + It.IsAny()), Times.Once); + } + + [TestMethod] + public async Task UnconfirmedRemoteCancellation_LeavesOperationRecoverable() + { + var fixture = new CoordinatorFixture(fileExists: true, candidateDownloaded: false); + fixture.DownloadClient.Setup(client => client.SubmitDownloadTaskAsync( + fixture.CandidateInfo.Id, + fixture.CandidateInfo.DownloadUrl, + fixture.CandidateInfo.CachedDownloadData, + fixture.CandidateInfo.AdditionalDownloadInfo, + It.IsAny())) + .ThrowsAsync(new IOException("submission outcome unknown")); + fixture.DownloadClient.Setup(client => client.CancelDownloadTaskAsync( + fixture.CandidateInfo.Id, + fixture.CandidateInfo.DownloadUrl, + fixture.CandidateInfo.CachedDownloadData, + fixture.CandidateInfo.AdditionalDownloadInfo, + false, + It.IsAny())) + .ReturnsAsync(new CancelDownloadResult(false, false)); + + var result = await fixture.Coordinator.ExecuteAsync( + fixture.Candidate, + dryRun: false, + CancellationToken.None); + + Assert.IsFalse(result.IsSuccess); + Assert.AreEqual("recovery_pending", result.Outcome); + Assert.IsTrue(result.RequiresDownload); + fixture.AnimationRepository.Verify(repository => repository.TryBeginCancelDownloadAsync( + fixture.CandidateInfo.Id, It.IsAny(), It.IsAny(), + It.IsAny()), Times.Once); + fixture.FileMappingRepository.Verify(repository => repository.TryFinalizeDownloadCancellationAsync( + It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny()), Times.Never); + fixture.UpgradeRepository.Verify(repository => repository.MarkFailedAsync( + It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + } + + [TestMethod] + public async Task LocalCancellationConflict_LeavesOperationRecoverable() + { + var fixture = new CoordinatorFixture(fileExists: true, candidateDownloaded: false); + fixture.DownloadClientProvider + .Setup(provider => provider.GetRequiredClient(fixture.CandidateInfo.DownloadType)) + .Throws(new InvalidOperationException("client unavailable")); + fixture.AnimationRepository.Setup(repository => repository.TryBeginCancelDownloadAsync( + fixture.CandidateInfo.Id, + It.IsAny(), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(false); + fixture.AnimationRepository.SetupSequence(repository => repository.FindByIdAsync( + fixture.CandidateInfo.Id, + It.IsAny())) + .ReturnsAsync(fixture.CandidateInfo) + .ReturnsAsync(fixture.CandidateInfo) + .ReturnsAsync(fixture.CandidateInfo with { IsDownloadTracked = true }); + + var result = await fixture.Coordinator.ExecuteAsync( + fixture.Candidate, + dryRun: false, + CancellationToken.None); + + Assert.AreEqual("recovery_pending", result.Outcome); + fixture.UpgradeRepository.Verify(repository => repository.MarkFailedAsync( + It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + } + + [TestMethod] + public async Task ActivationWinningBeforeCompensation_DoesNotDeleteRemoteCandidate() + { + var fixture = new CoordinatorFixture(fileExists: true, candidateDownloaded: false); + fixture.DownloadClient.Setup(client => client.SubmitDownloadTaskAsync( + fixture.CandidateInfo.Id, + fixture.CandidateInfo.DownloadUrl, + fixture.CandidateInfo.CachedDownloadData, + fixture.CandidateInfo.AdditionalDownloadInfo, + It.IsAny())) + .ThrowsAsync(new IOException("submission outcome unknown")); + fixture.AnimationRepository.Setup(repository => repository.TryBeginCancelDownloadAsync( + fixture.CandidateInfo.Id, + It.IsAny(), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(false); + fixture.AnimationRepository.SetupSequence(repository => repository.FindByIdAsync( + fixture.CandidateInfo.Id, + It.IsAny())) + .ReturnsAsync(fixture.CandidateInfo) + .ReturnsAsync(fixture.CandidateInfo) + .ReturnsAsync(fixture.CandidateInfo with { IsDownloadTracked = true }); + + var result = await fixture.Coordinator.ExecuteAsync( + fixture.Candidate, + dryRun: false, + CancellationToken.None); + + Assert.IsFalse(result.IsSuccess); + Assert.AreEqual("recovery_pending", result.Outcome); + fixture.DownloadClient.Verify(client => client.CancelDownloadTaskAsync( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny()), Times.Never); + fixture.FileMappingRepository.Verify(repository => repository.TryFinalizeDownloadCancellationAsync( + It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny()), Times.Never); + } + + [TestMethod] + public async Task RequestCancellation_AfterCompensationTerminatesOperation() + { + var fixture = new CoordinatorFixture(fileExists: true, candidateDownloaded: false); + fixture.DownloadClient.Setup(client => client.SubmitDownloadTaskAsync( + fixture.CandidateInfo.Id, + fixture.CandidateInfo.DownloadUrl, + fixture.CandidateInfo.CachedDownloadData, + fixture.CandidateInfo.AdditionalDownloadInfo, + It.IsAny())) + .ThrowsAsync(new OperationCanceledException()); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + await Assert.ThrowsExactlyAsync(() => + fixture.Coordinator.ExecuteAsync( + fixture.Candidate, + dryRun: false, + cancellation.Token)); + + fixture.AnimationRepository.Verify(repository => repository.TryBeginCancelDownloadAsync( + fixture.CandidateInfo.Id, + It.IsAny(), + It.IsAny(), + It.IsAny()), Times.Once); + fixture.FileMappingRepository.Verify(repository => repository.TryFinalizeDownloadCancellationAsync( + fixture.CandidateInfo.Id, + It.IsAny(), + It.IsAny(), + SubscriptionAutomationDisposition.AutoDownloadFailed, + It.IsAny()), Times.Once); + fixture.UpgradeRepository.Verify(repository => repository.MarkFailedAsync( + fixture.Operation.Id, + It.IsAny(), + It.IsAny()), Times.Once); + } + private sealed class CoordinatorFixture { public Mock UpgradeRepository { get; } = new(); + public Mock AnimationRepository { get; } = new(); + public Mock FileMappingRepository { get; } = new(); + public Mock DownloadClient { get; } = new(); + public Mock DownloadClientProvider { get; } = new(); public Mock FileStore { get; } = new(); public Mock IncidentReporter { get; } = new(); public ReleaseUpgradeOperation Operation { get; } + public ReleaseUpgradeCandidate Candidate { get; } + public AnimationInfo CandidateInfo { get; } public IReleaseUpgradeCoordinator Coordinator { get; } - public CoordinatorFixture(bool fileExists) + public CoordinatorFixture( + bool fileExists, + bool candidateDownloaded = true, + bool? candidateTracked = null, + ReleaseUpgradeStatus? operationStatus = null, + bool candidateMapped = true) { + var isCandidateTracked = candidateTracked ?? candidateDownloaded; Operation = new ReleaseUpgradeOperation( Guid.NewGuid(), Guid.NewGuid(), Guid.NewGuid(), - ReleaseUpgradeStatus.Verifying, 200, 500, + operationStatus ?? (candidateDownloaded + ? ReleaseUpgradeStatus.Verifying + : ReleaseUpgradeStatus.Downloading), + 200, 500, DateTimeOffset.UtcNow, null, null, null, null, null); + Candidate = new ReleaseUpgradeCandidate( + Operation.CurrentReleaseId, + Operation.CandidateReleaseId, + "candidate show", + 1, + 1, + 200, + 500, + [], + true); var previous = new FileMapping( Guid.NewGuid(), Operation.CurrentReleaseId, "/show/e01.mkv", "/store/old.mkv", "local"); var candidate = new FileMapping( @@ -79,15 +417,25 @@ public CoordinatorFixture(bool fileExists) .ReturnsAsync(Operation); UpgradeRepository.Setup(repository => repository.GetActivationAsync( Operation.CandidateReleaseId, It.IsAny())) - .ReturnsAsync(new ReleaseUpgradeActivation(Operation, [previous], [candidate])); + .ReturnsAsync(new ReleaseUpgradeActivation( + Operation, + [previous], + candidateMapped ? [candidate] : [])); UpgradeRepository.Setup(repository => repository.MarkFailedAsync( Operation.Id, It.IsAny(), It.IsAny())) .ReturnsAsync(new ReleaseUpgradeMutationResult(true, "failed", Operation)); UpgradeRepository.Setup(repository => repository.ActivateAsync( - Operation.Id, It.IsAny(), It.IsAny(), + Operation.Id, It.IsAny>(), + It.IsAny>(), + It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(new ReleaseUpgradeMutationResult(true, "applied", Operation with { Status = ReleaseUpgradeStatus.Applied })); + UpgradeRepository.Setup(repository => repository.TryBeginAsync( + Candidate, + It.IsAny(), + It.IsAny())) + .ReturnsAsync(Operation); FileStore.Setup(store => store.ExistAsync( candidate.PhysicalPath, It.IsAny())) @@ -99,21 +447,62 @@ public CoordinatorFixture(bool fileExists) storeProvider.Setup(provider => provider.GetRequiredClient("local")) .Returns(FileStore.Object); - var animationRepository = new Mock(); - animationRepository.Setup(repository => repository.FindByIdAsync( + CandidateInfo = new AnimationInfo( + Operation.CandidateReleaseId, "candidate", "", DateTimeOffset.UtcNow, + "https://example.test/new", FileDownloadTypes.TorrentDownload, [], "", + isCandidateTracked, default, default, candidateDownloaded, + candidateDownloaded ? "local" : null, + candidateDownloaded ? "/store/new" : null, + 1, 1, null, null, true, 0); + AnimationRepository.Setup(repository => repository.FindByIdAsync( Operation.CandidateReleaseId, It.IsAny())) - .ReturnsAsync(new AnimationInfo( - Operation.CandidateReleaseId, "candidate", "", DateTimeOffset.UtcNow, - "https://example.test/new", FileDownloadTypes.TorrentDownload, [], "", - true, default, default, true, "local", "/store/new", 1, 1, - null, null, true, 0)); + .ReturnsAsync(CandidateInfo); + AnimationRepository.Setup(repository => repository.TryStartDownloadAsync( + CandidateInfo.Id, + It.IsAny(), + It.IsAny(), + SubscriptionAutomationDisposition.AutoDownloadQueued, + It.IsAny())) + .ReturnsAsync(true); + AnimationRepository.Setup(repository => repository.TryBeginCancelDownloadAsync( + CandidateInfo.Id, + It.IsAny(), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(true); + FileMappingRepository.Setup(repository => repository.TryFinalizeDownloadCancellationAsync( + CandidateInfo.Id, + It.IsAny(), + It.IsAny(), + SubscriptionAutomationDisposition.AutoDownloadFailed, + It.IsAny())) + .ReturnsAsync(true); + + DownloadClientProvider + .Setup(provider => provider.GetRequiredClient(CandidateInfo.DownloadType)) + .Returns(DownloadClient.Object); + DownloadClient.Setup(client => client.SubmitDownloadTaskAsync( + CandidateInfo.Id, + CandidateInfo.DownloadUrl, + CandidateInfo.CachedDownloadData, + CandidateInfo.AdditionalDownloadInfo, + It.IsAny())) + .ReturnsAsync(true); + DownloadClient.Setup(client => client.CancelDownloadTaskAsync( + CandidateInfo.Id, + CandidateInfo.DownloadUrl, + CandidateInfo.CachedDownloadData, + CandidateInfo.AdditionalDownloadInfo, + false, + It.IsAny())) + .ReturnsAsync(new CancelDownloadResult(true, false)); Coordinator = new ReleaseUpgradeCoordinator( UpgradeRepository.Object, - animationRepository.Object, + AnimationRepository.Object, Mock.Of(), - Mock.Of(), - Mock.Of(), + FileMappingRepository.Object, + DownloadClientProvider.Object, storeProvider.Object, IncidentReporter.Object, Mock.Of>()); diff --git a/SecondDimensionWatcherReDive/Controllers/AnimationInfoController.cs b/SecondDimensionWatcherReDive/Controllers/AnimationInfoController.cs index 8e3ee5b..d601896 100644 --- a/SecondDimensionWatcherReDive/Controllers/AnimationInfoController.cs +++ b/SecondDimensionWatcherReDive/Controllers/AnimationInfoController.cs @@ -230,6 +230,7 @@ public async Task CancelDownload([FromRoute] Guid id, [FromQuery] id, info.DownloadAttemptId, cancellationAttemptId, + terminalDisposition: null, finalizeCancellation.Token); if (!cancelled) return Conflict(); diff --git a/SecondDimensionWatcherReDive/Migrations/20260830030724_EnforceSingleActiveEpisodeRelease.Designer.cs b/SecondDimensionWatcherReDive/Migrations/20260830030724_EnforceSingleActiveEpisodeRelease.Designer.cs new file mode 100644 index 0000000..24c1773 --- /dev/null +++ b/SecondDimensionWatcherReDive/Migrations/20260830030724_EnforceSingleActiveEpisodeRelease.Designer.cs @@ -0,0 +1,1199 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using SecondDimensionWatcherReDive.Models; + +#nullable disable + +namespace SecondDimensionWatcherReDive.Migrations +{ + [DbContext(typeof(ApplicationContext))] + [Migration("20260830030724_EnforceSingleActiveEpisodeRelease")] + partial class EnforceSingleActiveEpisodeRelease + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.Animation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("OriginalName") + .IsRequired() + .HasColumnType("text"); + + b.Property("PosterPath") + .HasColumnType("text"); + + b.Property("TmdbId") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("TmdbId") + .IsUnique(); + + b.ToTable("Animations"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.AnimationGroup", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("AnimationGroups"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.AnimationInfo", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AdditionalDownloadInfo") + .IsRequired() + .HasColumnType("text"); + + b.Property("AiRetryCount") + .HasColumnType("integer"); + + b.Property("AnimationId") + .HasColumnType("uuid"); + + b.Property("AutomationDisposition") + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("AutomationExplanationJson") + .HasColumnType("text"); + + b.Property("CachedDownloadData") + .IsRequired() + .HasColumnType("bytea"); + + b.Property("CurrentMetadataReviewOperationId") + .HasColumnType("uuid"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text"); + + b.Property("DownloadAttemptId") + .HasColumnType("uuid"); + + b.Property("DownloadCancellationId") + .HasColumnType("uuid"); + + b.Property("DownloadEndTime") + .HasColumnType("timestamp with time zone"); + + b.Property("DownloadStartTime") + .HasColumnType("timestamp with time zone"); + + b.Property("DownloadType") + .IsRequired() + .HasColumnType("text"); + + b.Property("DownloadUrl") + .IsRequired() + .HasColumnType("text"); + + b.Property("EnclosureId") + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("Episode") + .HasColumnType("integer"); + + b.Property("ExpectedEpisodeCount") + .HasColumnType("integer"); + + b.Property("FeedItemGuid") + .HasMaxLength(1024) + .HasColumnType("character varying(1024)"); + + b.Property("FileStore") + .HasColumnType("text"); + + b.Property("GroupId") + .HasColumnType("uuid"); + + b.Property("IngestedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("IsActiveRelease") + .HasColumnType("boolean"); + + b.Property("IsAiProcessed") + .HasColumnType("boolean"); + + b.Property("IsDownloadFinished") + .HasColumnType("boolean"); + + b.Property("IsDownloadTracked") + .HasColumnType("boolean"); + + b.Property("MediaLibraryMissingSince") + .HasColumnType("timestamp with time zone"); + + b.Property("MediaLibrarySourceId") + .HasColumnType("uuid"); + + b.Property("MetadataConfidence") + .HasColumnType("double precision"); + + b.Property("MetadataLastError") + .HasMaxLength(1024) + .HasColumnType("character varying(1024)"); + + b.Property("MetadataReviewedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("MetadataStatus") + .HasColumnType("integer"); + + b.Property("PublishTime") + .HasColumnType("timestamp with time zone"); + + b.Property("ReleaseCodec") + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("ReleaseIdentity") + .HasMaxLength(192) + .HasColumnType("character varying(192)"); + + b.PrimitiveCollection("ReleaseLanguages") + .IsRequired() + .HasColumnType("text[]"); + + b.Property("ReleaseResolution") + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("ReleaseScore") + .HasColumnType("integer"); + + b.Property("ReleaseScoreReasonsJson") + .HasColumnType("text"); + + b.Property("ReleaseSizeBytes") + .HasColumnType("bigint"); + + b.Property("ReleaseSubtitleGroup") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("Season") + .HasColumnType("integer"); + + b.Property("SourceFeedId") + .HasColumnType("uuid"); + + b.Property("StateVersion") + .IsConcurrencyToken() + .HasColumnType("bigint"); + + b.Property("StorePath") + .HasColumnType("text"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text"); + + b.Property("TorrentInfoHash") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.HasKey("Id"); + + b.HasIndex("CurrentMetadataReviewOperationId") + .IsUnique(); + + b.HasIndex("GroupId"); + + b.HasIndex("MediaLibrarySourceId"); + + b.HasIndex("ReleaseIdentity") + .IsUnique() + .HasDatabaseName("UX_AnimationInfo_ReleaseIdentity") + .HasFilter("\"ReleaseIdentity\" IS NOT NULL"); + + b.HasIndex("SourceFeedId"); + + b.HasIndex("AnimationId"); + + b.HasIndex("FileStore", "StorePath") + .IsUnique() + .HasFilter("\"DownloadType\" = 'http://schemas.hcgstudio.com/ws/2023/06/sdw/downloadtype/media-library-import'"); + + b.HasIndex("MetadataStatus", "PublishTime"); + + b.HasIndex("AnimationId", "Season", "Episode") + .IsUnique() + .HasDatabaseName("UX_AnimationInfo_ActiveEpisodeRelease") + .HasFilter("\"IsActiveRelease\" = TRUE AND \"AnimationId\" IS NOT NULL AND \"Season\" IS NOT NULL AND \"Episode\" IS NOT NULL"); + + b.HasIndex("Season", "Episode", "ReleaseScore"); + + b.ToTable("AnimationInfo", t => + { + t.HasCheckConstraint("CK_AnimationInfo_ExpectedEpisodeCount_Positive", "\"ExpectedEpisodeCount\" IS NULL OR \"ExpectedEpisodeCount\" > 0"); + + t.HasCheckConstraint("CK_AnimationInfo_MetadataConfidence_Range", "\"MetadataConfidence\" IS NULL OR (\"MetadataConfidence\" >= 0 AND \"MetadataConfidence\" <= 1)"); + + t.HasCheckConstraint("CK_AnimationInfo_ReleaseScore_NonNegative", "\"ReleaseScore\" >= 0"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ApplicationSettings", b => + { + b.Property("Id") + .HasColumnType("integer"); + + b.Property("ProtectedSecrets") + .HasColumnType("text"); + + b.Property("Revision") + .IsConcurrencyToken() + .HasColumnType("bigint"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("ValuesJson") + .IsRequired() + .HasColumnType("jsonb"); + + b.HasKey("Id"); + + b.ToTable("ApplicationSettings", t => + { + t.HasCheckConstraint("CK_ApplicationSettings_Revision_Positive", "\"Revision\" > 0"); + + t.HasCheckConstraint("CK_ApplicationSettings_Singleton", "\"Id\" = 1"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.BangumiSubgroup", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("MikanSubgroupId") + .HasColumnType("integer"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("ScrapedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("SeasonBangumiId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("SeasonBangumiId", "MikanSubgroupId") + .IsUnique(); + + b.ToTable("BangumiSubgroups"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ChatConversation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Title") + .HasColumnType("text"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.ToTable("ChatConversations"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ChatMessage", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Content") + .HasColumnType("text"); + + b.Property("ConversationId") + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Order") + .HasColumnType("integer"); + + b.Property("Role") + .IsRequired() + .HasColumnType("text"); + + b.Property("ToolCallId") + .HasColumnType("text"); + + b.Property("ToolCallsJson") + .HasColumnType("text"); + + b.Property("ToolName") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("ConversationId"); + + b.ToTable("ChatMessages"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.Feed", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Name") + .HasColumnType("text"); + + b.Property("Url") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("Feeds"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.FileMapping", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AnimationInfoId") + .HasColumnType("uuid"); + + b.Property("FileStore") + .IsRequired() + .HasColumnType("text"); + + b.Property("PhysicalPath") + .IsRequired() + .HasColumnType("text"); + + b.Property("VirtualPath") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("AnimationInfoId"); + + b.HasIndex("VirtualPath") + .IsUnique(); + + b.ToTable("FileMappings"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.FileNameRegexRule", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AnimationId") + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("Pattern") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("character varying(512)"); + + b.HasKey("Id"); + + b.HasIndex("AnimationId", "CreatedAt"); + + b.HasIndex("AnimationId", "Pattern") + .IsUnique(); + + b.ToTable("FileNameRegexRules"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.Incident", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Detail") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("DetectedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Fingerprint") + .IsRequired() + .HasMaxLength(96) + .HasColumnType("character varying(96)"); + + b.Property("LastRetryAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastRetryError") + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("ResolvedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("RetryCount") + .HasColumnType("integer"); + + b.Property("Severity") + .HasColumnType("integer"); + + b.Property("SourceId") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("Title") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("Type") + .HasColumnType("integer"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("Fingerprint") + .IsUnique(); + + b.HasIndex("ResolvedAt", "Type", "UpdatedAt"); + + b.ToTable("Incidents"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MediaLibrarySource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("IsMonitoring") + .HasColumnType("boolean"); + + b.Property("LastError") + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("LastImportedCount") + .HasColumnType("integer"); + + b.Property("LastRemovedCount") + .HasColumnType("integer"); + + b.Property("LastScanAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastSkippedCount") + .HasColumnType("integer"); + + b.Property("LastUpdatedCount") + .HasColumnType("integer"); + + b.Property("Path") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Path") + .IsUnique(); + + b.ToTable("MediaLibrarySources"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewMappingSnapshot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("FileStore") + .IsRequired() + .HasColumnType("text"); + + b.Property("Kind") + .HasColumnType("integer"); + + b.Property("OperationId") + .HasColumnType("uuid"); + + b.Property("PhysicalPath") + .IsRequired() + .HasColumnType("text"); + + b.Property("VirtualPath") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("OperationId", "Kind", "VirtualPath") + .IsUnique(); + + b.ToTable("MetadataReviewMappingSnapshots"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AnimationInfoId") + .HasColumnType("uuid"); + + b.Property("AppliedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("AppliedVersion") + .HasColumnType("bigint"); + + b.Property("BaseFileStore") + .HasColumnType("text"); + + b.Property("BaseIsDownloadFinished") + .HasColumnType("boolean"); + + b.Property("BaseStorePath") + .HasColumnType("text"); + + b.Property("BaseVersion") + .HasColumnType("bigint"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("ExpiresAt") + .HasColumnType("timestamp with time zone"); + + b.Property("PreviousAiRetryCount") + .HasColumnType("integer"); + + b.Property("PreviousAnimationId") + .HasColumnType("uuid"); + + b.Property("PreviousConfidence") + .HasColumnType("double precision"); + + b.Property("PreviousCurrentOperationId") + .HasColumnType("uuid"); + + b.Property("PreviousDescription") + .HasColumnType("text"); + + b.Property("PreviousEpisode") + .HasColumnType("integer"); + + b.Property("PreviousGroupId") + .HasColumnType("uuid"); + + b.Property("PreviousIsAiProcessed") + .HasColumnType("boolean"); + + b.Property("PreviousLastError") + .HasColumnType("text"); + + b.Property("PreviousMetadataStatus") + .HasColumnType("integer"); + + b.Property("PreviousReviewedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("PreviousSeason") + .HasColumnType("integer"); + + b.Property("ProposedAnimationName") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProposedAnimationOriginalName") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProposedAnimationPosterPath") + .HasColumnType("text"); + + b.Property("ProposedAnimationTmdbId") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProposedDescription") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProposedEpisode") + .HasColumnType("integer"); + + b.Property("ProposedGroupName") + .HasColumnType("text"); + + b.Property("ProposedSeason") + .HasColumnType("integer"); + + b.Property("State") + .HasColumnType("integer"); + + b.Property("UndoneAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("AnimationInfoId", "AppliedVersion") + .IsUnique(); + + b.HasIndex("AnimationInfoId", "State"); + + b.HasIndex("State", "ExpiresAt"); + + b.ToTable("MetadataReviewOperations", t => + { + t.HasCheckConstraint("CK_MetadataReviewOperations_Expiry", "\"ExpiresAt\" > \"CreatedAt\""); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MigrationMarker", b => + { + b.Property("Key") + .HasColumnType("text"); + + b.Property("AppliedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Key"); + + b.ToTable("MigrationMarkers"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.PlaybackPreference", b => + { + b.Property("UserId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AudioLanguage") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("AudioTrackLabel") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("AutoPlayNext") + .HasColumnType("boolean"); + + b.Property("SubtitleLanguage") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("SubtitleTrackLabel") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("UserId"); + + b.ToTable("PlaybackPreferences"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.PlaybackProgress", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AnimationInfoId") + .HasColumnType("uuid"); + + b.Property("DurationSeconds") + .HasColumnType("double precision"); + + b.Property("IsWatched") + .HasColumnType("boolean"); + + b.Property("PositionSeconds") + .HasColumnType("double precision"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.Property("VirtualPath") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("WatchedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("AnimationInfoId"); + + b.HasIndex("UserId", "AnimationInfoId", "VirtualPath") + .IsUnique(); + + b.HasIndex("UserId", "IsWatched", "UpdatedAt"); + + b.ToTable("PlaybackProgresses", t => + { + t.HasCheckConstraint("CK_PlaybackProgresses_Duration_NonNegative", "\"DurationSeconds\" >= 0"); + + t.HasCheckConstraint("CK_PlaybackProgresses_Position_NonNegative", "\"PositionSeconds\" >= 0"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ReleaseUpgradeMappingSnapshot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AnimationInfoId") + .HasColumnType("uuid"); + + b.Property("FileStore") + .IsRequired() + .HasColumnType("text"); + + b.Property("Kind") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("OperationId") + .HasColumnType("uuid"); + + b.Property("OriginalMappingId") + .HasColumnType("uuid"); + + b.Property("PhysicalPath") + .IsRequired() + .HasColumnType("text"); + + b.Property("VirtualPath") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.HasKey("Id"); + + b.HasIndex("OperationId", "Kind", "OriginalMappingId") + .IsUnique(); + + b.ToTable("ReleaseUpgradeMappingSnapshots"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ReleaseUpgradeOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AppliedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CandidateReleaseId") + .HasColumnType("uuid"); + + b.Property("CandidateScore") + .HasColumnType("integer"); + + b.Property("CompletedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CurrentReleaseId") + .HasColumnType("uuid"); + + b.Property("CurrentScore") + .HasColumnType("integer"); + + b.Property("FailureSummary") + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("RollbackUntil") + .HasColumnType("timestamp with time zone"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("VerifiedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("CandidateReleaseId") + .IsUnique() + .HasFilter("\"Status\" <> 'Failed'"); + + b.HasIndex("CurrentReleaseId") + .IsUnique() + .HasDatabaseName("UX_ReleaseUpgradeOperations_ActiveCurrentRelease") + .HasFilter("\"Status\" IN ('Downloading', 'Verifying', 'Applied')"); + + b.HasIndex("Status", "CreatedAt"); + + b.ToTable("ReleaseUpgradeOperations", t => + { + t.HasCheckConstraint("CK_ReleaseUpgradeOperations_ScoreIncrease", "\"CandidateScore\" > \"CurrentScore\""); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SeasonBangumi", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("DayOfWeek") + .HasColumnType("integer"); + + b.Property("ImageUrl") + .HasColumnType("text"); + + b.Property("MikanId") + .HasColumnType("integer"); + + b.Property("ScrapedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("MikanId") + .IsUnique(); + + b.ToTable("SeasonBangumis"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SubscriptionAutomationPolicy", b => + { + b.Property("FeedId") + .HasColumnType("uuid"); + + b.PrimitiveCollection("Codecs") + .IsRequired() + .HasColumnType("text[]"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("EnableVersionUpgrade") + .HasColumnType("boolean"); + + b.PrimitiveCollection("ExcludedKeywords") + .IsRequired() + .HasColumnType("text[]"); + + b.PrimitiveCollection("Languages") + .IsRequired() + .HasColumnType("text[]"); + + b.Property("MaxSizeBytes") + .HasColumnType("bigint"); + + b.Property("MinSizeBytes") + .HasColumnType("bigint"); + + b.Property("MinimumUpgradeScore") + .HasColumnType("integer"); + + b.Property("Mode") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.PrimitiveCollection("Resolutions") + .IsRequired() + .HasColumnType("text[]"); + + b.PrimitiveCollection("SubtitleGroups") + .IsRequired() + .HasColumnType("text[]"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("UpgradeRollbackHours") + .HasColumnType("integer"); + + b.HasKey("FeedId"); + + b.HasIndex("UpdatedAt"); + + b.ToTable("SubscriptionAutomationPolicies", t => + { + t.HasCheckConstraint("CK_SubscriptionAutomationPolicies_MinimumUpgradeScore", "\"MinimumUpgradeScore\" >= 1 AND \"MinimumUpgradeScore\" <= 1000"); + + t.HasCheckConstraint("CK_SubscriptionAutomationPolicies_UpgradeRollbackHours", "\"UpgradeRollbackHours\" >= 1 AND \"UpgradeRollbackHours\" <= 720"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.WebDavToken", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("TokenHash") + .IsRequired() + .HasColumnType("text"); + + b.Property("Username") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Username") + .IsUnique(); + + b.ToTable("WebDavTokens"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.AnimationInfo", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.Animation", "Animation") + .WithMany() + .HasForeignKey("AnimationId"); + + b.HasOne("SecondDimensionWatcherReDive.Models.AnimationGroup", "Group") + .WithMany() + .HasForeignKey("GroupId"); + + b.HasOne("SecondDimensionWatcherReDive.Models.MediaLibrarySource", null) + .WithMany() + .HasForeignKey("MediaLibrarySourceId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("SecondDimensionWatcherReDive.Models.Feed", null) + .WithMany() + .HasForeignKey("SourceFeedId") + .OnDelete(DeleteBehavior.SetNull); + + b.Navigation("Animation"); + + b.Navigation("Group"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.BangumiSubgroup", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.SeasonBangumi", "SeasonBangumi") + .WithMany("Subgroups") + .HasForeignKey("SeasonBangumiId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("SeasonBangumi"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ChatMessage", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.ChatConversation", "Conversation") + .WithMany("Messages") + .HasForeignKey("ConversationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Conversation"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.FileNameRegexRule", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.Animation", null) + .WithMany() + .HasForeignKey("AnimationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewMappingSnapshot", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.MetadataReviewOperation", "Operation") + .WithMany("MappingSnapshots") + .HasForeignKey("OperationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Operation"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewOperation", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.AnimationInfo", "AnimationInfo") + .WithMany() + .HasForeignKey("AnimationInfoId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AnimationInfo"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.PlaybackProgress", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.AnimationInfo", "AnimationInfo") + .WithMany() + .HasForeignKey("AnimationInfoId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AnimationInfo"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ReleaseUpgradeMappingSnapshot", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.ReleaseUpgradeOperation", "Operation") + .WithMany("MappingSnapshots") + .HasForeignKey("OperationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Operation"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ReleaseUpgradeOperation", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.AnimationInfo", "CandidateRelease") + .WithMany() + .HasForeignKey("CandidateReleaseId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("SecondDimensionWatcherReDive.Models.AnimationInfo", "CurrentRelease") + .WithMany() + .HasForeignKey("CurrentReleaseId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("CandidateRelease"); + + b.Navigation("CurrentRelease"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SubscriptionAutomationPolicy", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.Feed", "Feed") + .WithOne() + .HasForeignKey("SecondDimensionWatcherReDive.Models.SubscriptionAutomationPolicy", "FeedId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Feed"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ChatConversation", b => + { + b.Navigation("Messages"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewOperation", b => + { + b.Navigation("MappingSnapshots"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ReleaseUpgradeOperation", b => + { + b.Navigation("MappingSnapshots"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SeasonBangumi", b => + { + b.Navigation("Subgroups"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/SecondDimensionWatcherReDive/Migrations/20260830030724_EnforceSingleActiveEpisodeRelease.cs b/SecondDimensionWatcherReDive/Migrations/20260830030724_EnforceSingleActiveEpisodeRelease.cs new file mode 100644 index 0000000..90005a1 --- /dev/null +++ b/SecondDimensionWatcherReDive/Migrations/20260830030724_EnforceSingleActiveEpisodeRelease.cs @@ -0,0 +1,89 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace SecondDimensionWatcherReDive.Migrations +{ + /// + public partial class EnforceSingleActiveEpisodeRelease : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropIndex( + name: "IX_ReleaseUpgradeOperations_CandidateReleaseId", + table: "ReleaseUpgradeOperations"); + + migrationBuilder.Sql( + """ + WITH ranked_active_releases AS ( + SELECT info."Id", + row_number() OVER ( + PARTITION BY info."AnimationId", info."Season", info."Episode" + ORDER BY info."IngestedAt", info."PublishTime", info."Id" + ) AS rank + FROM "AnimationInfo" info + WHERE info."IsActiveRelease" = TRUE + AND info."AnimationId" IS NOT NULL + AND info."Season" IS NOT NULL + AND info."Episode" IS NOT NULL + ) + UPDATE "AnimationInfo" info + SET "IsActiveRelease" = FALSE + FROM ranked_active_releases ranked + WHERE info."Id" = ranked."Id" + AND ranked.rank > 1; + """); + + migrationBuilder.CreateIndex( + name: "UX_AnimationInfo_ActiveEpisodeRelease", + table: "AnimationInfo", + columns: new[] { "AnimationId", "Season", "Episode" }, + unique: true, + filter: "\"IsActiveRelease\" = TRUE AND \"AnimationId\" IS NOT NULL AND \"Season\" IS NOT NULL AND \"Episode\" IS NOT NULL"); + + migrationBuilder.CreateIndex( + name: "IX_ReleaseUpgradeOperations_CandidateReleaseId", + table: "ReleaseUpgradeOperations", + column: "CandidateReleaseId", + unique: true, + filter: "\"Status\" <> 'Failed'"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropIndex( + name: "UX_AnimationInfo_ActiveEpisodeRelease", + table: "AnimationInfo"); + + migrationBuilder.DropIndex( + name: "IX_ReleaseUpgradeOperations_CandidateReleaseId", + table: "ReleaseUpgradeOperations"); + + migrationBuilder.Sql( + """ + WITH ranked_candidate_operations AS ( + SELECT operation."Id", + row_number() OVER ( + PARTITION BY operation."CandidateReleaseId" + ORDER BY (operation."Status" <> 'Failed') DESC, + operation."CreatedAt" DESC, + operation."Id" + ) AS rank + FROM "ReleaseUpgradeOperations" operation + ) + DELETE FROM "ReleaseUpgradeOperations" operation + USING ranked_candidate_operations ranked + WHERE operation."Id" = ranked."Id" + AND ranked.rank > 1; + """); + + migrationBuilder.CreateIndex( + name: "IX_ReleaseUpgradeOperations_CandidateReleaseId", + table: "ReleaseUpgradeOperations", + column: "CandidateReleaseId", + unique: true); + } + } +} diff --git a/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs b/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs index 78612ac..7ea80e1 100644 --- a/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs +++ b/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs @@ -234,8 +234,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasKey("Id"); - b.HasIndex("AnimationId"); - b.HasIndex("CurrentMetadataReviewOperationId") .IsUnique(); @@ -250,12 +248,19 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasIndex("SourceFeedId"); + b.HasIndex("AnimationId"); + b.HasIndex("FileStore", "StorePath") .IsUnique() .HasFilter("\"DownloadType\" = 'http://schemas.hcgstudio.com/ws/2023/06/sdw/downloadtype/media-library-import'"); b.HasIndex("MetadataStatus", "PublishTime"); + b.HasIndex("AnimationId", "Season", "Episode") + .IsUnique() + .HasDatabaseName("UX_AnimationInfo_ActiveEpisodeRelease") + .HasFilter("\"IsActiveRelease\" = TRUE AND \"AnimationId\" IS NOT NULL AND \"Season\" IS NOT NULL AND \"Episode\" IS NOT NULL"); + b.HasIndex("Season", "Episode", "ReleaseScore"); b.ToTable("AnimationInfo", t => @@ -899,7 +904,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasKey("Id"); b.HasIndex("CandidateReleaseId") - .IsUnique(); + .IsUnique() + .HasFilter("\"Status\" <> 'Failed'"); b.HasIndex("CurrentReleaseId") .IsUnique() diff --git a/SecondDimensionWatcherReDive/Models/AnimationInfo.cs b/SecondDimensionWatcherReDive/Models/AnimationInfo.cs index f929279..dc4b7ef 100644 --- a/SecondDimensionWatcherReDive/Models/AnimationInfo.cs +++ b/SecondDimensionWatcherReDive/Models/AnimationInfo.cs @@ -93,5 +93,5 @@ public class AnimationInfo public int? ExpectedEpisodeCount { get; set; } - public bool IsActiveRelease { get; set; } = true; + public bool IsActiveRelease { get; set; } } diff --git a/SecondDimensionWatcherReDive/Models/ApplicationContext.cs b/SecondDimensionWatcherReDive/Models/ApplicationContext.cs index dab05a9..30b6ea0 100644 --- a/SecondDimensionWatcherReDive/Models/ApplicationContext.cs +++ b/SecondDimensionWatcherReDive/Models/ApplicationContext.cs @@ -319,6 +319,18 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) modelBuilder.Entity() .HasIndex(info => info.SourceFeedId); + modelBuilder.Entity() + .HasIndex("AnimationId") + .HasDatabaseName("IX_AnimationInfo_AnimationId"); + + modelBuilder.Entity() + .HasIndex("AnimationId", "Season", "Episode") + .IsUnique() + .HasFilter( + "\"IsActiveRelease\" = TRUE AND \"AnimationId\" IS NOT NULL " + + "AND \"Season\" IS NOT NULL AND \"Episode\" IS NOT NULL") + .HasDatabaseName("UX_AnimationInfo_ActiveEpisodeRelease"); + modelBuilder.Entity() .HasKey(policy => policy.FeedId); @@ -370,7 +382,8 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) modelBuilder.Entity() .HasIndex(operation => operation.CandidateReleaseId) - .IsUnique(); + .IsUnique() + .HasFilter("\"Status\" <> 'Failed'"); modelBuilder.Entity() .HasIndex(operation => operation.CurrentReleaseId) diff --git a/SecondDimensionWatcherReDive/Repositories/AnimationInfoRepository.cs b/SecondDimensionWatcherReDive/Repositories/AnimationInfoRepository.cs index 2bfa7e0..771c0fc 100644 --- a/SecondDimensionWatcherReDive/Repositories/AnimationInfoRepository.cs +++ b/SecondDimensionWatcherReDive/Repositories/AnimationInfoRepository.cs @@ -241,12 +241,21 @@ public async Task RemoveMediaLibraryEntryAsync( || entity.MediaLibrarySourceId != expectedSourceId || entity.DownloadType != FileDownloadTypes.MediaLibraryImport) return false; + var previousEpisodeIdentity = GetEpisodeIdentity(writeContext, entity); + var wasActiveRelease = entity.IsActiveRelease; await writeContext.FileMappings .Where(mapping => mapping.AnimationInfoId == id) .ExecuteDeleteAsync(cancellationToken); writeContext.AnimationInfo.Remove(entity); await writeContext.SaveChangesAsync(cancellationToken); + await PromotePreviousEpisodeSuccessorAsync( + writeContext, + entity.Id, + wasActiveRelease, + previousEpisodeIdentity, + currentIdentity: null, + cancellationToken); await transaction.CommitAsync(cancellationToken); return true; }); @@ -338,10 +347,10 @@ public async Task TryAddReleaseAsync( return true; } catch (DbUpdateException exception) when (exception.InnerException is PostgresException - { - SqlState: PostgresErrorCodes.UniqueViolation, - ConstraintName: "UX_AnimationInfo_ReleaseIdentity" - }) + { + SqlState: PostgresErrorCodes.UniqueViolation, + ConstraintName: "UX_AnimationInfo_ReleaseIdentity" + }) { context.Entry(entity).State = EntityState.Detached; return false; @@ -350,24 +359,48 @@ public async Task TryAddReleaseAsync( public async Task UpdateAsync(AnimationInfo info, CancellationToken cancellationToken) { - var entity = await context.AnimationInfo.FindAsync([info.Id], cancellationToken) - ?? throw new InvalidOperationException($"AnimationInfo {info.Id} not found"); - var currentStateVersion = entity.StateVersion; - if (currentStateVersion != info.StateVersion) - throw new DbUpdateConcurrencyException( - $"AnimationInfo {info.Id} changed from revision {info.StateVersion} to {currentStateVersion}."); - - info.ApplyTo(entity); - - entity.Animation = info.Animation is null - ? null - : await context.Animations.FindAsync([info.Animation.Id], cancellationToken); - entity.Group = info.Group is null - ? null - : await context.AnimationGroups.FindAsync([info.Group.Id], cancellationToken); - entity.StateVersion = checked(currentStateVersion + 1); + var strategy = context.Database.CreateExecutionStrategy(); + await strategy.ExecuteAsync(async () => + { + await using var writeContext = new Models.ApplicationContext(contextOptions); + await using var transaction = await writeContext.Database + .BeginTransactionAsync(cancellationToken); + await MappingTransactionLock.AcquireAsync(writeContext, cancellationToken); + var entity = await MappingTransactionLock.LockAnimationInfoAsync( + writeContext, + info.Id, + cancellationToken) + ?? throw new InvalidOperationException($"AnimationInfo {info.Id} not found"); + var currentStateVersion = entity.StateVersion; + if (currentStateVersion != info.StateVersion) + throw new DbUpdateConcurrencyException( + $"AnimationInfo {info.Id} changed from revision {info.StateVersion} to {currentStateVersion}."); + var previousEpisodeIdentity = GetEpisodeIdentity(writeContext, entity); + var wasActiveRelease = entity.IsActiveRelease; + + info.ApplyTo(entity); + entity.Animation = info.Animation is null + ? null + : await writeContext.Animations.FindAsync([info.Animation.Id], cancellationToken); + entity.Group = info.Group is null + ? null + : await writeContext.AnimationGroups.FindAsync([info.Group.Id], cancellationToken); + writeContext.Entry(entity).Property("AnimationId").CurrentValue = info.Animation?.Id; + writeContext.Entry(entity).Property("GroupId").CurrentValue = info.Group?.Id; + await SetEpisodeReleaseActivityAsync(writeContext, entity, cancellationToken); + var currentEpisodeIdentity = GetEpisodeIdentity(writeContext, entity); + entity.StateVersion = checked(currentStateVersion + 1); - await context.SaveChangesAsync(cancellationToken); + await writeContext.SaveChangesAsync(cancellationToken); + await PromotePreviousEpisodeSuccessorAsync( + writeContext, + entity.Id, + wasActiveRelease, + previousEpisodeIdentity, + currentEpisodeIdentity, + cancellationToken); + await transaction.CommitAsync(cancellationToken); + }); } public async Task TryStartDownloadAsync( @@ -383,6 +416,7 @@ public async Task TryStartDownloadAsync( await using var writeContext = new Models.ApplicationContext(contextOptions); await using var transaction = await writeContext.Database .BeginTransactionAsync(cancellationToken); + await MappingTransactionLock.AcquireAsync(writeContext, cancellationToken); var entity = await MappingTransactionLock.LockAnimationInfoAsync( writeContext, id, @@ -436,6 +470,7 @@ public async Task TryBeginCancelDownloadAsync( await using var writeContext = new Models.ApplicationContext(contextOptions); await using var transaction = await writeContext.Database .BeginTransactionAsync(cancellationToken); + await MappingTransactionLock.AcquireAsync(writeContext, cancellationToken); var entity = await MappingTransactionLock.LockAnimationInfoAsync( writeContext, id, @@ -445,16 +480,41 @@ public async Task TryBeginCancelDownloadAsync( || entity.DownloadAttemptId != downloadAttemptId) return false; - if (entity.DownloadCancellationId == cancellationAttemptId) + if (entity.DownloadCancellationId is not null && + entity.DownloadCancellationId != cancellationAttemptId) + return false; + + // If activation acquired the shared mapping lock first, this is a + // stale cancellation request for a release that is now live. Do + // not let the caller delete its remote files after activation. + if (await writeContext.ReleaseUpgradeOperations.AnyAsync( + operation => operation.CandidateReleaseId == entity.Id && + operation.Status == ReleaseUpgradeStatus.Applied, + cancellationToken)) + return false; + + if (entity.DownloadCancellationId is null) { - await transaction.CommitAsync(cancellationToken); - return true; + entity.DownloadCancellationId = cancellationAttemptId; + entity.StateVersion = checked(entity.StateVersion + 1); } - if (entity.DownloadCancellationId is not null) - return false; - entity.DownloadCancellationId = cancellationAttemptId; - entity.StateVersion = checked(entity.StateVersion + 1); + // Persist the cancellation intent and terminate the pending + // upgrade atomically. Activation uses the same global lock, so it + // can no longer commit between remote deletion and local finalize. + var cancelledAt = DateTimeOffset.UtcNow; + await writeContext.ReleaseUpgradeOperations + .Where(operation => operation.CandidateReleaseId == entity.Id && + (operation.Status == ReleaseUpgradeStatus.Downloading || + operation.Status == ReleaseUpgradeStatus.Verifying)) + .ExecuteUpdateAsync( + setters => setters + .SetProperty(operation => operation.Status, ReleaseUpgradeStatus.Failed) + .SetProperty( + operation => operation.FailureSummary, + "Candidate download cancellation was requested before upgrade activation.") + .SetProperty(operation => operation.CompletedAt, cancelledAt), + cancellationToken); await writeContext.SaveChangesAsync(cancellationToken); await transaction.CommitAsync(cancellationToken); return true; @@ -542,6 +602,7 @@ await writeContext.Entry(entity) await using var writeContext = new Models.ApplicationContext(contextOptions); await using var transaction = await writeContext.Database .BeginTransactionAsync(cancellationToken); + await MappingTransactionLock.AcquireAsync(writeContext, cancellationToken); var entity = await MappingTransactionLock.LockAnimationInfoAsync( writeContext, id, @@ -567,6 +628,19 @@ SubscriptionAutomationDisposition.ManualDownloadQueued or ? SubscriptionAutomationDisposition.DownloadCancelled : entity.AutomationDisposition); entity.StateVersion = checked(entity.StateVersion + 1); + var cancelledAt = DateTimeOffset.UtcNow; + await writeContext.ReleaseUpgradeOperations + .Where(operation => operation.CandidateReleaseId == entity.Id && + (operation.Status == ReleaseUpgradeStatus.Downloading || + operation.Status == ReleaseUpgradeStatus.Verifying)) + .ExecuteUpdateAsync( + setters => setters + .SetProperty(operation => operation.Status, ReleaseUpgradeStatus.Failed) + .SetProperty( + operation => operation.FailureSummary, + "Candidate download was cancelled before upgrade activation.") + .SetProperty(operation => operation.CompletedAt, cancelledAt), + cancellationToken); await writeContext.SaveChangesAsync(cancellationToken); } else if (entity.DownloadAttemptId is not null @@ -591,30 +665,130 @@ public async Task TryUpdateAsync( long expectedStateVersion, CancellationToken cancellationToken) { - var entity = await context.AnimationInfo - .FirstOrDefaultAsync(candidate => candidate.Id == info.Id, cancellationToken); - if (entity is null || entity.StateVersion != expectedStateVersion) - return false; - - info.ApplyTo(entity); - entity.Animation = info.Animation is null - ? null - : await context.Animations.FindAsync([info.Animation.Id], cancellationToken); - entity.Group = info.Group is null - ? null - : await context.AnimationGroups.FindAsync([info.Group.Id], cancellationToken); - entity.StateVersion = checked(expectedStateVersion + 1); - context.Entry(entity).Property(candidate => candidate.StateVersion).OriginalValue = expectedStateVersion; - - try + var strategy = context.Database.CreateExecutionStrategy(); + return await strategy.ExecuteAsync(async () => { - await context.SaveChangesAsync(cancellationToken); + await using var writeContext = new Models.ApplicationContext(contextOptions); + await using var transaction = await writeContext.Database + .BeginTransactionAsync(cancellationToken); + await MappingTransactionLock.AcquireAsync(writeContext, cancellationToken); + var entity = await MappingTransactionLock.LockAnimationInfoAsync( + writeContext, + info.Id, + cancellationToken); + if (entity is null || entity.StateVersion != expectedStateVersion) + return false; + var previousEpisodeIdentity = GetEpisodeIdentity(writeContext, entity); + var wasActiveRelease = entity.IsActiveRelease; + + info.ApplyTo(entity); + entity.Animation = info.Animation is null + ? null + : await writeContext.Animations.FindAsync([info.Animation.Id], cancellationToken); + entity.Group = info.Group is null + ? null + : await writeContext.AnimationGroups.FindAsync([info.Group.Id], cancellationToken); + writeContext.Entry(entity).Property("AnimationId").CurrentValue = info.Animation?.Id; + writeContext.Entry(entity).Property("GroupId").CurrentValue = info.Group?.Id; + await SetEpisodeReleaseActivityAsync(writeContext, entity, cancellationToken); + var currentEpisodeIdentity = GetEpisodeIdentity(writeContext, entity); + entity.StateVersion = checked(expectedStateVersion + 1); + writeContext.Entry(entity).Property(candidate => candidate.StateVersion).OriginalValue = + expectedStateVersion; + + await writeContext.SaveChangesAsync(cancellationToken); + await PromotePreviousEpisodeSuccessorAsync( + writeContext, + entity.Id, + wasActiveRelease, + previousEpisodeIdentity, + currentEpisodeIdentity, + cancellationToken); + await transaction.CommitAsync(cancellationToken); return true; - } - catch (DbUpdateConcurrencyException) + }); + } + + internal static async Task SetEpisodeReleaseActivityAsync( + Models.ApplicationContext writeContext, + Models.AnimationInfo entity, + CancellationToken cancellationToken) + { + var identity = GetEpisodeIdentity(writeContext, entity); + if (identity is null) { - context.Entry(entity).State = EntityState.Detached; - return false; + entity.IsActiveRelease = false; + return; } + + var value = identity.Value; + entity.IsActiveRelease = !await writeContext.AnimationInfo + .AsNoTracking() + .AnyAsync(other => other.Id != entity.Id && + other.IsActiveRelease && + EF.Property(other, "AnimationId") == value.AnimationId && + other.Season == value.Season && + other.Episode == value.Episode, + cancellationToken); + } + + internal static EpisodeReleaseIdentity? GetEpisodeIdentity( + Models.ApplicationContext writeContext, + Models.AnimationInfo entity) + { + writeContext.ChangeTracker.DetectChanges(); + var animationId = entity.Animation?.Id ?? writeContext.Entry(entity) + .Property("AnimationId") + .CurrentValue; + return animationId is { } id && entity.Season is { } season && entity.Episode is { } episode + ? new EpisodeReleaseIdentity(id, season, episode) + : null; + } + + internal static async Task PromotePreviousEpisodeSuccessorAsync( + Models.ApplicationContext writeContext, + Guid changedReleaseId, + bool wasActiveRelease, + EpisodeReleaseIdentity? previousIdentity, + EpisodeReleaseIdentity? currentIdentity, + CancellationToken cancellationToken) + { + if (!wasActiveRelease || previousIdentity is not { } previous || previous == currentIdentity) + return; + if (await writeContext.AnimationInfo.AsNoTracking().AnyAsync( + info => info.Id != changedReleaseId && + info.IsActiveRelease && + EF.Property(info, "AnimationId") == previous.AnimationId && + info.Season == previous.Season && + info.Episode == previous.Episode, + cancellationToken)) + return; + + var successorId = await writeContext.AnimationInfo + .AsNoTracking() + .Where(info => info.Id != changedReleaseId && + info.MediaLibraryMissingSince == null && + EF.Property(info, "AnimationId") == previous.AnimationId && + info.Season == previous.Season && + info.Episode == previous.Episode) + .OrderByDescending(info => info.IsDownloadFinished && + writeContext.FileMappings.Any(mapping => + mapping.AnimationInfoId == info.Id)) + .ThenByDescending(info => info.ReleaseScore) + .ThenByDescending(info => info.PublishTime) + .ThenBy(info => info.Id) + .Select(info => (Guid?)info.Id) + .FirstOrDefaultAsync(cancellationToken); + if (successorId is null) return; + + await writeContext.AnimationInfo + .Where(info => info.Id == successorId.Value) + .ExecuteUpdateAsync( + setters => setters + .SetProperty(info => info.IsActiveRelease, true) + .SetProperty(info => info.StateVersion, info => info.StateVersion + 1), + cancellationToken); } + + internal readonly record struct EpisodeReleaseIdentity(Guid AnimationId, int Season, int Episode); } diff --git a/SecondDimensionWatcherReDive/Repositories/FileMappingRepository.cs b/SecondDimensionWatcherReDive/Repositories/FileMappingRepository.cs index 0404fac..c168010 100644 --- a/SecondDimensionWatcherReDive/Repositories/FileMappingRepository.cs +++ b/SecondDimensionWatcherReDive/Repositories/FileMappingRepository.cs @@ -208,6 +208,7 @@ public async Task TryFinalizeDownloadCancellationAsync( Guid animationInfoId, Guid? downloadAttemptId, Guid cancellationAttemptId, + SubscriptionAutomationDisposition? terminalDisposition, CancellationToken cancellationToken) { var strategy = context.Database.CreateExecutionStrategy(); @@ -249,13 +250,27 @@ await finalizeContext.FileMappings // Retain the completed cancellation id until the next Start so a // lost commit acknowledgement can be retried idempotently. animationInfo.DownloadCancellationId = cancellationAttemptId; - animationInfo.AutomationDisposition = animationInfo.AutomationDisposition is - SubscriptionAutomationDisposition.AutoDownloadQueued or - SubscriptionAutomationDisposition.ManualDownloadQueued or - SubscriptionAutomationDisposition.DownloadCompleted - ? SubscriptionAutomationDisposition.DownloadCancelled - : animationInfo.AutomationDisposition; + animationInfo.AutomationDisposition = terminalDisposition + ?? (animationInfo.AutomationDisposition is + SubscriptionAutomationDisposition.AutoDownloadQueued or + SubscriptionAutomationDisposition.ManualDownloadQueued or + SubscriptionAutomationDisposition.DownloadCompleted + ? SubscriptionAutomationDisposition.DownloadCancelled + : animationInfo.AutomationDisposition); animationInfo.StateVersion = checked(animationInfo.StateVersion + 1); + var cancelledAt = DateTimeOffset.UtcNow; + await finalizeContext.ReleaseUpgradeOperations + .Where(operation => operation.CandidateReleaseId == animationInfoId && + (operation.Status == ReleaseUpgradeStatus.Downloading || + operation.Status == ReleaseUpgradeStatus.Verifying)) + .ExecuteUpdateAsync( + setters => setters + .SetProperty(operation => operation.Status, ReleaseUpgradeStatus.Failed) + .SetProperty( + operation => operation.FailureSummary, + "Candidate download was cancelled before upgrade activation.") + .SetProperty(operation => operation.CompletedAt, cancelledAt), + cancellationToken); await finalizeContext.SaveChangesAsync(cancellationToken); await transaction.CommitAsync(cancellationToken); return true; diff --git a/SecondDimensionWatcherReDive/Repositories/FileMappingRepositoryPostgreSqlTestFixture.cs b/SecondDimensionWatcherReDive/Repositories/FileMappingRepositoryPostgreSqlTestFixture.cs index dd21bf3..b1581e5 100644 --- a/SecondDimensionWatcherReDive/Repositories/FileMappingRepositoryPostgreSqlTestFixture.cs +++ b/SecondDimensionWatcherReDive/Repositories/FileMappingRepositoryPostgreSqlTestFixture.cs @@ -1,4 +1,6 @@ using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; using SecondDimensionWatcherReDive.Framework.DataRepository; using SecondDimensionWatcherReDive.Framework.FileDownload; @@ -29,6 +31,92 @@ await context.Database.ExecuteSqlRawAsync( cancellationToken); } + public async Task<( + Guid ExpectedActiveId, + IReadOnlyList ActiveIds, + int DowngradedCandidateOperationCount)> + MigrateDuplicateActiveReleasesAsync(CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + var migrator = context.GetService(); + await migrator.MigrateAsync( + "20260829151303_AddLibrarySearchAndReleaseUpgrades", + cancellationToken); + var animation = new Models.Animation + { + Id = Guid.NewGuid(), + TmdbId = "migration-active-show", + Name = "Migration Active Show", + OriginalName = "Migration Active Show" + }; + var earlier = Release( + animation, + null, + 1, + 1, + 100, + DateTimeOffset.UtcNow.AddMinutes(-2), + FileDownloadTypes.TorrentDownload, + false, + "migration:earlier-" + Guid.NewGuid().ToString("N"), + 1); + var later = Release( + animation, + null, + 1, + 1, + 200, + DateTimeOffset.UtcNow.AddMinutes(-1), + FileDownloadTypes.TorrentDownload, + false, + "migration:later-" + Guid.NewGuid().ToString("N"), + 1); + context.AnimationInfo.AddRange(earlier, later); + await context.SaveChangesAsync(cancellationToken); + context.ChangeTracker.Clear(); + + await migrator.MigrateAsync(null, cancellationToken); + var activeIds = await context.AnimationInfo + .AsNoTracking() + .Where(info => info.IsActiveRelease) + .Select(info => info.Id) + .ToListAsync(cancellationToken); + context.ReleaseUpgradeOperations.AddRange( + new Models.ReleaseUpgradeOperation + { + Id = Guid.NewGuid(), + CurrentReleaseId = earlier.Id, + CandidateReleaseId = later.Id, + Status = ReleaseUpgradeStatus.Failed, + CurrentScore = earlier.ReleaseScore, + CandidateScore = later.ReleaseScore, + CreatedAt = DateTimeOffset.UtcNow.AddMinutes(-1), + CompletedAt = DateTimeOffset.UtcNow.AddMinutes(-1) + }, + new Models.ReleaseUpgradeOperation + { + Id = Guid.NewGuid(), + CurrentReleaseId = earlier.Id, + CandidateReleaseId = later.Id, + Status = ReleaseUpgradeStatus.Failed, + CurrentScore = earlier.ReleaseScore, + CandidateScore = later.ReleaseScore, + CreatedAt = DateTimeOffset.UtcNow, + CompletedAt = DateTimeOffset.UtcNow + }); + await context.SaveChangesAsync(cancellationToken); + context.ChangeTracker.Clear(); + await migrator.MigrateAsync( + "20260829151303_AddLibrarySearchAndReleaseUpgrades", + cancellationToken); + var downgradedCandidateOperationCount = await context.ReleaseUpgradeOperations + .CountAsync( + operation => operation.CandidateReleaseId == later.Id, + cancellationToken); + await migrator.MigrateAsync(null, cancellationToken); + return (earlier.Id, activeIds, downgradedCandidateOperationCount); + } + public async Task SeedDownloadedAnimationAsync(CancellationToken cancellationToken) { await using var context = new Models.ApplicationContext(_contextOptions); @@ -226,43 +314,113 @@ FROM pg_indexes WHERE schemaname = 'public' AND (indexname LIKE 'IX_%_Trgm' OR indexname = 'IX_AnimationInfo_ReleaseLanguages_Gin' - OR indexname = 'UX_AnimationInfo_ReleaseIdentity') + OR indexname = 'IX_AnimationInfo_AnimationId' + OR indexname IN ('UX_AnimationInfo_ReleaseIdentity', + 'UX_AnimationInfo_ActiveEpisodeRelease')) ORDER BY indexname """) .ToListAsync(cancellationToken); } - public async Task SeedUpgradeScenarioAsync(CancellationToken cancellationToken) + public async Task SeedUpgradeScenarioAsync( + CancellationToken cancellationToken, + bool includeDuplicateVideoRole = false, + bool includeCandidateProgress = false) { await using var context = new Models.ApplicationContext(_contextOptions); var animation = new Models.Animation { - Id = Guid.NewGuid(), TmdbId = "upgrade-show", Name = "Upgrade Show", OriginalName = "Upgrade Show" + Id = Guid.NewGuid(), + TmdbId = "upgrade-show", + Name = "Upgrade Show", + OriginalName = "Upgrade Show" }; var current = Release(animation, null, 1, 1, 200, DateTimeOffset.UtcNow.AddMinutes(-2), FileDownloadTypes.TorrentDownload, true, "torrent:old-" + Guid.NewGuid().ToString("N"), 1); var candidate = Release(animation, null, 1, 1, 500, DateTimeOffset.UtcNow.AddMinutes(-1), FileDownloadTypes.TorrentDownload, true, "torrent:new-" + Guid.NewGuid().ToString("N"), 1); candidate.IsActiveRelease = false; + var canonicalPath = "/Upgrade Show/Old/Upgrade Show S01E01.MKV"; + var canonicalSubtitlePath = "/Upgrade Show/Old/Upgrade Show S01E01.EN.srt"; context.AnimationInfo.AddRange(current, candidate); context.FileMappings.AddRange( new Models.FileMapping { - Id = Guid.NewGuid(), AnimationInfoId = current.Id, - VirtualPath = "/Upgrade Show/Old/Upgrade Show S01E01.mkv", - PhysicalPath = "/store/old.mkv", FileStore = "local" + Id = Guid.NewGuid(), + AnimationInfoId = current.Id, + VirtualPath = canonicalPath, + PhysicalPath = "/store/old.mkv", + FileStore = "local" }, new Models.FileMapping { - Id = Guid.NewGuid(), AnimationInfoId = candidate.Id, + Id = Guid.NewGuid(), + AnimationInfoId = current.Id, + VirtualPath = canonicalSubtitlePath, + PhysicalPath = "/store/old.en.srt", + FileStore = "local" + }, + new Models.FileMapping + { + Id = Guid.NewGuid(), + AnimationInfoId = candidate.Id, + VirtualPath = "/Upgrade Show/New/Upgrade Show S01E01 (2).mkv", + PhysicalPath = "/store/new.mkv", + FileStore = "local" + }, + new Models.FileMapping + { + Id = Guid.NewGuid(), + AnimationInfoId = candidate.Id, + VirtualPath = "/Upgrade Show/New/Upgrade Show S01E01.en (2).srt", + PhysicalPath = "/store/new.en.srt", + FileStore = "local" + }); + if (includeDuplicateVideoRole) + { + context.FileMappings.Add(new Models.FileMapping + { + Id = Guid.NewGuid(), + AnimationInfoId = candidate.Id, + VirtualPath = "/Upgrade Show/New/Upgrade Show S01E01 (3).mkv", + PhysicalPath = "/store/new-alternate.mkv", + FileStore = "local" + }); + } + var userId = Guid.NewGuid(); + context.PlaybackProgresses.Add(new Models.PlaybackProgress + { + Id = Guid.NewGuid(), + UserId = userId, + AnimationInfoId = current.Id, + VirtualPath = canonicalPath, + PositionSeconds = 321, + DurationSeconds = 1200, + IsWatched = false, + UpdatedAt = DateTimeOffset.UtcNow + }); + if (includeCandidateProgress) + { + context.PlaybackProgresses.Add(new Models.PlaybackProgress + { + Id = Guid.NewGuid(), + UserId = userId, + AnimationInfoId = candidate.Id, VirtualPath = "/Upgrade Show/New/Upgrade Show S01E01 (2).mkv", - PhysicalPath = "/store/new.mkv", FileStore = "local" + PositionSeconds = 1200, + DurationSeconds = 1200, + IsWatched = true, + UpdatedAt = DateTimeOffset.UtcNow.AddMinutes(1), + WatchedAt = DateTimeOffset.UtcNow.AddMinutes(1) }); + } await context.SaveChangesAsync(cancellationToken); return new UpgradeScenario(new ReleaseUpgradeCandidate( current.Id, candidate.Id, animation.Name, 1, 1, 200, 500, ["resolution:2160p:+400"], false), - "/Upgrade Show/Old/Upgrade Show S01E01.mkv"); + canonicalPath, + canonicalSubtitlePath, + userId); } public async Task BeginUpgradeAsync( @@ -277,10 +435,42 @@ public async Task SeedUpgradeScenarioAsync(CancellationToken ca public async Task ActivateUpgradeAsync( Guid operationId, CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + var operation = await context.ReleaseUpgradeOperations + .AsNoTracking() + .SingleAsync(item => item.Id == operationId, cancellationToken); + var repository = new ReleaseUpgradeRepository(context, _contextOptions); + var activation = await repository.GetActivationAsync( + operation.CandidateReleaseId, + cancellationToken) + ?? throw new InvalidOperationException("Upgrade activation was not found."); + return await ActivateUpgradeAsync(operationId, activation, cancellationToken); + } + + public async Task GetUpgradeActivationAsync( + Guid candidateReleaseId, + CancellationToken cancellationToken) { await using var context = new Models.ApplicationContext(_contextOptions); return await new ReleaseUpgradeRepository(context, _contextOptions) - .ActivateAsync(operationId, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow.AddHours(24), cancellationToken); + .GetActivationAsync(candidateReleaseId, cancellationToken); + } + + public async Task ActivateUpgradeAsync( + Guid operationId, + ReleaseUpgradeActivation expected, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + return await new ReleaseUpgradeRepository(context, _contextOptions) + .ActivateAsync( + operationId, + expected.PreviousMappings, + expected.CandidateMappings, + DateTimeOffset.UtcNow, + DateTimeOffset.UtcNow.AddHours(24), + cancellationToken); } public async Task> GetReadyUpgradeCandidateIdsAsync( @@ -300,6 +490,57 @@ public async Task RollbackUpgradeAsync( .RollbackAsync(operationId, DateTimeOffset.UtcNow, cancellationToken); } + public async Task MarkUpgradeFailedAsync( + Guid operationId, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + return await new ReleaseUpgradeRepository(context, _contextOptions) + .MarkFailedAsync(operationId, "late failure", cancellationToken); + } + + public async Task BeginUpgradeCandidateCancellationAsync( + Guid animationInfoId, + Guid? downloadAttemptId, + Guid cancellationAttemptId, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + return await new AnimationInfoRepository(context, _contextOptions) + .TryBeginCancelDownloadAsync( + animationInfoId, + downloadAttemptId, + cancellationAttemptId, + cancellationToken); + } + + public async Task FinalizeUpgradeCandidateCancellationAsync( + Guid animationInfoId, + Guid? downloadAttemptId, + Guid cancellationAttemptId, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + return await new FileMappingRepository(context, _contextOptions) + .TryFinalizeDownloadCancellationAsync( + animationInfoId, + downloadAttemptId, + cancellationAttemptId, + terminalDisposition: null, + cancellationToken); + } + + public async Task GetUpgradeOperationAsync( + Guid operationId, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + return (await context.ReleaseUpgradeOperations + .AsNoTracking() + .SingleAsync(operation => operation.Id == operationId, cancellationToken)) + .ToRecord(); + } + public async Task> GetMappingsAsync( Guid animationInfoId, CancellationToken cancellationToken) @@ -309,6 +550,238 @@ public async Task> GetMappingsAsync( .GetForAnimationInfoAsync(animationInfoId, cancellationToken); } + public async Task> GetPlaybackProgressesAsync( + Guid userId, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + return (await context.PlaybackProgresses + .AsNoTracking() + .Where(progress => progress.UserId == userId) + .OrderBy(progress => progress.VirtualPath) + .ToListAsync(cancellationToken)) + .Select(progress => progress.ToRecord()) + .ToList(); + } + + public async Task ChangeReleaseEpisodeAsync( + Guid animationInfoId, + int episode, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + await context.AnimationInfo + .Where(info => info.Id == animationInfoId) + .ExecuteUpdateAsync( + setters => setters.SetProperty(info => info.Episode, episode), + cancellationToken); + } + + public async Task SetCandidateDownloadInProgressAsync( + Guid animationInfoId, + CancellationToken cancellationToken) + { + var downloadAttemptId = Guid.NewGuid(); + await using var context = new Models.ApplicationContext(_contextOptions); + await context.AnimationInfo + .Where(info => info.Id == animationInfoId) + .ExecuteUpdateAsync( + setters => setters + .SetProperty(info => info.IsDownloadTracked, true) + .SetProperty(info => info.IsDownloadFinished, false) + .SetProperty(info => info.DownloadAttemptId, downloadAttemptId) + .SetProperty(info => info.FileStore, (string?)null) + .SetProperty(info => info.StorePath, (string?)null), + cancellationToken); + return downloadAttemptId; + } + + public async Task CancelUpgradeCandidateAsync( + Guid animationInfoId, + Guid downloadAttemptId, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + var animationRepository = new AnimationInfoRepository(context, _contextOptions); + var cancellationAttemptId = Guid.NewGuid(); + if (!await animationRepository.TryBeginCancelDownloadAsync( + animationInfoId, + downloadAttemptId, + cancellationAttemptId, + cancellationToken)) + return null; + var mappingRepository = new FileMappingRepository(context, _contextOptions); + if (!await mappingRepository.TryFinalizeDownloadCancellationAsync( + animationInfoId, + downloadAttemptId, + cancellationAttemptId, + terminalDisposition: null, + cancellationToken)) + return null; + return await animationRepository.FindByIdAsync(animationInfoId, cancellationToken); + } + + public async Task ChangeMappingPhysicalPathAsync( + Guid animationInfoId, + string virtualPath, + string physicalPath, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + await context.FileMappings + .Where(mapping => mapping.AnimationInfoId == animationInfoId && + mapping.VirtualPath == virtualPath) + .ExecuteUpdateAsync( + setters => setters.SetProperty(mapping => mapping.PhysicalPath, physicalPath), + cancellationToken); + } + + public async Task RemapCandidatePlaybackAsync( + Guid animationInfoId, + Guid userId, + string currentPath, + string replacementPath, + CancellationToken cancellationToken) + { + await using var context = new Models.ApplicationContext(_contextOptions); + await using var transaction = await context.Database.BeginTransactionAsync(cancellationToken); + await MappingTransactionLock.AcquireAsync(context, cancellationToken); + await context.FileMappings + .Where(mapping => mapping.AnimationInfoId == animationInfoId && + mapping.VirtualPath == currentPath) + .ExecuteUpdateAsync( + setters => setters.SetProperty(mapping => mapping.VirtualPath, replacementPath), + cancellationToken); + await context.PlaybackProgresses + .Where(progress => progress.AnimationInfoId == animationInfoId && + progress.UserId == userId && + progress.VirtualPath == currentPath) + .ExecuteUpdateAsync( + setters => setters.SetProperty(progress => progress.VirtualPath, replacementPath), + cancellationToken); + await transaction.CommitAsync(cancellationToken); + } + + public async Task<(bool FirstActive, bool SecondActive, int FirstEpisode)> + IdentifyCompetingReleasesAsync( + CancellationToken cancellationToken, + bool moveFirst = false, + bool deidentifyFirst = false, + bool concurrent = false) + { + var animation = new Models.Animation + { + Id = Guid.NewGuid(), + TmdbId = "single-active-show", + Name = "Single Active Show", + OriginalName = "Single Active Show" + }; + var first = Release( + animation, + null, + 1, + null, + 100, + DateTimeOffset.UtcNow.AddMinutes(-2), + FileDownloadTypes.TorrentDownload, + false, + "torrent:first-" + Guid.NewGuid().ToString("N"), + 12); + var second = Release( + animation, + null, + 1, + null, + 200, + DateTimeOffset.UtcNow.AddMinutes(-1), + FileDownloadTypes.TorrentDownload, + false, + "torrent:second-" + Guid.NewGuid().ToString("N"), + 12); + first.IsActiveRelease = false; + second.IsActiveRelease = false; + await using (var seedContext = new Models.ApplicationContext(_contextOptions)) + { + seedContext.AnimationInfo.AddRange(first, second); + await seedContext.SaveChangesAsync(cancellationToken); + } + + if (concurrent) + { + async Task IdentifyAsync(Guid releaseId) + { + await using var updateContext = new Models.ApplicationContext(_contextOptions); + var updateRepository = new AnimationInfoRepository(updateContext, _contextOptions); + var record = await updateRepository.FindByIdAsync(releaseId, cancellationToken) + ?? throw new InvalidOperationException("Seeded release could not be reloaded."); + if (!await updateRepository.TryUpdateAsync( + record with + { + Animation = animation.ToRecord(), + Season = 1, + Episode = 1 + }, + record.StateVersion, + cancellationToken)) + throw new InvalidOperationException("Seeded release could not be identified."); + } + + await Task.WhenAll(IdentifyAsync(first.Id), IdentifyAsync(second.Id)); + } + else + { + await using var repositoryContext = new Models.ApplicationContext(_contextOptions); + var repository = new AnimationInfoRepository(repositoryContext, _contextOptions); + var firstRecord = await repository.FindByIdAsync(first.Id, cancellationToken); + var secondRecord = await repository.FindByIdAsync(second.Id, cancellationToken); + if (firstRecord is null || secondRecord is null) + throw new InvalidOperationException("Seeded releases could not be reloaded."); + + var animationRecord = animation.ToRecord(); + if (!await repository.TryUpdateAsync( + firstRecord with { Animation = animationRecord, Season = 1, Episode = 1 }, + firstRecord.StateVersion, + cancellationToken) || + !await repository.TryUpdateAsync( + secondRecord with { Animation = animationRecord, Season = 1, Episode = 1 }, + secondRecord.StateVersion, + cancellationToken)) + throw new InvalidOperationException("Seeded releases could not be identified."); + + if (moveFirst) + { + var identifiedFirst = await repository.FindByIdAsync(first.Id, cancellationToken) + ?? throw new InvalidOperationException( + "Active release could not be reloaded."); + if (!await repository.TryUpdateAsync( + identifiedFirst with { Episode = 2 }, + identifiedFirst.StateVersion, + cancellationToken)) + throw new InvalidOperationException("Active release could not be moved."); + } + else if (deidentifyFirst) + { + var identifiedFirst = await repository.FindByIdAsync(first.Id, cancellationToken) + ?? throw new InvalidOperationException( + "Active release could not be reloaded."); + if (!await repository.TryUpdateAsync( + identifiedFirst with { Animation = null }, + identifiedFirst.StateVersion, + cancellationToken)) + throw new InvalidOperationException("Active release could not be de-identified."); + } + } + + await using var readContext = new Models.ApplicationContext(_contextOptions); + var releases = await readContext.AnimationInfo + .Where(info => info.Id == first.Id || info.Id == second.Id) + .ToDictionaryAsync(info => info.Id, cancellationToken); + return ( + releases[first.Id].IsActiveRelease, + releases[second.Id].IsActiveRelease, + releases[first.Id].Episode!.Value); + } + private static Models.AnimationInfo Release( Models.Animation animation, Models.AnimationGroup? group, @@ -320,28 +793,29 @@ private static Models.AnimationInfo Release( bool downloaded, string identity, int expected) => new() - { - Id = Guid.NewGuid(), - Animation = animation, - Group = group, - Title = $"{animation.Name} S{season:D2}E{episode:D2}", - Description = animation.OriginalName, - PublishTime = ingestedAt, - IngestedAt = ingestedAt, - DownloadUrl = "https://example.test/" + Guid.NewGuid().ToString("N"), - DownloadType = downloadType, - IsDownloadTracked = downloaded, - IsDownloadFinished = downloaded, - FileStore = downloaded ? "local" : null, - StorePath = downloaded ? "/store/" + Guid.NewGuid().ToString("N") : null, - Season = season, - Episode = episode, - ReleaseIdentity = identity, - ReleaseSubtitleGroup = group?.Name, - ReleaseScore = score, - ExpectedEpisodeCount = expected, - IsAiProcessed = true - }; + { + Id = Guid.NewGuid(), + Animation = animation, + Group = group, + Title = $"{animation.Name} S{season:D2}E{episode:D2}", + Description = animation.OriginalName, + PublishTime = ingestedAt, + IngestedAt = ingestedAt, + DownloadUrl = "https://example.test/" + Guid.NewGuid().ToString("N"), + DownloadType = downloadType, + IsDownloadTracked = downloaded, + IsDownloadFinished = downloaded, + FileStore = downloaded ? "local" : null, + StorePath = downloaded ? "/store/" + Guid.NewGuid().ToString("N") : null, + Season = season, + Episode = episode, + ReleaseIdentity = identity, + ReleaseSubtitleGroup = group?.Name, + ReleaseScore = score, + ExpectedEpisodeCount = expected, + IsAiProcessed = true, + IsActiveRelease = true + }; private static Models.FileMapping MappingEntity(Guid animationInfoId, string path) => new() { @@ -361,4 +835,6 @@ internal sealed record LibraryScenario( internal sealed record UpgradeScenario( ReleaseUpgradeCandidate Candidate, - string CanonicalPath); + string CanonicalPath, + string CanonicalSubtitlePath, + Guid UserId); diff --git a/SecondDimensionWatcherReDive/Repositories/MetadataReviewRepository.cs b/SecondDimensionWatcherReDive/Repositories/MetadataReviewRepository.cs index 233c45d..decee7c 100644 --- a/SecondDimensionWatcherReDive/Repositories/MetadataReviewRepository.cs +++ b/SecondDimensionWatcherReDive/Repositories/MetadataReviewRepository.cs @@ -290,6 +290,10 @@ public async Task ApplyPreviewAsync( var mappingsBefore = existingMappings.Select(mapping => mapping.ToRecord()).ToList(); var animationInfoEntry = applyContext.Entry(animationInfo); + var previousEpisodeIdentity = AnimationInfoRepository.GetEpisodeIdentity( + applyContext, + animationInfo); + var wasActiveRelease = animationInfo.IsActiveRelease; operation.PreviousDescription = animationInfo.Description; operation.PreviousAnimationId = animationInfoEntry .Property("AnimationId") @@ -335,6 +339,8 @@ await applyContext.MetadataReviewMappingSnapshots.AddRangeAsync( animationInfo.Description = operation.ProposedDescription; animationInfo.Animation = animation; animationInfo.Group = group; + animationInfoEntry.Property("AnimationId").CurrentValue = animation.Id; + animationInfoEntry.Property("GroupId").CurrentValue = group?.Id; animationInfo.Season = operation.ProposedSeason; animationInfo.Episode = operation.ProposedEpisode; animationInfo.MetadataStatus = MetadataReviewStatus.Reviewed; @@ -344,6 +350,13 @@ await applyContext.MetadataReviewMappingSnapshots.AddRangeAsync( animationInfo.AiRetryCount = 0; animationInfo.MetadataReviewedAt = appliedAt; animationInfo.CurrentMetadataReviewOperationId = operation.Id; + await AnimationInfoRepository.SetEpisodeReleaseActivityAsync( + applyContext, + animationInfo, + cancellationToken); + var currentEpisodeIdentity = AnimationInfoRepository.GetEpisodeIdentity( + applyContext, + animationInfo); animationInfo.StateVersion = checked(animationInfo.StateVersion + 1); var replacementMappings = proposedSnapshots @@ -374,6 +387,13 @@ await applyContext.FileMappings operation.AppliedVersion = animationInfo.StateVersion; await applyContext.SaveChangesAsync(cancellationToken); + await AnimationInfoRepository.PromotePreviousEpisodeSuccessorAsync( + applyContext, + animationInfo.Id, + wasActiveRelease, + previousEpisodeIdentity, + currentEpisodeIdentity, + cancellationToken); await transaction.CommitAsync(cancellationToken); return new MetadataReviewMutationResult( MetadataReviewMutationOutcome.Success, @@ -547,9 +567,17 @@ public async Task UndoAsync( animationInfo.Id); } + var previousEpisodeIdentity = AnimationInfoRepository.GetEpisodeIdentity( + undoContext, + animationInfo); + var wasActiveRelease = animationInfo.IsActiveRelease; animationInfo.Description = operation.PreviousDescription; animationInfo.Animation = previousAnimation; animationInfo.Group = previousGroup; + undoContext.Entry(animationInfo).Property("AnimationId").CurrentValue = + previousAnimation?.Id; + undoContext.Entry(animationInfo).Property("GroupId").CurrentValue = + previousGroup?.Id; animationInfo.Season = operation.PreviousSeason; animationInfo.Episode = operation.PreviousEpisode; animationInfo.MetadataStatus = operation.PreviousMetadataStatus.Value; @@ -559,6 +587,13 @@ public async Task UndoAsync( animationInfo.AiRetryCount = operation.PreviousAiRetryCount.Value; animationInfo.MetadataReviewedAt = operation.PreviousReviewedAt; animationInfo.CurrentMetadataReviewOperationId = operation.PreviousCurrentOperationId; + await AnimationInfoRepository.SetEpisodeReleaseActivityAsync( + undoContext, + animationInfo, + cancellationToken); + var currentEpisodeIdentity = AnimationInfoRepository.GetEpisodeIdentity( + undoContext, + animationInfo); animationInfo.StateVersion = checked(animationInfo.StateVersion + 1); var restoredMappings = previousSnapshots @@ -594,6 +629,13 @@ await undoContext.FileMappings previousOperation.AppliedVersion = animationInfo.StateVersion; await undoContext.SaveChangesAsync(cancellationToken); + await AnimationInfoRepository.PromotePreviousEpisodeSuccessorAsync( + undoContext, + animationInfo.Id, + wasActiveRelease, + previousEpisodeIdentity, + currentEpisodeIdentity, + cancellationToken); await transaction.CommitAsync(cancellationToken); return new MetadataReviewMutationResult( MetadataReviewMutationOutcome.Success, diff --git a/SecondDimensionWatcherReDive/Repositories/ReleaseUpgradeRepository.cs b/SecondDimensionWatcherReDive/Repositories/ReleaseUpgradeRepository.cs index cff38da..e69543c 100644 --- a/SecondDimensionWatcherReDive/Repositories/ReleaseUpgradeRepository.cs +++ b/SecondDimensionWatcherReDive/Repositories/ReleaseUpgradeRepository.cs @@ -1,10 +1,11 @@ +using System.Text.RegularExpressions; using Microsoft.EntityFrameworkCore; using Npgsql; using SecondDimensionWatcherReDive.Framework.DataRepository; namespace SecondDimensionWatcherReDive.Repositories; -public sealed class ReleaseUpgradeRepository( +public sealed partial class ReleaseUpgradeRepository( Models.ApplicationContext context, DbContextOptions contextOptions) : IReleaseUpgradeRepository { @@ -30,16 +31,17 @@ public async Task> GetCandidatesAsync( var policies = await context.SubscriptionAutomationPolicies.AsNoTracking() .ToDictionaryAsync(policy => policy.FeedId, cancellationToken); var attempted = await context.ReleaseUpgradeOperations.AsNoTracking() + .Where(operation => automaticOnly || operation.Status != ReleaseUpgradeStatus.Failed) .Select(operation => operation.CandidateReleaseId) .ToHashSetAsync(cancellationToken); var candidates = new List(); foreach (var episode in releases.GroupBy(info => new - { - AnimationId = info.Animation!.Id, - info.Season, - info.Episode - })) + { + AnimationId = info.Animation!.Id, + info.Season, + info.Episode + })) { var current = episode .Where(info => info.IsActiveRelease && info.IsDownloadFinished && mapped.Contains(info.Id)) @@ -109,6 +111,9 @@ public async Task> GetCandidatesAsync( current.Season != next.Season || current.Episode != next.Episode || next.ReleaseScore <= current.ReleaseScore || + !current.IsActiveRelease || + next.IsActiveRelease || + (next.IsDownloadTracked && next.DownloadCancellationId is not null) || !current.IsDownloadFinished || !await writeContext.FileMappings.AnyAsync( mapping => mapping.AnimationInfoId == current.Id, @@ -116,7 +121,8 @@ public async Task> GetCandidatesAsync( return null; if (await writeContext.ReleaseUpgradeOperations.AnyAsync( - operation => operation.CandidateReleaseId == next.Id || + operation => (operation.CandidateReleaseId == next.Id && + operation.Status != ReleaseUpgradeStatus.Failed) || (operation.CurrentReleaseId == current.Id && (operation.Status == ReleaseUpgradeStatus.Downloading || operation.Status == ReleaseUpgradeStatus.Verifying || @@ -144,7 +150,7 @@ public async Task> GetCandidatesAsync( return entity.ToRecord(); } catch (DbUpdateException exception) when (exception.InnerException is PostgresException - { SqlState: PostgresErrorCodes.UniqueViolation }) + { SqlState: PostgresErrorCodes.UniqueViolation }) { return null; } @@ -207,6 +213,8 @@ public async Task> GetReadyCandidateIdsAsync( public async Task ActivateAsync( Guid operationId, + IReadOnlyList expectedPreviousMappings, + IReadOnlyList expectedCandidateMappings, DateTimeOffset verifiedAt, DateTimeOffset rollbackUntil, CancellationToken cancellationToken) @@ -235,6 +243,14 @@ public async Task ActivateAsync( !infos.TryGetValue(operation.CandidateReleaseId, out var candidate) || !candidate.IsDownloadFinished) return new ReleaseUpgradeMutationResult(false, "candidate_not_ready", operation.ToRecord()); + if (current.DownloadCancellationId is not null || + candidate.DownloadCancellationId is not null) + return new ReleaseUpgradeMutationResult(false, "download_cancelling", operation.ToRecord()); + if (!AreSameEpisode(writeContext, current, candidate) || + !current.IsActiveRelease || + candidate.IsActiveRelease || + candidate.ReleaseScore <= current.ReleaseScore) + return new ReleaseUpgradeMutationResult(false, "release_changed", operation.ToRecord()); var mappings = await writeContext.FileMappings .Where(mapping => mapping.AnimationInfoId == current.Id || @@ -245,6 +261,9 @@ public async Task ActivateAsync( var next = mappings.Where(mapping => mapping.AnimationInfoId == candidate.Id).ToList(); if (previous.Count == 0 || next.Count == 0) return new ReleaseUpgradeMutationResult(false, "mapping_missing", operation.ToRecord()); + if (!MappingSetsMatch(previous, expectedPreviousMappings) || + !MappingSetsMatch(next, expectedCandidateMappings)) + return new ReleaseUpgradeMutationResult(false, "mapping_changed", operation.ToRecord()); var snapshots = previous .Select(mapping => ToSnapshot( @@ -262,7 +281,11 @@ await writeContext.ReleaseUpgradeMappingSnapshots.AddRangeAsync( writeContext.FileMappings.RemoveRange(mappings); var replacement = BuildCandidateReplacement(previous, next, candidate.Id); - await writeContext.FileMappings.AddRangeAsync(replacement, cancellationToken); + await writeContext.FileMappings.AddRangeAsync(replacement.Mappings, cancellationToken); + await TransferPlaybackProgressAsync( + writeContext, + BuildActivationPlaybackTransfers(current.Id, candidate.Id, replacement), + cancellationToken); await writeContext.AnimationInfo .Where(info => info.Id == current.Id) .ExecuteUpdateAsync(setters => setters @@ -290,19 +313,33 @@ public async Task MarkFailedAsync( string failureSummary, CancellationToken cancellationToken) { - var operation = await context.ReleaseUpgradeOperations - .SingleOrDefaultAsync(item => item.Id == operationId, cancellationToken); - if (operation is null) - return new ReleaseUpgradeMutationResult(false, "not_found", null); - if (operation.Status is ReleaseUpgradeStatus.Completed or ReleaseUpgradeStatus.RolledBack) - return new ReleaseUpgradeMutationResult(false, "invalid_state", operation.ToRecord()); - operation.Status = ReleaseUpgradeStatus.Failed; - operation.FailureSummary = failureSummary.Length <= 2048 - ? failureSummary - : failureSummary[..2048]; - operation.CompletedAt = DateTimeOffset.UtcNow; - await context.SaveChangesAsync(cancellationToken); - return new ReleaseUpgradeMutationResult(true, "failed", operation.ToRecord()); + var strategy = context.Database.CreateExecutionStrategy(); + return await strategy.ExecuteAsync(async () => + { + await using var writeContext = new Models.ApplicationContext(contextOptions); + await using var transaction = await writeContext.Database.BeginTransactionAsync(cancellationToken); + await MappingTransactionLock.AcquireAsync(writeContext, cancellationToken); + await writeContext.Database.ExecuteSqlInterpolatedAsync( + $"SELECT 1 FROM \"ReleaseUpgradeOperations\" WHERE \"Id\" = {operationId} FOR UPDATE", + cancellationToken); + var operation = await writeContext.ReleaseUpgradeOperations + .SingleOrDefaultAsync(item => item.Id == operationId, cancellationToken); + if (operation is null) + return new ReleaseUpgradeMutationResult(false, "not_found", null); + if (operation.Status == ReleaseUpgradeStatus.Failed) + return new ReleaseUpgradeMutationResult(true, "already_failed", operation.ToRecord()); + if (operation.Status is not (ReleaseUpgradeStatus.Downloading or ReleaseUpgradeStatus.Verifying)) + return new ReleaseUpgradeMutationResult(false, "invalid_state", operation.ToRecord()); + + operation.Status = ReleaseUpgradeStatus.Failed; + operation.FailureSummary = failureSummary.Length <= 2048 + ? failureSummary + : failureSummary[..2048]; + operation.CompletedAt = DateTimeOffset.UtcNow; + await writeContext.SaveChangesAsync(cancellationToken); + await transaction.CommitAsync(cancellationToken); + return new ReleaseUpgradeMutationResult(true, "failed", operation.ToRecord()); + }); } public async Task RollbackAsync( @@ -330,12 +367,39 @@ public async Task RollbackAsync( writeContext, [operation.CurrentReleaseId, operation.CandidateReleaseId], cancellationToken); + if (!infos.TryGetValue(operation.CurrentReleaseId, out var current) || + !infos.TryGetValue(operation.CandidateReleaseId, out var activeCandidate) || + !AreSameEpisode(writeContext, current, activeCandidate) || + current.IsActiveRelease || + !activeCandidate.IsActiveRelease) + return new ReleaseUpgradeMutationResult(false, "release_changed", operation.ToRecord()); var previous = operation.MappingSnapshots .Where(snapshot => snapshot.Kind == ReleaseUpgradeMappingKind.Previous) .ToList(); - if (previous.Count == 0) + var candidate = operation.MappingSnapshots + .Where(snapshot => snapshot.Kind == ReleaseUpgradeMappingKind.Candidate) + .ToList(); + if (previous.Count == 0 || candidate.Count == 0) return new ReleaseUpgradeMutationResult(false, "snapshot_missing", operation.ToRecord()); + var replacement = BuildCandidateReplacement( + previous.Select(FromSnapshot).ToList(), + candidate.Select(FromSnapshot).ToList(), + operation.CandidateReleaseId); + var currentCandidateMappings = await writeContext.FileMappings + .AsNoTracking() + .Where(mapping => mapping.AnimationInfoId == operation.CandidateReleaseId) + .ToListAsync(cancellationToken); + if (!MappingSetsMatch(currentCandidateMappings, replacement.Mappings)) + return new ReleaseUpgradeMutationResult(false, "mapping_changed", operation.ToRecord()); + await TransferPlaybackProgressAsync( + writeContext, + BuildRollbackPlaybackTransfers( + operation.CurrentReleaseId, + operation.CandidateReleaseId, + replacement), + cancellationToken); + await writeContext.FileMappings .Where(mapping => mapping.AnimationInfoId == operation.CandidateReleaseId) .ExecuteDeleteAsync(cancellationToken); @@ -348,16 +412,16 @@ await writeContext.FileMappings FileStore = snapshot.FileStore }), cancellationToken); await writeContext.AnimationInfo - .Where(info => info.Id == operation.CurrentReleaseId) + .Where(info => info.Id == operation.CandidateReleaseId) .ExecuteUpdateAsync(setters => setters .SetProperty(info => info.StateVersion, info => info.StateVersion + 1) - .SetProperty(info => info.IsActiveRelease, true), + .SetProperty(info => info.IsActiveRelease, false), cancellationToken); await writeContext.AnimationInfo - .Where(info => info.Id == operation.CandidateReleaseId) + .Where(info => info.Id == operation.CurrentReleaseId) .ExecuteUpdateAsync(setters => setters .SetProperty(info => info.StateVersion, info => info.StateVersion + 1) - .SetProperty(info => info.IsActiveRelease, false), + .SetProperty(info => info.IsActiveRelease, true), cancellationToken); operation.Status = ReleaseUpgradeStatus.RolledBack; operation.CompletedAt = rolledBackAt; @@ -395,34 +459,50 @@ private static Models.ReleaseUpgradeMappingSnapshot ToSnapshot( Guid operationId, Models.FileMapping mapping, ReleaseUpgradeMappingKind kind) => new() - { - Id = Guid.NewGuid(), - OperationId = operationId, - Kind = kind, - OriginalMappingId = mapping.Id, - AnimationInfoId = mapping.AnimationInfoId, - VirtualPath = mapping.VirtualPath, - PhysicalPath = mapping.PhysicalPath, - FileStore = mapping.FileStore - }; + { + Id = Guid.NewGuid(), + OperationId = operationId, + Kind = kind, + OriginalMappingId = mapping.Id, + AnimationInfoId = mapping.AnimationInfoId, + VirtualPath = mapping.VirtualPath, + PhysicalPath = mapping.PhysicalPath, + FileStore = mapping.FileStore + }; - private static IReadOnlyList BuildCandidateReplacement( + private static CandidateReplacementPlan BuildCandidateReplacement( IReadOnlyList previous, IReadOnlyList candidate, Guid candidateReleaseId) { - var remaining = new HashSet(candidate.Select(item => item.VirtualPath), StringComparer.Ordinal); + var previousByRole = previous + .GroupBy(mapping => GetStableFileRole(mapping.VirtualPath), StringComparer.OrdinalIgnoreCase) + .Where(group => group.Count() == 1) + .ToDictionary(group => group.Key, group => group.Single(), StringComparer.OrdinalIgnoreCase); + var matchedPreviousIds = new HashSet(); var used = new HashSet(StringComparer.Ordinal); var result = new List(candidate.Count); - for (var index = 0; index < candidate.Count; index++) + var candidatePathReplacements = new Dictionary(StringComparer.Ordinal); + var previousPathReplacements = new Dictionary(StringComparer.Ordinal); + foreach (var mapping in candidate + .OrderBy(item => item.VirtualPath, StringComparer.Ordinal) + .ThenBy(item => item.PhysicalPath, StringComparer.Ordinal) + .ThenBy(item => item.Id)) { - var mapping = candidate[index]; - remaining.Remove(mapping.VirtualPath); - var preferred = index < previous.Count ? previous[index].VirtualPath : mapping.VirtualPath; - var virtualPath = !used.Contains(preferred) && !remaining.Contains(preferred) - ? preferred + var role = GetStableFileRole(mapping.VirtualPath); + var matchedPrevious = previousByRole.GetValueOrDefault(role); + var usesPreviousPath = matchedPrevious is not null && + matchedPreviousIds.Add(matchedPrevious.Id); + var virtualPath = usesPreviousPath + ? matchedPrevious!.VirtualPath : mapping.VirtualPath; - used.Add(virtualPath); + if (!used.Add(virtualPath)) + throw new InvalidOperationException( + $"Candidate replacement produced duplicate virtual path '{virtualPath}'."); + + candidatePathReplacements.Add(mapping.VirtualPath, virtualPath); + if (usesPreviousPath) + previousPathReplacements.TryAdd(matchedPrevious!.VirtualPath, virtualPath); result.Add(new Models.FileMapping { Id = Guid.NewGuid(), @@ -433,9 +513,190 @@ private static Models.ReleaseUpgradeMappingSnapshot ToSnapshot( }); } - return result; + return new CandidateReplacementPlan( + result, + candidatePathReplacements, + previousPathReplacements); + } + + private static string GetStableFileRole(string virtualPath) + { + var fileName = virtualPath[(virtualPath.LastIndexOf('/') + 1)..]; + var extension = Path.GetExtension(fileName); + var stem = extension.Length == 0 ? fileName : fileName[..^extension.Length]; + return CollisionSuffixRegex().Replace(stem, string.Empty) + extension; + } + + private static Models.FileMapping FromSnapshot(Models.ReleaseUpgradeMappingSnapshot snapshot) => new() + { + Id = snapshot.OriginalMappingId, + AnimationInfoId = snapshot.AnimationInfoId, + VirtualPath = snapshot.VirtualPath, + PhysicalPath = snapshot.PhysicalPath, + FileStore = snapshot.FileStore + }; + + private static bool AreSameEpisode( + Models.ApplicationContext writeContext, + Models.AnimationInfo current, + Models.AnimationInfo candidate) + { + var currentAnimationId = writeContext.Entry(current) + .Property("AnimationId") + .CurrentValue; + var candidateAnimationId = writeContext.Entry(candidate) + .Property("AnimationId") + .CurrentValue; + return currentAnimationId is not null && + currentAnimationId == candidateAnimationId && + current.Season is not null && + current.Season == candidate.Season && + current.Episode is not null && + current.Episode == candidate.Episode; + } + + private static bool MappingSetsMatch( + IReadOnlyCollection actual, + IReadOnlyCollection expected) + { + if (actual.Count != expected.Count) return false; + var expectedMappings = expected + .Select(mapping => (mapping.VirtualPath, mapping.PhysicalPath, mapping.FileStore)) + .ToHashSet(); + return actual.All(mapping => expectedMappings.Contains( + (mapping.VirtualPath, mapping.PhysicalPath, mapping.FileStore))); + } + + private static bool MappingSetsMatch( + IReadOnlyCollection actual, + IReadOnlyCollection expected) + { + if (actual.Count != expected.Count) return false; + var expectedMappings = expected + .Select(mapping => (mapping.VirtualPath, mapping.PhysicalPath, mapping.FileStore)) + .ToHashSet(); + return actual.All(mapping => expectedMappings.Contains( + (mapping.VirtualPath, mapping.PhysicalPath, mapping.FileStore))); + } + + private static IReadOnlyDictionary + BuildActivationPlaybackTransfers( + Guid currentReleaseId, + Guid candidateReleaseId, + CandidateReplacementPlan replacement) + { + var transfers = replacement.CandidatePathReplacements.ToDictionary( + pair => new PlaybackLocation(candidateReleaseId, pair.Key), + pair => new PlaybackLocation(candidateReleaseId, pair.Value)); + foreach (var pair in replacement.PreviousPathReplacements) + { + transfers[new PlaybackLocation(currentReleaseId, pair.Key)] = + new PlaybackLocation(candidateReleaseId, pair.Value); + } + + return transfers; + } + + private static IReadOnlyDictionary + BuildRollbackPlaybackTransfers( + Guid currentReleaseId, + Guid candidateReleaseId, + CandidateReplacementPlan replacement) + { + var transfers = new Dictionary(); + foreach (var pair in replacement.PreviousPathReplacements) + { + transfers[new PlaybackLocation(candidateReleaseId, pair.Value)] = + new PlaybackLocation(currentReleaseId, pair.Key); + } + + return transfers; } + private static async Task TransferPlaybackProgressAsync( + Models.ApplicationContext writeContext, + IReadOnlyDictionary transfers, + CancellationToken cancellationToken) + { + if (transfers.Count == 0) return; + + var ownerIds = transfers.Keys + .Select(location => location.AnimationInfoId) + .Concat(transfers.Values.Select(location => location.AnimationInfoId)) + .Distinct() + .ToArray(); + var rows = await writeContext.PlaybackProgresses + .AsNoTracking() + .Where(progress => ownerIds.Contains(progress.AnimationInfoId)) + .ToListAsync(cancellationToken); + var targets = transfers.Values.ToHashSet(); + var affected = rows + .Where(progress => + { + var location = new PlaybackLocation( + progress.AnimationInfoId, + progress.VirtualPath); + return transfers.ContainsKey(location) || targets.Contains(location); + }) + .ToList(); + if (affected.Count == 0) return; + + var affectedIds = affected.Select(progress => progress.Id).ToArray(); + await writeContext.PlaybackProgresses + .Where(progress => affectedIds.Contains(progress.Id)) + .ExecuteDeleteAsync(cancellationToken); + + var merged = affected + .Select(progress => + { + var source = new PlaybackLocation( + progress.AnimationInfoId, + progress.VirtualPath); + var target = transfers.GetValueOrDefault(source, source); + return (Progress: progress, Target: target); + }) + .GroupBy(item => new + { + item.Progress.UserId, + item.Target.AnimationInfoId, + item.Target.VirtualPath + }) + .Select(group => + { + // If both releases have progress for the same user/file role, + // retain the last user action rather than reviving stale state. + var winner = group + .OrderByDescending(item => item.Progress.UpdatedAt) + .ThenByDescending(item => item.Progress.Id) + .First() + .Progress; + return new Models.PlaybackProgress + { + Id = winner.Id, + UserId = group.Key.UserId, + AnimationInfoId = group.Key.AnimationInfoId, + VirtualPath = group.Key.VirtualPath, + PositionSeconds = winner.PositionSeconds, + DurationSeconds = winner.DurationSeconds, + IsWatched = winner.IsWatched, + UpdatedAt = winner.UpdatedAt, + WatchedAt = winner.WatchedAt + }; + }) + .ToList(); + await writeContext.PlaybackProgresses.AddRangeAsync(merged, cancellationToken); + } + + [GeneratedRegex(@" \(\d+\)$", RegexOptions.CultureInvariant)] + private static partial Regex CollisionSuffixRegex(); + + private readonly record struct PlaybackLocation(Guid AnimationInfoId, string VirtualPath); + + private sealed record CandidateReplacementPlan( + IReadOnlyList Mappings, + IReadOnlyDictionary CandidatePathReplacements, + IReadOnlyDictionary PreviousPathReplacements); + private static IReadOnlyList ParseReasons(string? json) { if (string.IsNullOrWhiteSpace(json)) return []; diff --git a/SecondDimensionWatcherReDive/Utils/ReleaseUpgrades/ReleaseUpgradeCoordinator.cs b/SecondDimensionWatcherReDive/Utils/ReleaseUpgrades/ReleaseUpgradeCoordinator.cs index 2424349..61fc6a0 100644 --- a/SecondDimensionWatcherReDive/Utils/ReleaseUpgrades/ReleaseUpgradeCoordinator.cs +++ b/SecondDimensionWatcherReDive/Utils/ReleaseUpgrades/ReleaseUpgradeCoordinator.cs @@ -48,21 +48,45 @@ public async Task ExecuteAsync( return Result(false, "upgrade_already_started", false, requiresDownload, null, ["Another worker already claimed this upgrade."]); - if (!requiresDownload) + if (operation.Status == ReleaseUpgradeStatus.Verifying) return await ActivateAsync(operation.CandidateReleaseId, cancellationToken); + next = await animationInfoRepository.FindByIdAsync( + candidate.CandidateReleaseId, + cancellationToken); + if (next is null) + return await FailAsync(operation, "Candidate release disappeared after claim.", cancellationToken); + if (next.IsDownloadFinished) + return await ActivateAsync(operation.CandidateReleaseId, cancellationToken); + if (next.IsDownloadTracked) + return Result(true, "download_in_progress", false, true, operation, []); + var downloadAttemptId = Guid.NewGuid(); + IFileDownloadClient? client = null; + var downloadStartAttempted = false; + var submissionAttempted = false; try { + downloadStartAttempted = true; if (!await animationInfoRepository.TryStartDownloadAsync( next.Id, downloadAttemptId, DateTimeOffset.UtcNow, SubscriptionAutomationDisposition.AutoDownloadQueued, cancellationToken)) + { + var racedCandidate = await animationInfoRepository.FindByIdAsync( + next.Id, + cancellationToken); + if (racedCandidate?.IsDownloadFinished == true) + return await ActivateAsync(operation.CandidateReleaseId, cancellationToken); + if (racedCandidate?.IsDownloadTracked == true) + return Result(true, "download_in_progress", false, true, operation, []); return await FailAsync(operation, "Candidate download state changed.", cancellationToken); + } - var client = downloadClientProvider.GetRequiredClient(next.DownloadType); + client = downloadClientProvider.GetRequiredClient(next.DownloadType); + submissionAttempted = true; if (!await client.SubmitDownloadTaskAsync( next.Id, next.DownloadUrl, @@ -70,11 +94,16 @@ public async Task ExecuteAsync( next.AdditionalDownloadInfo, cancellationToken)) { - await animationInfoRepository.TryCancelDownloadAsync( - next.Id, + var compensation = await CompensateDownloadStartAsync( + next, + client, downloadAttemptId, - SubscriptionAutomationDisposition.AutoDownloadFailed, - cancellationToken); + remoteMayHaveAccepted: false); + if (compensation == DownloadCompensationOutcome.RetainedForRecovery) + return await RecoveryPendingAsync( + operation, + "Download client rejected the candidate, but local state could not be restored.", + cancellationToken); return await FailAsync(operation, "Download client rejected the candidate.", cancellationToken); } @@ -82,11 +111,33 @@ await animationInfoRepository.TryCancelDownloadAsync( } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { + var compensation = downloadStartAttempted + ? await CompensateDownloadStartAsync( + next, + client, + downloadAttemptId, + submissionAttempted) + : DownloadCompensationOutcome.LocalStateRestored; + await FinalizeCancelledOperationAsync(operation, compensation); + throw; } catch (Exception exception) { + var compensation = downloadStartAttempted + ? await CompensateDownloadStartAsync( + next, + client, + downloadAttemptId, + submissionAttempted) + : DownloadCompensationOutcome.LocalStateRestored; + logger.LogWarning(exception, "Failed to queue release upgrade {OperationId}", operation.Id); + if (compensation == DownloadCompensationOutcome.RetainedForRecovery) + return await RecoveryPendingAsync( + operation, + $"{exception.Message} Download tracking was retained because cancellation could not be confirmed.", + cancellationToken); return await FailAsync(operation, exception.Message, cancellationToken); } } @@ -132,6 +183,8 @@ private async Task ActivateAsync( if (activation is null) return Result(false, "operation_not_found", false, false, null, ["No active upgrade was found for this candidate."]); + if (activation.CandidateMappings.Count == 0) + return Result(true, "mapping_pending", false, false, activation.Operation, []); // Validation is deliberately external to the mapping transaction. Until every // candidate file passes, the old virtual paths and physical files remain untouched. @@ -151,6 +204,8 @@ private async Task ActivateAsync( var now = DateTimeOffset.UtcNow; var mutation = await upgradeRepository.ActivateAsync( activation.Operation.Id, + activation.PreviousMappings, + activation.CandidateMappings, now, now.AddHours(rollbackHours), cancellationToken); @@ -238,6 +293,21 @@ private async Task FailAsync( operation.Id, summary, cancellationToken); + if (!mutation.IsSuccess) + { + var settled = mutation.Operation?.Status is + ReleaseUpgradeStatus.Applied or + ReleaseUpgradeStatus.Completed or + ReleaseUpgradeStatus.RolledBack; + return Result( + settled, + settled ? "already_settled" : mutation.Outcome, + false, + false, + mutation.Operation ?? operation, + errors ?? [summary]); + } + await incidentReporter.ReportAsync(new IncidentReport( IncidentType.FileMappingFailure, IncidentSeverity.Error, @@ -249,8 +319,175 @@ await incidentReporter.ReportAsync(new IncidentReport( errors ?? [summary]); } + private async Task RecoveryPendingAsync( + ReleaseUpgradeOperation operation, + string summary, + CancellationToken cancellationToken) + { + await incidentReporter.ReportAsync(new IncidentReport( + IncidentType.FileMappingFailure, + IncidentSeverity.Error, + "Release upgrade download recovery pending", + summary, + UpgradeSource(operation.Id)), + cancellationToken); + return Result(false, "recovery_pending", false, true, operation, [summary]); + } + + private async Task FinalizeCancelledOperationAsync( + ReleaseUpgradeOperation operation, + DownloadCompensationOutcome compensation) + { + using var cleanup = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + var summary = compensation == DownloadCompensationOutcome.LocalStateRestored + ? "Release upgrade request was cancelled and its download state was restored." + : "Release upgrade request was cancelled, but download tracking was retained for recovery."; + try + { + if (compensation == DownloadCompensationOutcome.LocalStateRestored) + await upgradeRepository.MarkFailedAsync(operation.Id, summary, cleanup.Token); + await incidentReporter.ReportAsync(new IncidentReport( + IncidentType.FileMappingFailure, + IncidentSeverity.Error, + compensation == DownloadCompensationOutcome.LocalStateRestored + ? "Release upgrade cancelled" + : "Release upgrade download recovery pending", + summary, + UpgradeSource(operation.Id)), + cleanup.Token); + } + catch (Exception exception) + { + logger.LogWarning( + exception, + "Could not finalize cancelled release upgrade {OperationId}", + operation.Id); + } + } + + private async Task CompensateDownloadStartAsync( + AnimationInfo info, + IFileDownloadClient? downloadClient, + Guid downloadAttemptId, + bool remoteMayHaveAccepted) + { + using var cleanup = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + var cancellationAttemptId = Guid.NewGuid(); + try + { + // Register cancellation before any remote I/O. Activation and the + // cancellation saga share the mapping lock, so an activation that + // already committed makes this return false and its live files are + // never deleted by stale compensation. + if (!await animationInfoRepository.TryBeginCancelDownloadAsync( + info.Id, + downloadAttemptId, + cancellationAttemptId, + cleanup.Token)) + { + var current = await animationInfoRepository.FindByIdAsync( + info.Id, + cleanup.Token); + return current is null || !current.IsDownloadTracked + ? DownloadCompensationOutcome.LocalStateRestored + : DownloadCompensationOutcome.RetainedForRecovery; + } + } + catch (Exception exception) + { + logger.LogWarning( + exception, + "Could not register compensation for release upgrade download {AnimationInfoId}", + info.Id); + return DownloadCompensationOutcome.RetainedForRecovery; + } + + if (remoteMayHaveAccepted && downloadClient is not null) + { + try + { + var cancellation = await downloadClient.CancelDownloadTaskAsync( + info.Id, + info.DownloadUrl, + info.CachedDownloadData, + info.AdditionalDownloadInfo, + removeFile: false, + cleanup.Token); + if (!cancellation.IsSuccess) + { + await QueryDownloadProgressSafelyAsync(downloadClient, info, cleanup.Token); + return DownloadCompensationOutcome.RetainedForRecovery; + } + } + catch (Exception exception) + { + logger.LogWarning( + exception, + "Could not confirm compensation for release upgrade download {AnimationInfoId}", + info.Id); + await QueryDownloadProgressSafelyAsync(downloadClient, info, cleanup.Token); + return DownloadCompensationOutcome.RetainedForRecovery; + } + } + else if (remoteMayHaveAccepted) + { + return DownloadCompensationOutcome.RetainedForRecovery; + } + + try + { + var restored = await fileMappingRepository.TryFinalizeDownloadCancellationAsync( + info.Id, + downloadAttemptId, + cancellationAttemptId, + SubscriptionAutomationDisposition.AutoDownloadFailed, + cleanup.Token); + if (restored) + return DownloadCompensationOutcome.LocalStateRestored; + + var current = await animationInfoRepository.FindByIdAsync(info.Id, cleanup.Token); + return current is null || !current.IsDownloadTracked + ? DownloadCompensationOutcome.LocalStateRestored + : DownloadCompensationOutcome.RetainedForRecovery; + } + catch (Exception exception) + { + logger.LogWarning( + exception, + "Could not restore local download state for release upgrade {AnimationInfoId}", + info.Id); + return DownloadCompensationOutcome.RetainedForRecovery; + } + } + + private static async Task QueryDownloadProgressSafelyAsync( + IFileDownloadClient downloadClient, + AnimationInfo info, + CancellationToken cancellationToken) + { + try + { + await downloadClient.SubmitQueryDownloadProgressAsync( + info.Id, + info.DownloadUrl, + info.CachedDownloadData, + info.AdditionalDownloadInfo, + cancellationToken); + } + catch + { + // Startup recovery can rediscover the persisted attempt. + } + } + private static string UpgradeSource(Guid operationId) => $"release-upgrade:{operationId:N}"; + private enum DownloadCompensationOutcome + { + LocalStateRestored, + RetainedForRecovery + } + private static ReleaseUpgradeExecutionResult Result( bool success, string outcome, From ec4ba1be6ff6e1b19e93315422c258e7edaef87f Mon Sep 17 00:00:00 2001 From: mahoshojoHCG Date: Mon, 31 Aug 2026 10:27:35 +0800 Subject: [PATCH 16/45] fix: harden backup and logical data recovery --- .containerignore | 3 +- .github/workflows/backup-restore.yml | 182 +++++-- .github/workflows/build.yml | 25 +- .github/workflows/container.yml | 2 + Containerfile | 16 +- .../DataRepository/LogicalDataTransfer.cs | 8 + .../LogicalDataTransferPostgreSqlTests.cs | 293 ++++++++++- .../LogicalDataTransferControllerTests.cs | 49 ++ .../LogicalDataTransferController.cs | 49 +- .../ApplicationContextModelSnapshot.cs | 1 + SecondDimensionWatcherReDive/Program.cs | 1 + .../LogicalDataTransferRepository.cs | 82 ++- deployments/sdw-backup | 489 +++++++++++++++--- deployments/tests/backup-restore-smoke.sh | 147 +++++- docs/backup-restore.md | 28 +- packaging/backup.env | 9 +- packaging/nfpm.yaml | 12 +- packaging/postinstall.sh | 7 +- packaging/preremove.sh | 19 +- 19 files changed, 1232 insertions(+), 190 deletions(-) diff --git a/.containerignore b/.containerignore index 6259054..f7e1b63 100644 --- a/.containerignore +++ b/.containerignore @@ -8,7 +8,8 @@ **/obj/ SecondDimensionWatcherReDive.Test/ packaging/ -deployments/ +deployments/* +!deployments/sdw-backup docs/ SecondDimensionWatcherReDive.Client/dist/ SecondDimensionWatcherReDive.Client/.parcel-cache/ diff --git a/.github/workflows/backup-restore.yml b/.github/workflows/backup-restore.yml index 67dc844..b3fd59d 100644 --- a/.github/workflows/backup-restore.yml +++ b/.github/workflows/backup-restore.yml @@ -1,46 +1,40 @@ name: Backup Restore Drill on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] + workflow_call: workflow_dispatch: permissions: contents: read -concurrency: - group: backup-restore-${{ github.ref }} - cancel-in-progress: true - jobs: drill: + name: PostgreSQL ${{ matrix.postgres }} rollback-safe drill runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + postgres: ["16", "17"] services: postgres: - image: postgres:16-alpine + image: postgres:${{ matrix.postgres }}-alpine env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres - POSTGRES_DB: sdw_source + POSTGRES_DB: postgres ports: - 5432:5432 options: >- - --health-cmd "pg_isready -U postgres -d sdw_source" + --health-cmd "pg_isready -U postgres -d postgres" --health-interval 5s --health-timeout 5s --health-retries 12 env: PGHOST: 127.0.0.1 PGPORT: 5432 - PGUSER: postgres - PGPASSWORD: postgres - PGDATABASE: sdw_source - ConnectionStrings__sdw: Host=127.0.0.1;Port=5432;Username=postgres;Password=postgres;Database=sdw_source - JwtSecret: backup-restore-drill-jwt-secret-with-at-least-32-bytes ASPNETCORE_ENVIRONMENT: Production + JwtSecret: backup-restore-drill-jwt-secret-with-at-least-32-bytes steps: - uses: actions/checkout@v7 @@ -48,25 +42,42 @@ jobs: with: dotnet-version: "10.0.x" - - name: Configure drill workspace - run: printf 'WORK_DIR=%s\n' "$RUNNER_TEMP/sdw-backup-drill" >> "$GITHUB_ENV" - - name: Install PostgreSQL client run: sudo apt-get update && sudo apt-get install -y postgresql-client + - name: Configure distinct database roles and measured capacity + shell: bash + env: + SERVICE_CONTAINER: ${{ job.services.postgres.id }} + run: | + set -Eeuo pipefail + export PGUSER=postgres PGPASSWORD=postgres PGDATABASE=postgres + psql --no-psqlrc --set=ON_ERROR_STOP=1 <<'SQL' + CREATE ROLE sdw_app LOGIN PASSWORD 'app-password'; + CREATE ROLE sdw_restore_admin LOGIN CREATEDB PASSWORD 'restore-password'; + GRANT sdw_app TO sdw_restore_admin; + CREATE DATABASE sdw_source OWNER sdw_app; + SQL + available_kib=$(docker exec "$SERVICE_CONTAINER" \ + df -Pk /var/lib/postgresql/data | awk 'NR == 2 {print $4}') + [[ "$available_kib" =~ ^[0-9]+$ ]] + printf 'POSTGRES_AVAILABLE_BYTES=%s\n' "$((available_kib * 1024))" >> "$GITHUB_ENV" + printf 'WORK_DIR=%s\n' "$RUNNER_TEMP/sdw-backup-drill-${{ matrix.postgres }}" >> "$GITHUB_ENV" + - name: Build backend run: >- dotnet build SecondDimensionWatcherReDive/SecondDimensionWatcherReDive.csproj -c Release /p:Version="$(tr -d '[:space:]' < VERSION)" - - name: Migrate and health-check source instance + - name: Migrate, register, and persist an encrypted runtime secret shell: bash run: | set -Eeuo pipefail mkdir -p "$WORK_DIR/source/keys" "$WORK_DIR/plugins/example" "$WORK_DIR/backups" - printf '{}\n' > "$WORK_DIR/source/password.json" printf 'DisableCors: true\n' > "$WORK_DIR/source/appsettings.yml" printf '{"name":"example","version":"1"}\n' > "$WORK_DIR/plugins/example/manifest.json" + export PGUSER=sdw_app PGPASSWORD=app-password PGDATABASE=sdw_source + export ConnectionStrings__sdw='Host=127.0.0.1;Port=5432;Username=sdw_app;Password=app-password;Database=sdw_source' export PasswordFile="$WORK_DIR/source/password.json" export DataProtection__KeyRingPath="$WORK_DIR/source/keys" export Config="$WORK_DIR/source/appsettings.yml" @@ -75,22 +86,43 @@ jobs: app_pid=$! trap 'kill "$app_pid" 2>/dev/null || true' EXIT for attempt in {1..60}; do - if curl --silent --fail http://127.0.0.1:5097/api/auth/allowRegister >/dev/null; then - break - fi + curl --silent --fail http://127.0.0.1:5097/api/auth/allowRegister >/dev/null && break sleep 1 done - curl --fail http://127.0.0.1:5097/api/auth/allowRegister >/dev/null + registration=$(curl --silent --show-error --fail \ + -H 'Content-Type: application/json' \ + -d '{"password":"drill-password"}' \ + http://127.0.0.1:5097/api/auth/register) + token=$(jq -er .token <<<"$registration") + curl --silent --show-error --fail \ + -H 'Content-Type: application/json' \ + -d '{"password":"drill-password"}' \ + http://127.0.0.1:5097/api/auth/login >/dev/null + revision=$(curl --silent --show-error --fail \ + -H "Authorization: Bearer $token" \ + http://127.0.0.1:5097/api/settings | jq -er .revision) + jq -n --argjson revision "$revision" \ + '{expectedRevision:$revision,tmdb:{apiKey:{operation:"set",value:"encrypted-drill-secret"}}}' | + curl --silent --show-error --fail -X PATCH \ + -H "Authorization: Bearer $token" -H 'Content-Type: application/json' \ + --data-binary @- http://127.0.0.1:5097/api/settings >/dev/null kill "$app_pid" wait "$app_pid" || true trap - EXIT - psql --no-psqlrc --command \ + psql --no-psqlrc --set=ON_ERROR_STOP=1 --command \ "INSERT INTO \"Feeds\" (\"Id\", \"Url\", \"Name\", \"CreatedAt\") VALUES ('11111111-1111-1111-1111-111111111111', 'https://example.com/feed', 'drill', now())" + schema=$(psql --no-psqlrc -Atc \ + 'SELECT "MigrationId" FROM "__EFMigrationsHistory" ORDER BY "MigrationId" DESC LIMIT 1') + [[ "$schema" =~ ^[A-Za-z0-9._:+/@-]+$ ]] + printf 'BACKUP_SCHEMA=%s\n' "$schema" >> "$GITHUB_ENV" + test -s "$WORK_DIR/source/password.json" + test "$(find "$WORK_DIR/source/keys" -type f | wc -l)" -gt 0 - - name: Create, verify, and reject corruption + - name: Create, verify, and reject archive corruption shell: bash run: | set -Eeuo pipefail + export PGUSER=sdw_app PGPASSWORD=app-password PGDATABASE=sdw_source archive=$(deployments/sdw-backup create \ --output "$WORK_DIR/backups" \ --config "$WORK_DIR/source/appsettings.yml" \ @@ -102,7 +134,7 @@ jobs: deployments/sdw-backup verify "$archive" cp "$archive" "$WORK_DIR/corrupt.tar.gz" archive_size=$(stat --format=%s "$WORK_DIR/corrupt.tar.gz") - printf 'CORRUPT' | dd of="$WORK_DIR/corrupt.tar.gz" bs=1 \ + printf CORRUPT | dd of="$WORK_DIR/corrupt.tar.gz" bs=1 \ seek=$((archive_size / 2)) conv=notrunc status=none if deployments/sdw-backup verify "$WORK_DIR/corrupt.tar.gz"; then echo "corrupted archive unexpectedly verified" >&2 @@ -110,46 +142,88 @@ jobs: fi printf '%s\n' "$archive" > "$WORK_DIR/archive-path" - - name: Restore into a fresh database and health-check it + - name: Reject failures without changing the target database shell: bash run: | set -Eeuo pipefail - createdb sdw_restore - export PGDATABASE=sdw_restore - export ConnectionStrings__sdw='Host=127.0.0.1;Port=5432;Username=postgres;Password=postgres;Database=sdw_restore' - archive=$(cat "$WORK_DIR/archive-path") - mkdir -p "$WORK_DIR/restored" - psql --no-psqlrc --command \ + export PGUSER=postgres PGPASSWORD=postgres PGDATABASE=postgres + createdb --template=sdw_source --owner=sdw_app sdw_restore + export PGUSER=sdw_restore_admin PGPASSWORD=restore-password PGDATABASE=sdw_restore + export PGMAINTENANCEDATABASE=postgres + psql --no-psqlrc --set=ON_ERROR_STOP=1 --command \ 'CREATE TABLE restore_guard (value integer NOT NULL); INSERT INTO restore_guard VALUES (1)' - if deployments/sdw-backup restore "$WORK_DIR/corrupt.tar.gz" \ - --confirm-replace --expected-version "$(tr -d '[:space:]' < VERSION)"; then + archive=$(cat "$WORK_DIR/archive-path") + mkdir -p "$WORK_DIR/failure-state" + common=(--confirm-replace --expected-version "$(tr -d '[:space:]' < VERSION)" \ + --expected-schema "$BACKUP_SCHEMA" \ + --postgres-available-bytes "$POSTGRES_AVAILABLE_BYTES" \ + --config-destination "$WORK_DIR/failure-state/appsettings.yml" \ + --password-destination "$WORK_DIR/failure-state/password.json" \ + --key-ring-destination "$WORK_DIR/failure-state/keys" \ + --plugin-destination "$WORK_DIR/failure-state/plugins" \ + --safety-directory "$WORK_DIR/backups") + if deployments/sdw-backup restore "$WORK_DIR/corrupt.tar.gz" "${common[@]}"; then echo "corrupted restore unexpectedly started" >&2 exit 1 fi - test "$(psql --no-psqlrc --tuples-only --no-align --command \ - 'SELECT value FROM restore_guard')" = "1" - if deployments/sdw-backup restore "$archive" \ - --confirm-replace --expected-version 999.0.0; then - echo "incompatible restore unexpectedly started" >&2 + test "$(psql --no-psqlrc -Atc 'SELECT value FROM restore_guard')" = 1 + test "$(psql --no-psqlrc -Atc \ + "SELECT count(*) FROM pg_database WHERE datname LIKE 'sdw_restore_%'")" = 0 + if deployments/sdw-backup restore "$archive" "${common[@]}" \ + --expected-schema definitely-not-this-schema; then + echo "incompatible schema unexpectedly restored" >&2 + exit 1 + fi + test "$(psql --no-psqlrc -Atc 'SELECT value FROM restore_guard')" = 1 + + mkdir -p "$WORK_DIR/fault-bin" + real_pg_restore=$(command -v pg_restore) + cat > "$WORK_DIR/fault-bin/pg_restore" <&2 exit 1 fi - test "$(psql --no-psqlrc --tuples-only --no-align --command \ - 'SELECT value FROM restore_guard')" = "1" + test "$(psql --no-psqlrc -Atc 'SELECT value FROM restore_guard')" = 1 + test "$(psql --no-psqlrc -Atc \ + "SELECT count(*) FROM pg_database WHERE datname LIKE 'sdw_restore_%'")" = 0 + + - name: Restore, verify owners, and start as the application role + shell: bash + run: | + set -Eeuo pipefail + export PGUSER=sdw_restore_admin PGPASSWORD=restore-password PGDATABASE=sdw_restore + export PGMAINTENANCEDATABASE=postgres + archive=$(cat "$WORK_DIR/archive-path") + mkdir -p "$WORK_DIR/restored" deployments/sdw-backup restore "$archive" \ --confirm-replace \ --expected-version "$(tr -d '[:space:]' < VERSION)" \ + --expected-schema "$BACKUP_SCHEMA" \ + --postgres-available-bytes "$POSTGRES_AVAILABLE_BYTES" \ --config-destination "$WORK_DIR/restored/appsettings.yml" \ --password-destination "$WORK_DIR/restored/password.json" \ --key-ring-destination "$WORK_DIR/restored/keys" \ --plugin-destination "$WORK_DIR/restored/plugins" \ --safety-directory "$WORK_DIR/backups" - test "$(psql --no-psqlrc --tuples-only --no-align --command \ - "SELECT to_regclass('public.restore_guard') IS NULL")" = "t" - test "$(psql --no-psqlrc --tuples-only --no-align --command 'SELECT count(*) FROM "Feeds"')" = "1" + test "$(psql --no-psqlrc -Atc "SELECT to_regclass('public.restore_guard') IS NULL")" = t + test "$(psql --no-psqlrc -Atc 'SELECT count(*) FROM "Feeds"')" = 1 + test "$(psql --no-psqlrc -Atc \ + "SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND c.relkind IN ('r','p','v','m','S','f') AND pg_get_userbyid(c.relowner) <> 'sdw_app'")" = 0 cmp "$WORK_DIR/source/password.json" "$WORK_DIR/restored/password.json" cmp "$WORK_DIR/source/appsettings.yml" "$WORK_DIR/restored/appsettings.yml" cmp "$WORK_DIR/plugins/example/manifest.json" \ "$WORK_DIR/restored/plugins/example/manifest.json" + + export PGUSER=sdw_app PGPASSWORD=app-password PGDATABASE=sdw_restore + export ConnectionStrings__sdw='Host=127.0.0.1;Port=5432;Username=sdw_app;Password=app-password;Database=sdw_restore' export PasswordFile="$WORK_DIR/restored/password.json" export DataProtection__KeyRingPath="$WORK_DIR/restored/keys" export Config="$WORK_DIR/restored/appsettings.yml" @@ -158,12 +232,16 @@ jobs: app_pid=$! trap 'kill "$app_pid" 2>/dev/null || true' EXIT for attempt in {1..60}; do - if curl --silent --fail http://127.0.0.1:5098/api/auth/allowRegister >/dev/null; then - break - fi + curl --silent --fail http://127.0.0.1:5098/api/auth/allowRegister >/dev/null && break sleep 1 done - curl --fail http://127.0.0.1:5098/api/auth/allowRegister >/dev/null + login=$(curl --silent --show-error --fail \ + -H 'Content-Type: application/json' -d '{"password":"drill-password"}' \ + http://127.0.0.1:5098/api/auth/login) + token=$(jq -er .token <<<"$login") + test "$(curl --silent --show-error --fail \ + -H "Authorization: Bearer $token" http://127.0.0.1:5098/api/settings | + jq -r '.tmdb.apiKey.isConfigured')" = true kill "$app_pid" wait "$app_pid" || true trap - EXIT diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 98cb3e1..17f5c6d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,6 +16,9 @@ permissions: contents: read jobs: + backup-restore: + uses: ./.github/workflows/backup-restore.yml + test: runs-on: ubuntu-latest steps: @@ -42,9 +45,24 @@ jobs: **/TestResults/*.trx **/TestResults/**/coverage.cobertura.xml + quality_gate: + name: Required quality gate + if: always() + needs: [test, backup-restore] + runs-on: ubuntu-latest + steps: + - name: Require tests and rollback-safe backup drill + env: + BACKUP_RESULT: ${{ needs.backup-restore.result }} + TEST_RESULT: ${{ needs.test.result }} + run: | + set -euo pipefail + test "$TEST_RESULT" = success + test "$BACKUP_RESULT" = success + build-frontend: if: github.event_name != 'pull_request' - needs: test + needs: quality_gate runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 @@ -110,6 +128,7 @@ jobs: --self-contained false \ /p:Version=${VERSION} \ -o publish + printf '%s\n' "$VERSION" > publish/VERSION - name: Install nfpm run: | @@ -257,8 +276,8 @@ jobs: # NativeAOT cross-compile is not supported, so each architecture builds on a # native runner. ubuntu-24.04-arm is GA for public repos; private repos pay # for arm minutes — switch to a self-hosted arm runner if that becomes an - # issue. The job depends on `test` only (no frontend artifact needed). - needs: test + # issue. The job needs the quality gate but no frontend artifact. + needs: quality_gate strategy: fail-fast: false matrix: diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index cc30ca5..3e6e9d4 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -67,3 +67,5 @@ jobs: ${{ env.TAG2 }} cache-from: type=gha cache-to: type=gha,mode=max + build-args: | + VERSION=${{ env.VERSION }} diff --git a/Containerfile b/Containerfile index 4bc08c8..d7ce455 100644 --- a/Containerfile +++ b/Containerfile @@ -1,3 +1,5 @@ +ARG VERSION + # Stage 1: Build frontend FROM node:24 AS frontend-build WORKDIR /app @@ -8,6 +10,7 @@ RUN yarn build # Stage 2: Build backend FROM mcr.microsoft.com/dotnet/sdk:10.0 AS backend-build +ARG VERSION WORKDIR /src COPY SecondDimensionWatcherReDive.slnx . COPY VERSION . @@ -17,19 +20,22 @@ COPY Plugins/ Plugins/ COPY Share/ Share/ COPY --from=frontend-build /app/dist SecondDimensionWatcherReDive/wwwroot/ RUN dotnet restore SecondDimensionWatcherReDive/SecondDimensionWatcherReDive.csproj -RUN dotnet publish SecondDimensionWatcherReDive/SecondDimensionWatcherReDive.csproj \ - -c Release -o /app --no-restore \ - /p:Version="$(tr -d '[:space:]' < VERSION)" +RUN effective_version="${VERSION:-$(tr -d '[:space:]' < VERSION)}" \ + && dotnet publish SecondDimensionWatcherReDive/SecondDimensionWatcherReDive.csproj \ + -c Release -o /app --no-restore \ + /p:Version="${effective_version}" \ + && printf '%s\n' "${effective_version}" > /app/VERSION # Stage 3: Runtime FROM mcr.microsoft.com/dotnet/aspnet:10.0 WORKDIR /app RUN apt-get update \ - && apt-get install -y --no-install-recommends postgresql-client \ + && apt-get install -y --no-install-recommends postgresql-client curl \ && rm -rf /var/lib/apt/lists/* COPY --from=backend-build /app . COPY deployments/sdw-backup /usr/local/bin/sdw-backup -COPY VERSION /usr/lib/sdw-redive/VERSION +RUN mkdir -p /usr/lib/sdw-redive \ + && install -m 0644 /app/VERSION /usr/lib/sdw-redive/VERSION EXPOSE 8080 # Optional: read-only NFSv4 export (set Nfs:Enabled=true to activate; publish port at run time). EXPOSE 2049 diff --git a/SecondDimensionWatcherReDive.Framework/DataRepository/LogicalDataTransfer.cs b/SecondDimensionWatcherReDive.Framework/DataRepository/LogicalDataTransfer.cs index d8d4028..7ab2049 100644 --- a/SecondDimensionWatcherReDive.Framework/DataRepository/LogicalDataTransfer.cs +++ b/SecondDimensionWatcherReDive.Framework/DataRepository/LogicalDataTransfer.cs @@ -2,6 +2,12 @@ namespace SecondDimensionWatcherReDive.Framework.DataRepository; +public static class LogicalDataTransferLimits +{ + public const int MaximumItemsPerCategory = 10_000; + public const int MaximumPayloadBytes = 10 * 1024 * 1024; +} + [Flags] [JsonConverter(typeof(JsonStringEnumConverter))] public enum LogicalDataCategory @@ -103,3 +109,5 @@ public sealed record LogicalImportResult( IReadOnlyList Messages); public sealed class LogicalDataImportConflictException(string message) : InvalidOperationException(message); + +public sealed class LogicalDataExportLimitException(string message) : InvalidOperationException(message); diff --git a/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/LogicalDataTransferPostgreSqlTests.cs b/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/LogicalDataTransferPostgreSqlTests.cs index 94437f9..bfaa3e4 100644 --- a/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/LogicalDataTransferPostgreSqlTests.cs +++ b/SecondDimensionWatcherReDive.IntegrationTest/PostgreSql/LogicalDataTransferPostgreSqlTests.cs @@ -1,5 +1,9 @@ +using System.Data.Common; using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.Extensions.DependencyInjection; using Moq; +using Npgsql; using SecondDimensionWatcherReDive.Framework.DataRepository; using SecondDimensionWatcherReDive.Repositories; using SecondDimensionWatcherReDive.Utils.FileStore; @@ -549,6 +553,245 @@ await Assert.ThrowsAsync(() => Assert.AreEqual("Existing", (await verification.Feeds.SingleAsync()).Name); } + [TestMethod] + public async Task MetadataImportReusesSharedAnimationWithoutMutatingGlobalFieldsAndUndoIsLossless() + { + var animation = Animation("tv:shared", "Target Canonical Name"); + animation.OriginalName = "Target Original"; + animation.PosterPath = "/target.jpg"; + var targetId = Guid.NewGuid(); + var siblingId = Guid.NewGuid(); + var publishedAt = DateTimeOffset.UtcNow.AddDays(-1); + const string TargetUrl = "https://example.com/shared-target.torrent"; + const string TargetPath = "/Target Canonical Name/Group/target.mkv"; + const string SiblingPath = "/Target Canonical Name/Group/sibling.mkv"; + + await using (var seed = new Models.ApplicationContext(Options)) + { + var target = Release(targetId, TargetUrl, publishedAt); + target.Animation = animation; + var sibling = Release( + siblingId, + "https://example.com/shared-sibling.torrent", + publishedAt.AddMinutes(1)); + sibling.Animation = animation; + seed.AddRange( + animation, + target, + sibling, + Mapping(targetId, TargetPath), + Mapping(siblingId, SiblingPath)); + await seed.SaveChangesAsync(); + } + + var operationId = Guid.NewGuid(); + var bundle = new LogicalDataBundle( + 1, + DateTimeOffset.UtcNow, + "1.0.0", + LogicalDataCategory.MetadataCorrections, + [], + [], + [], + [ + new LogicalMetadataCorrection( + operationId, + TargetUrl, + "[Group] Example - 02", + publishedAt, + animation.TmdbId, + "Foreign Backup Name", + "Foreign Original", + "/foreign.jpg", + "imported correction", + 1, + 2, + null, + DateTimeOffset.UtcNow) + ], + [], + null); + + await using (var importing = new Models.ApplicationContext(Options)) + { + var result = await Repository(importing).ImportAsync( + bundle, + LogicalImportConflictStrategy.Overwrite, + Guid.Empty, + CancellationToken.None); + Assert.AreEqual(1, result.Added); + } + + await using (var verification = new Models.ApplicationContext(Options)) + { + var shared = await verification.Animations.SingleAsync(); + Assert.AreEqual("Target Canonical Name", shared.Name); + Assert.AreEqual("Target Original", shared.OriginalName); + Assert.AreEqual("/target.jpg", shared.PosterPath); + Assert.AreEqual(2, await verification.AnimationInfo.CountAsync(info => info.Animation == shared)); + Assert.AreEqual( + SiblingPath, + (await verification.FileMappings.SingleAsync(mapping => + mapping.AnimationInfoId == siblingId)).VirtualPath); + } + + await using (var undoContext = new Models.ApplicationContext(Options)) + { + var undone = await new MetadataReviewRepository(undoContext, Options).UndoAsync( + operationId, + 1, + CancellationToken.None); + Assert.AreEqual(MetadataReviewMutationOutcome.Success, undone.Outcome); + } + + await using var afterUndo = new Models.ApplicationContext(Options); + var canonical = await afterUndo.Animations.SingleAsync(); + Assert.AreEqual("Target Canonical Name", canonical.Name); + Assert.AreEqual("Target Original", canonical.OriginalName); + Assert.AreEqual("/target.jpg", canonical.PosterPath); + Assert.AreEqual( + SiblingPath, + (await afterUndo.FileMappings.SingleAsync(mapping => + mapping.AnimationInfoId == siblingId)).VirtualPath); + } + + [TestMethod] + public async Task ProductionRetryStrategyRetriesWithFreshScopedStateAndCommitsOnce() + { + var transientFailure = new FailFirstConnectionInterceptor(); + var mapperScopes = 0; + var services = new ServiceCollection(); + services.AddDbContext(options => + options.UseNpgsql( + Database.GetConnectionString(), + npgsql => npgsql.EnableRetryOnFailure( + 2, + TimeSpan.Zero, + null)) + .AddInterceptors(transientFailure)); + services.AddScoped(_ => + { + Interlocked.Increment(ref mapperScopes); + return Mock.Of(); + }); + services.AddScoped(); + services.AddScoped(); + await using var provider = services.BuildServiceProvider(); + await using var requestScope = provider.CreateAsyncScope(); + var repository = requestScope.ServiceProvider + .GetRequiredService(); + var bundle = new LogicalDataBundle( + 1, + DateTimeOffset.UtcNow, + "1.0.0", + LogicalDataCategory.Feeds, + [new LogicalFeed( + Guid.NewGuid(), + "https://example.com/retried.xml", + "Retried", + DateTimeOffset.UtcNow)], + [], + [], + [], + [], + null); + + var result = await repository.ImportAsync( + bundle, + LogicalImportConflictStrategy.Fail, + Guid.Empty, + CancellationToken.None); + + Assert.AreEqual(1, result.Added); + Assert.IsGreaterThanOrEqualTo(2, transientFailure.Attempts); + Assert.IsGreaterThanOrEqualTo(2, mapperScopes, + "Every execution-strategy retry must resolve a fresh scoped worker graph."); + await using var verification = new Models.ApplicationContext(Options); + Assert.AreEqual(1, await verification.Feeds.CountAsync(feed => + feed.Url == "https://example.com/retried.xml")); + } + + [TestMethod] + public async Task ExportUsesOneRepeatableReadSnapshotAcrossCategories() + { + var feed = new Models.Feed + { + Id = Guid.NewGuid(), + Url = "https://example.com/snapshot.xml", + Name = "Snapshot", + CreatedAt = DateTimeOffset.UtcNow + }; + await using (var seed = new Models.ApplicationContext(Options)) + { + seed.Feeds.Add(feed); + await seed.SaveChangesAsync(); + } + + var barrier = new BlockAfterFeedReadInterceptor(); + var exportOptions = new DbContextOptionsBuilder() + .UseNpgsql(Database.GetConnectionString()) + .AddInterceptors(barrier) + .Options; + await using var exporting = new Models.ApplicationContext(exportOptions); + var exportTask = Repository(exporting).ExportAsync( + LogicalDataCategory.Feeds | LogicalDataCategory.AutomationPolicies, + Guid.Empty, + "1.0.0", + CancellationToken.None); + + await barrier.FeedRead.WaitAsync(TimeSpan.FromSeconds(10)); + await using (var writer = new Models.ApplicationContext(Options)) + { + writer.SubscriptionAutomationPolicies.Add(new Models.SubscriptionAutomationPolicy + { + FeedId = feed.Id, + SubtitleGroups = [], + Resolutions = [], + Codecs = [], + Languages = [], + ExcludedKeywords = [], + Mode = SubscriptionAutomationMode.ManualConfirm, + CreatedAt = DateTimeOffset.UtcNow, + UpdatedAt = DateTimeOffset.UtcNow + }); + await writer.SaveChangesAsync(); + } + barrier.Release(); + + var bundle = await exportTask; + Assert.HasCount(1, bundle.Feeds); + Assert.HasCount(0, bundle.AutomationPolicies, + "A policy committed after the first export query must not appear in the same bundle."); + } + + [TestMethod] + public async Task ExportRefusesMoreItemsThanTheImporterAccepts() + { + await using (var seed = new Models.ApplicationContext(Options)) + { + var createdAt = DateTimeOffset.UtcNow; + seed.Feeds.AddRange(Enumerable.Range( + 0, + LogicalDataTransferLimits.MaximumItemsPerCategory + 1) + .Select(index => new Models.Feed + { + Id = Guid.NewGuid(), + Url = $"https://example.com/limit/{index}", + Name = $"Feed {index}", + CreatedAt = createdAt.AddTicks(index) + })); + await seed.SaveChangesAsync(); + } + + await using var exporting = new Models.ApplicationContext(Options); + await Assert.ThrowsAsync(() => + Repository(exporting).ExportAsync( + LogicalDataCategory.Feeds, + Guid.Empty, + "1.0.0", + CancellationToken.None)); + } + private static Models.Animation Animation(string tmdbId, string name) => new() { @@ -585,8 +828,56 @@ private static Models.FileMapping Mapping(Guid animationInfoId, string virtualPa FileStore = "local" }; - private static LogicalDataTransferRepository Repository( + private static LogicalDataTransferWorker Repository( Models.ApplicationContext context, IFileMapper? fileMapper = null) => new(context, fileMapper ?? Mock.Of()); + + private sealed class FailFirstConnectionInterceptor : DbConnectionInterceptor + { + private int _attempts; + + public int Attempts => Volatile.Read(ref _attempts); + + public override ValueTask ConnectionOpeningAsync( + DbConnection connection, + ConnectionEventData eventData, + InterceptionResult result, + CancellationToken cancellationToken = default) + { + if (Interlocked.Increment(ref _attempts) == 1) + throw new NpgsqlException( + "Synthetic transient connection failure.", + new TimeoutException()); + return ValueTask.FromResult(result); + } + } + + private sealed class BlockAfterFeedReadInterceptor : DbCommandInterceptor + { + private readonly TaskCompletionSource _feedRead = new( + TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _release = new( + TaskCreationOptions.RunContinuationsAsynchronously); + private int _blocked; + + public Task FeedRead => _feedRead.Task; + + public void Release() => _release.TrySetResult(); + + public override async ValueTask ReaderExecutedAsync( + DbCommand command, + CommandExecutedEventData eventData, + DbDataReader result, + CancellationToken cancellationToken = default) + { + if (command.CommandText.Contains("FROM \"Feeds\"", StringComparison.Ordinal) && + Interlocked.Exchange(ref _blocked, 1) == 0) + { + _feedRead.TrySetResult(); + await _release.Task.WaitAsync(cancellationToken); + } + return result; + } + } } diff --git a/SecondDimensionWatcherReDive.Test/LogicalDataTransferControllerTests.cs b/SecondDimensionWatcherReDive.Test/LogicalDataTransferControllerTests.cs index 56c5744..44e7996 100644 --- a/SecondDimensionWatcherReDive.Test/LogicalDataTransferControllerTests.cs +++ b/SecondDimensionWatcherReDive.Test/LogicalDataTransferControllerTests.cs @@ -111,6 +111,55 @@ public async Task ImportPassesValidatedBundleAndConflictStrategy() Assert.AreEqual(1, result.Added); } + [TestMethod] + public async Task ExportRejectsARepositoryCategoryOverflow() + { + var repository = new Mock(); + repository.Setup(item => item.ExportAsync( + It.IsAny(), + Guid.Empty, + It.IsAny(), + It.IsAny())) + .ThrowsAsync(new LogicalDataExportLimitException("too many items")); + + var action = await Controller(repository.Object) + .ExportAsync("feeds", CancellationToken.None); + + var result = Assert.IsInstanceOfType(action); + Assert.AreEqual(StatusCodes.Status413PayloadTooLarge, result.StatusCode); + } + + [TestMethod] + public async Task ExportRejectsABundleThatCannotFitTheImportRequestLimit() + { + var repository = new Mock(); + repository.Setup(item => item.ExportAsync( + LogicalDataCategory.Feeds, + Guid.Empty, + It.IsAny(), + It.IsAny())) + .ReturnsAsync((LogicalDataCategory categories, Guid _, string version, CancellationToken _) => + { + var bundle = Bundle(version, categories); + return bundle with + { + Feeds = + [ + bundle.Feeds[0] with + { + Name = new string('x', LogicalDataTransferLimits.MaximumPayloadBytes) + } + ] + }; + }); + + var action = await Controller(repository.Object) + .ExportAsync("feeds", CancellationToken.None); + + var result = Assert.IsInstanceOfType(action); + Assert.AreEqual(StatusCodes.Status413PayloadTooLarge, result.StatusCode); + } + private static LogicalDataTransferController Controller( ILogicalDataTransferRepository repository) => new(repository) diff --git a/SecondDimensionWatcherReDive/Controllers/LogicalDataTransferController.cs b/SecondDimensionWatcherReDive/Controllers/LogicalDataTransferController.cs index 3a6f7fc..9e9f0a9 100644 --- a/SecondDimensionWatcherReDive/Controllers/LogicalDataTransferController.cs +++ b/SecondDimensionWatcherReDive/Controllers/LogicalDataTransferController.cs @@ -18,7 +18,6 @@ internal sealed class LogicalDataTransferController( ILogicalDataTransferRepository repository) : ControllerBase { private const int SupportedFormatVersion = 1; - private const int MaximumItemsPerCategory = 10_000; private static readonly Guid CurrentUserId = Guid.Empty; private static readonly string ApplicationVersion = typeof(LogicalDataTransferController).Assembly @@ -34,12 +33,34 @@ public async Task ExportAsync( if (!TryParseCategories(categories, out var selected)) return BadRequest(new { error = "Unknown data category." }); - var bundle = await repository.ExportAsync( - selected, - CurrentUserId, - ApplicationVersion, - cancellationToken); - var envelope = new External.LogicalDataExportEnvelope(bundle, Digest(bundle)); + LogicalDataBundle bundle; + try + { + bundle = await repository.ExportAsync( + selected, + CurrentUserId, + ApplicationVersion, + cancellationToken); + } + catch (LogicalDataExportLimitException exception) + { + return StatusCode(StatusCodes.Status413PayloadTooLarge, new { error = exception.Message }); + } + + var digest = Digest(bundle); + var envelope = new External.LogicalDataExportEnvelope(bundle, digest); + // Validate the larger import representation, including the longest conflict + // strategy name, so every successful export fits the import request limit. + var importBytes = JsonSerializer.SerializeToUtf8Bytes( + new External.LogicalDataImportRequest( + bundle, + digest, + LogicalImportConflictStrategy.Overwrite), + External.AppJsonSerializerContext.Default.LogicalDataImportRequest); + if (importBytes.Length > LogicalDataTransferLimits.MaximumPayloadBytes) + return StatusCode( + StatusCodes.Status413PayloadTooLarge, + new { error = $"Logical export exceeds {LogicalDataTransferLimits.MaximumPayloadBytes} bytes." }); var bytes = JsonSerializer.SerializeToUtf8Bytes( envelope, External.AppJsonSerializerContext.Default.LogicalDataExportEnvelope); @@ -49,7 +70,7 @@ public async Task ExportAsync( } [HttpPost("import")] - [RequestSizeLimit(10 * 1024 * 1024)] + [RequestSizeLimit(LogicalDataTransferLimits.MaximumPayloadBytes)] public async Task ImportAsync( [FromBody] External.LogicalDataImportRequest request, CancellationToken cancellationToken) @@ -133,13 +154,13 @@ bundle.FileNameRules is null || bundle.MetadataCorrections is null || error = "Logical export was created by an incompatible application major version."; return false; } - if (bundle.Feeds.Count > MaximumItemsPerCategory || - bundle.AutomationPolicies.Count > MaximumItemsPerCategory || - bundle.FileNameRules.Count > MaximumItemsPerCategory || - bundle.MetadataCorrections.Count > MaximumItemsPerCategory || - bundle.PlaybackProgress.Count > MaximumItemsPerCategory) + if (bundle.Feeds.Count > LogicalDataTransferLimits.MaximumItemsPerCategory || + bundle.AutomationPolicies.Count > LogicalDataTransferLimits.MaximumItemsPerCategory || + bundle.FileNameRules.Count > LogicalDataTransferLimits.MaximumItemsPerCategory || + bundle.MetadataCorrections.Count > LogicalDataTransferLimits.MaximumItemsPerCategory || + bundle.PlaybackProgress.Count > LogicalDataTransferLimits.MaximumItemsPerCategory) { - error = $"A logical export category exceeds {MaximumItemsPerCategory} items."; + error = $"A logical export category exceeds {LogicalDataTransferLimits.MaximumItemsPerCategory} items."; return false; } if ((!bundle.Categories.HasFlag(LogicalDataCategory.Feeds) && bundle.Feeds.Count > 0) || diff --git a/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs b/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs index 7364757..e9074c0 100644 --- a/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs +++ b/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs @@ -674,6 +674,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) modelBuilder.Entity("SecondDimensionWatcherReDive.Models.PlaybackPreference", b => { b.Property("UserId") + .ValueGeneratedNever() .HasColumnType("uuid"); b.Property("AudioLanguage") diff --git a/SecondDimensionWatcherReDive/Program.cs b/SecondDimensionWatcherReDive/Program.cs index 00c04c7..690cef1 100644 --- a/SecondDimensionWatcherReDive/Program.cs +++ b/SecondDimensionWatcherReDive/Program.cs @@ -278,6 +278,7 @@ builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); +builder.Services.AddScoped(); builder.Services.AddSingleton(); builder.Services.AddScoped(); builder.Services.AddScoped(); diff --git a/SecondDimensionWatcherReDive/Repositories/LogicalDataTransferRepository.cs b/SecondDimensionWatcherReDive/Repositories/LogicalDataTransferRepository.cs index fff9139..2c57b89 100644 --- a/SecondDimensionWatcherReDive/Repositories/LogicalDataTransferRepository.cs +++ b/SecondDimensionWatcherReDive/Repositories/LogicalDataTransferRepository.cs @@ -1,4 +1,6 @@ using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using System.Data; using System.Security.Cryptography; using System.Text; using SecondDimensionWatcherReDive.Framework.DataRepository; @@ -7,10 +9,53 @@ namespace SecondDimensionWatcherReDive.Repositories; -public sealed class LogicalDataTransferRepository( +public sealed class LogicalDataTransferRepository(IServiceScopeFactory scopeFactory) + : ILogicalDataTransferRepository +{ + public Task ExportAsync( + LogicalDataCategory categories, + Guid userId, + string applicationVersion, + CancellationToken cancellationToken) => + ExecuteWithFreshScopeAsync( + (worker, token) => worker.ExportAsync(categories, userId, applicationVersion, token), + cancellationToken); + + public Task ImportAsync( + LogicalDataBundle bundle, + LogicalImportConflictStrategy conflictStrategy, + Guid userId, + CancellationToken cancellationToken) => + ExecuteWithFreshScopeAsync( + (worker, token) => worker.ImportAsync(bundle, conflictStrategy, userId, token), + cancellationToken); + + private async Task ExecuteWithFreshScopeAsync( + Func> operation, + CancellationToken cancellationToken) + { + await using var strategyScope = scopeFactory.CreateAsyncScope(); + var strategyContext = strategyScope.ServiceProvider + .GetRequiredService(); + var strategy = strategyContext.Database.CreateExecutionStrategy(); + + return await strategy.ExecuteAsync( + async token => + { + // A failed Npgsql attempt can leave both EF tracking state and scoped + // collaborators unusable. Resolve the complete attempt graph again. + await using var attemptScope = scopeFactory.CreateAsyncScope(); + var worker = attemptScope.ServiceProvider + .GetRequiredService(); + return await operation(worker, token); + }, + cancellationToken); + } +} + +internal sealed class LogicalDataTransferWorker( Models.ApplicationContext context, IFileMapper fileMapper) - : ILogicalDataTransferRepository { private const int FormatVersion = 1; @@ -20,10 +65,15 @@ public async Task ExportAsync( string applicationVersion, CancellationToken cancellationToken) { + await using var transaction = await context.Database.BeginTransactionAsync( + IsolationLevel.RepeatableRead, + cancellationToken); + var feeds = categories.HasFlag(LogicalDataCategory.Feeds) ? await context.Feeds.AsNoTracking() .OrderBy(feed => feed.CreatedAt) .Select(feed => new LogicalFeed(feed.Id, feed.Url, feed.Name, feed.CreatedAt)) + .Take(LogicalDataTransferLimits.MaximumItemsPerCategory + 1) .ToListAsync(cancellationToken) : []; @@ -42,6 +92,7 @@ public async Task ExportAsync( policy.Mode, policy.CreatedAt, policy.UpdatedAt)) + .Take(LogicalDataTransferLimits.MaximumItemsPerCategory + 1) .ToListAsync(cancellationToken) : []; @@ -59,6 +110,7 @@ on rule.AnimationId equals animation.Id rule.Pattern, rule.Description, rule.CreatedAt)) + .Take(LogicalDataTransferLimits.MaximumItemsPerCategory + 1) .ToListAsync(cancellationToken) : []; @@ -82,6 +134,7 @@ on rule.AnimationId equals animation.Id operation.ProposedEpisode, operation.ProposedGroupName, operation.AppliedAt!.Value)) + .Take(LogicalDataTransferLimits.MaximumItemsPerCategory + 1) .ToListAsync(cancellationToken) : []; @@ -96,6 +149,7 @@ on rule.AnimationId equals animation.Id item.IsWatched, item.UpdatedAt, item.WatchedAt)) + .Take(LogicalDataTransferLimits.MaximumItemsPerCategory + 1) .ToListAsync(cancellationToken) : []; @@ -114,7 +168,7 @@ on rule.AnimationId equals animation.Id .FirstOrDefaultAsync(cancellationToken); } - return new LogicalDataBundle( + var result = new LogicalDataBundle( FormatVersion, DateTimeOffset.UtcNow, applicationVersion, @@ -125,6 +179,9 @@ on rule.AnimationId equals animation.Id corrections, progress, preferences); + EnsureExportCountLimits(result); + await transaction.CommitAsync(cancellationToken); + return result; } public async Task ImportAsync( @@ -370,14 +427,14 @@ await MappingTransactionLock.LockAnimationInfosAsync( animation = new Models.Animation { Id = Guid.NewGuid(), - TmdbId = imported.AnimationTmdbId + TmdbId = imported.AnimationTmdbId, + Name = imported.AnimationName, + OriginalName = imported.AnimationOriginalName, + PosterPath = imported.AnimationPosterPath }; context.Animations.Add(animation); animations.Add(animation.TmdbId, animation); } - animation.Name = imported.AnimationName; - animation.OriginalName = imported.AnimationOriginalName; - animation.PosterPath = imported.AnimationPosterPath; Models.AnimationGroup? group = null; if (!string.IsNullOrWhiteSpace(imported.GroupName) && @@ -624,6 +681,17 @@ private static string Identifier(string kind, string value) return $"{kind}:{Convert.ToHexString(digest.AsSpan(0, 8))}"; } + private static void EnsureExportCountLimits(LogicalDataBundle bundle) + { + if (bundle.Feeds.Count > LogicalDataTransferLimits.MaximumItemsPerCategory || + bundle.AutomationPolicies.Count > LogicalDataTransferLimits.MaximumItemsPerCategory || + bundle.FileNameRules.Count > LogicalDataTransferLimits.MaximumItemsPerCategory || + bundle.MetadataCorrections.Count > LogicalDataTransferLimits.MaximumItemsPerCategory || + bundle.PlaybackProgress.Count > LogicalDataTransferLimits.MaximumItemsPerCategory) + throw new LogicalDataExportLimitException( + $"A logical export category exceeds {LogicalDataTransferLimits.MaximumItemsPerCategory} items."); + } + private static void ApplyPolicy( LogicalAutomationPolicy source, Models.SubscriptionAutomationPolicy target) diff --git a/deployments/sdw-backup b/deployments/sdw-backup index 1303ae8..086f579 100755 --- a/deployments/sdw-backup +++ b/deployments/sdw-backup @@ -3,12 +3,73 @@ set -Eeuo pipefail umask 077 -readonly BACKUP_FORMAT_VERSION=1 +readonly BACKUP_FORMAT_VERSION=2 +readonly MIGRATION_LOCK_KEY=6000016593017852465 backup_temp_dir="" backup_partial_archive="" backup_partial_checksum="" +restore_candidate_database="" +migration_lease_active=false +restore_staging_paths=() +restore_state_rollback_active=false +restore_config_destination="" +restore_password_destination="" +restore_key_ring_destination="" +restore_plugin_destination="" +restore_state_timestamp="" +restore_config_had_original=false +restore_password_had_original=false +restore_key_ring_had_original=false +restore_plugin_had_original=false + +release_migration_lease() { + if [[ "${migration_lease_active}" == true ]]; then + printf '\\q\n' >&"${SDW_MIGRATION_LEASE[1]}" 2>/dev/null || true + wait "${SDW_MIGRATION_LEASE_PID}" 2>/dev/null || true + migration_lease_active=false + fi +} cleanup() { + release_migration_lease + if [[ -n "${restore_candidate_database}" ]] && command -v dropdb >/dev/null 2>&1; then + dropdb --if-exists --force \ + --maintenance-db="${PGMAINTENANCEDATABASE:-postgres}" \ + -- "${restore_candidate_database}" >/dev/null 2>&1 || true + restore_candidate_database="" + fi + if [[ "${restore_state_rollback_active}" == true ]]; then + rm -f -- "${restore_config_destination}" "${restore_password_destination}" 2>/dev/null || true + if [[ "${restore_config_had_original}" == true ]]; then + install -m 0600 \ + "${restore_config_destination}.pre-restore-${restore_state_timestamp}" \ + "${restore_config_destination}" 2>/dev/null || true + fi + if [[ "${restore_password_had_original}" == true ]]; then + install -m 0600 \ + "${restore_password_destination}.pre-restore-${restore_state_timestamp}" \ + "${restore_password_destination}" 2>/dev/null || true + fi + rm -rf -- "${restore_key_ring_destination}" "${restore_plugin_destination}" \ + 2>/dev/null || true + if [[ "${restore_key_ring_had_original}" == true ]]; then + mv -- "${restore_key_ring_destination}.pre-restore-${restore_state_timestamp}" \ + "${restore_key_ring_destination}" 2>/dev/null || true + fi + if [[ "${restore_plugin_had_original}" == true ]]; then + mv -- "${restore_plugin_destination}.pre-restore-${restore_state_timestamp}" \ + "${restore_plugin_destination}" 2>/dev/null || true + fi + restore_state_rollback_active=false + fi + local staging_path + for staging_path in "${restore_staging_paths[@]}"; do + case "${staging_path}" in + *.sdw-restore-[0-9]*.partial) + rm -rf -- "${staging_path}" 2>/dev/null || true + ;; + esac + done case "${backup_temp_dir}" in "${TMPDIR:-/tmp}"/sdw-backup.*) [[ -d "${backup_temp_dir}" ]] && rm -rf -- "${backup_temp_dir}" @@ -22,19 +83,24 @@ cleanup() { fi } -notify_failure() { +finish() { local status=$? + trap - EXIT + cleanup if [[ ${status} -ne 0 && -n "${SDW_BACKUP_FAILURE_WEBHOOK:-}" ]] && command -v curl >/dev/null 2>&1; then - curl --silent --show-error --fail --max-time 10 \ - --header 'Content-Type: application/json' \ - --data '{"event":"sdw_backup_failed"}' \ - "${SDW_BACKUP_FAILURE_WEBHOOK}" >/dev/null 2>&1 || true + local webhook_url=${SDW_BACKUP_FAILURE_WEBHOOK//\\/\\\\} + webhook_url=${webhook_url//\"/\\\"} + # Feed the secret-bearing URL through stdin so it is absent from process + # arguments and diagnostics. Only the fixed event body is on argv. + printf 'url = "%s"\n' "${webhook_url}" | + curl --config - --silent --show-error --fail --max-time 10 \ + --header 'Content-Type: application/json' \ + --data '{"event":"sdw_backup_failed"}' >/dev/null 2>&1 || true fi - return "${status}" + exit "${status}" } -trap notify_failure ERR -trap cleanup EXIT +trap finish EXIT usage() { cat <<'EOF' @@ -53,6 +119,7 @@ Create options: --retention-days DAYS Delete completed backups older than DAYS --app-version VERSION Application version written to non-secret metadata --age-recipient RECIPIENT Encrypt the final archive with age + --migration-lock-held Pre-migration hook already holds the SDWMIGR1 lease Restore options: --config-destination FILE @@ -63,12 +130,14 @@ Restore options: --expected-schema MIGRATION --age-identity FILE --safety-directory DIR Write a pre-restore database dump here + --postgres-available-bytes BYTES + Free bytes measured on the PostgreSQL default tablespace PostgreSQL is read from PGHOST/PGPORT/PGUSER/PGPASSWORD/PGDATABASE. If those are absent, ConnectionStrings__sdw may contain an ASP.NET semicolon connection string. PGMAINTENANCEDATABASE defaults to postgres and is used only to replace -the target database during restore. Restore requires a target-owning role with -CREATEDB (or a superuser). Secrets are never printed. +the target database during restore. Restore requires a superuser, or a role +with CREATEDB plus membership in the target owner role. Secrets are never printed. EOF } @@ -77,6 +146,18 @@ die() { exit 1 } +validate_failure_webhook() { + if [[ -n "${SDW_BACKUP_FAILURE_WEBHOOK:-}" ]]; then + command -v curl >/dev/null 2>&1 || + die "failure webhook is configured but curl is unavailable" + [[ ${#SDW_BACKUP_FAILURE_WEBHOOK} -le 2048 && + "${SDW_BACKUP_FAILURE_WEBHOOK}" =~ ^https?:// && + "${SDW_BACKUP_FAILURE_WEBHOOK}" != *$'\n'* && + "${SDW_BACKUP_FAILURE_WEBHOOK}" != *$'\r'* ]] || + die "failure webhook URL is invalid" + fi +} + require_command() { command -v "$1" >/dev/null 2>&1 || die "required command is missing: $1" } @@ -114,6 +195,39 @@ parse_connection_string() { export PGHOST PGPORT="${PGPORT:-5432}" PGUSER PGPASSWORD="${PGPASSWORD:-}" PGDATABASE } +acquire_migration_lease() { + local database=$1 already_held=$2 line acquired=false + if [[ "${already_held}" == true ]]; then + return + fi + [[ "${already_held}" == false ]] || die "invalid migration-lock-held state" + + coproc SDW_MIGRATION_LEASE { + psql --no-psqlrc --quiet --tuples-only --no-align \ + --set=ON_ERROR_STOP=1 --dbname "${database}" + } + migration_lease_active=true + printf 'SELECT pg_advisory_lock(%s);\n\\echo SDW_MIGRATION_LEASE_ACQUIRED\n' \ + "${MIGRATION_LOCK_KEY}" >&"${SDW_MIGRATION_LEASE[1]}" + while IFS= read -r line <&"${SDW_MIGRATION_LEASE[0]}"; do + if [[ "${line}" == SDW_MIGRATION_LEASE_ACQUIRED ]]; then + acquired=true + break + fi + done + [[ "${acquired}" == true ]] || die "could not acquire the PostgreSQL migration lease" +} + +assert_migration_lease_held() { + local database=$1 held + held=$(psql --no-psqlrc --quiet --tuples-only --no-align \ + --set=ON_ERROR_STOP=1 --dbname "${database}" --command \ + "SELECT NOT pg_try_advisory_lock(${MIGRATION_LOCK_KEY})") || + die "cannot verify the PostgreSQL migration lease" + held=${held//$'\n'/} + [[ "${held}" == t ]] || die "PostgreSQL migration lease was not held" +} + validate_output_directory() { local directory=$1 [[ -n "${directory}" && "${directory}" != "/" ]] || die "refusing unsafe backup output directory" @@ -145,10 +259,21 @@ safe_token() { } database_schema_version() { - local schema - schema=$(psql --no-psqlrc --tuples-only --no-align --command \ - 'SELECT "MigrationId" FROM "__EFMigrationsHistory" ORDER BY "MigrationId" DESC LIMIT 1' \ - 2>/dev/null || true) + local database=${1:-${PGDATABASE}} history schema + history=$(psql --no-psqlrc --tuples-only --no-align --set=ON_ERROR_STOP=1 \ + --dbname "${database}" --command \ + "SELECT to_regclass('\"__EFMigrationsHistory\"') IS NOT NULL") || + die "cannot inspect database migration history" + history=${history//$'\n'/} + case "${history}" in + f) printf 'uninitialized'; return ;; + t) ;; + *) die "database returned an invalid migration-history state" ;; + esac + schema=$(psql --no-psqlrc --tuples-only --no-align --set=ON_ERROR_STOP=1 \ + --dbname "${database}" --command \ + 'SELECT "MigrationId" FROM "__EFMigrationsHistory" ORDER BY "MigrationId" DESC LIMIT 1') || + die "cannot read database schema version" schema=${schema//$'\n'/} if [[ -z "${schema}" ]]; then printf 'uninitialized' @@ -158,6 +283,17 @@ database_schema_version() { fi } +database_size_bytes() { + local database=${1:-${PGDATABASE}} size + size=$(psql --no-psqlrc --tuples-only --no-align --set=ON_ERROR_STOP=1 \ + --dbname "${database}" --command 'SELECT pg_database_size(current_database())') || + die "cannot inspect database size" + size=${size//$'\n'/} + [[ "${size}" =~ ^[0-9]+$ && ${#size} -le 18 ]] || + die "database returned an invalid size" + printf '%s' "${size}" +} + copy_plugin_manifests() { local plugin_dir=$1 destination=$2 mkdir -p "${destination}" @@ -202,6 +338,7 @@ create_backup() { local retention_days="${SDW_BACKUP_RETENTION_DAYS:-14}" local app_version="${SDW_APP_VERSION:-}" local age_recipient="${SDW_BACKUP_AGE_RECIPIENT:-}" + local migration_lock_held="${SDW_MIGRATION_LOCK_HELD:-false}" while [[ $# -gt 0 ]]; do case "$1" in @@ -213,6 +350,7 @@ create_backup() { --retention-days) retention_days=$2; shift 2 ;; --app-version) app_version=$2; shift 2 ;; --age-recipient) age_recipient=$2; shift 2 ;; + --migration-lock-held) migration_lock_held=true; shift ;; *) die "unknown create option: $1" ;; esac done @@ -238,19 +376,37 @@ create_backup() { local stage="${backup_temp_dir}/stage" mkdir -p "${stage}/state/config" "${stage}/state/data-protection-keys" + + # Serialize schema migrations with the same cluster-wide lease as application + # startup. A pre-migration hook explicitly opts out because its parent process + # already owns this lease on a dedicated session. + acquire_migration_lease "${PGDATABASE}" "${migration_lock_held}" + assert_migration_lease_held "${PGDATABASE}" + local schema schema_after database_bytes + schema=$(database_schema_version "${PGDATABASE}") + pg_dump --format=custom --compress=6 --no-owner --no-acl --no-tablespaces \ + --file "${stage}/database.dump" + pg_restore --list "${stage}/database.dump" >/dev/null + assert_migration_lease_held "${PGDATABASE}" + schema_after=$(database_schema_version "${PGDATABASE}") + [[ "${schema_after}" == "${schema}" ]] || + die "database schema changed during backup" + database_bytes=$(database_size_bytes "${PGDATABASE}") + + # Data Protection key files are append-only. Copying them after the database + # snapshot produces a safe superset for every encrypted value in the dump. install -m 0600 -- "${config}" "${stage}/state/config/appsettings.yml" install -m 0600 -- "${password_file}" "${stage}/state/password.json" cp -a -- "${key_ring}/." "${stage}/state/data-protection-keys/" find "${stage}/state/data-protection-keys" -type d -exec chmod 0700 {} + find "${stage}/state/data-protection-keys" -type f -exec chmod 0600 {} + copy_plugin_manifests "${plugin_dir}" "${stage}/state/plugin-manifests" + assert_migration_lease_held "${PGDATABASE}" + release_migration_lease validate_stage_paths "${stage}" - pg_dump --format=custom --compress=6 --no-owner --no-acl --file "${stage}/database.dump" - pg_restore --list "${stage}/database.dump" >/dev/null - local created schema stage_bytes required_bytes + local created stage_bytes required_bytes created=$(date -u +%Y-%m-%dT%H:%M:%SZ) - schema=$(database_schema_version) stage_bytes=$(du -sb "${stage}" | awk '{print $1}') required_bytes=$((stage_bytes * 2 + 67108864)) { @@ -259,6 +415,7 @@ create_backup() { printf 'application_version=%s\n' "${app_version}" printf 'schema_version=%s\n' "${schema}" printf 'required_free_bytes=%s\n' "${required_bytes}" + printf 'database_bytes=%s\n' "${database_bytes}" printf 'media_files_included=false\n' } >"${stage}/manifest.env" ( @@ -435,16 +592,19 @@ prepare_archive() { ) || die "backup checksum verification failed" pg_restore --list "${backup_temp_dir}/extracted/database.dump" >/dev/null || die "PostgreSQL dump verification failed" - local format required created application schema media_scope + local format required database_bytes created application schema media_scope format=$(manifest_value "${backup_temp_dir}/extracted/manifest.env" format_version) [[ "${format}" == "${BACKUP_FORMAT_VERSION}" ]] || die "unsupported backup format" required=$(manifest_value "${backup_temp_dir}/extracted/manifest.env" required_free_bytes) + database_bytes=$(manifest_value "${backup_temp_dir}/extracted/manifest.env" database_bytes) created=$(manifest_value "${backup_temp_dir}/extracted/manifest.env" created_utc) application=$(manifest_value "${backup_temp_dir}/extracted/manifest.env" application_version) schema=$(manifest_value "${backup_temp_dir}/extracted/manifest.env" schema_version) media_scope=$(manifest_value "${backup_temp_dir}/extracted/manifest.env" media_files_included) [[ "${required}" =~ ^[0-9]+$ && ${#required} -le 18 ]] || die "backup has invalid space metadata" + [[ "${database_bytes}" =~ ^[0-9]+$ && ${#database_bytes} -le 18 ]] || + die "backup has invalid database size metadata" [[ "${created}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || die "backup has invalid timestamp metadata" safe_token "${application}" && safe_token "${schema}" || @@ -511,38 +671,197 @@ prepare_database_replacement() { die "PostgreSQL role must own the target and have CREATEDB, or be superuser" } -replace_database_from_archive() { - local archive=$1 maintenance_database=${PGMAINTENANCEDATABASE:-postgres} - local database_owner database_encoding database_collation database_ctype +postgresql_available_bytes() { + local database=$1 override=$2 path value + if [[ -n "${override}" ]]; then + [[ "${override}" =~ ^[0-9]+$ && ${#override} -le 18 ]] || + die "PostgreSQL available bytes must be a non-negative integer" + printf '%s' "${override}" + return + fi + + path=$(psql --no-psqlrc --dbname "${database}" --tuples-only --no-align \ + --set=ON_ERROR_STOP=1 --command \ + "SELECT CASE WHEN tablespace.spcname = 'pg_default' + THEN current_setting('data_directory') || '/base' + ELSE pg_tablespace_location(tablespace.oid) + END + FROM pg_database AS database + JOIN pg_tablespace AS tablespace ON tablespace.oid = database.dattablespace + WHERE database.datname = current_database()") || + die "cannot inspect the PostgreSQL default tablespace" + path=${path//$'\n'/} + [[ -n "${path}" ]] || die "PostgreSQL returned an empty tablespace path" + [[ -d "${path}" ]] || + die "PostgreSQL storage is remote; provide --postgres-available-bytes from database-host monitoring" + value=$(available_bytes "${path}") || die "cannot inspect PostgreSQL storage capacity" + [[ "${value}" =~ ^[0-9]+$ ]] || die "PostgreSQL storage returned invalid capacity" + printf '%s' "${value}" +} + +restore_candidate_from_archive() { + local archive=$1 backup_schema=$2 postgres_available=$3 + local maintenance_database=${PGMAINTENANCEDATABASE:-postgres} + local database_owner database_encoding database_collation database_ctype database_tablespace database_owner=$(psql --no-psqlrc --dbname "${PGDATABASE}" \ --tuples-only --no-align --set=ON_ERROR_STOP=1 --command \ - "SELECT pg_get_userbyid(datdba) FROM pg_database WHERE datname = current_database()") + "SELECT pg_get_userbyid(datdba) FROM pg_database WHERE datname = current_database()") || + die "cannot inspect target database owner" database_encoding=$(psql --no-psqlrc --dbname "${PGDATABASE}" \ --tuples-only --no-align --set=ON_ERROR_STOP=1 --command \ - "SELECT pg_encoding_to_char(encoding) FROM pg_database WHERE datname = current_database()") + "SELECT pg_encoding_to_char(encoding) FROM pg_database WHERE datname = current_database()") || + die "cannot inspect target database encoding" database_collation=$(psql --no-psqlrc --dbname "${PGDATABASE}" \ --tuples-only --no-align --set=ON_ERROR_STOP=1 --command \ - "SELECT datcollate FROM pg_database WHERE datname = current_database()") + "SELECT datcollate FROM pg_database WHERE datname = current_database()") || + die "cannot inspect target database collation" database_ctype=$(psql --no-psqlrc --dbname "${PGDATABASE}" \ --tuples-only --no-align --set=ON_ERROR_STOP=1 --command \ - "SELECT datctype FROM pg_database WHERE datname = current_database()") + "SELECT datctype FROM pg_database WHERE datname = current_database()") || + die "cannot inspect target database character classification" + database_tablespace=$(psql --no-psqlrc --dbname "${PGDATABASE}" \ + --tuples-only --no-align --set=ON_ERROR_STOP=1 --command \ + "SELECT tablespace.spcname + FROM pg_database AS database + JOIN pg_tablespace AS tablespace ON tablespace.oid = database.dattablespace + WHERE database.datname = current_database()") || + die "cannot inspect target database tablespace" [[ -n "${database_owner}" && -n "${database_encoding}" && - -n "${database_collation}" && -n "${database_ctype}" ]] || + -n "${database_collation}" && -n "${database_ctype}" && + -n "${database_tablespace}" ]] || die "target database properties are unavailable" - # pg_restore --clean only drops objects present in the archive. Recreate the - # database so objects introduced after the backup cannot survive a replacement - # restore and conflict with the restored EF migration history. - dropdb --force --maintenance-db="${maintenance_database}" -- "${PGDATABASE}" - createdb --maintenance-db="${maintenance_database}" \ - --template=template0 \ - --owner="${database_owner}" \ - --encoding="${database_encoding}" \ - --lc-collate="${database_collation}" \ - --lc-ctype="${database_ctype}" \ - -- "${PGDATABASE}" - pg_restore --exit-on-error --no-owner --no-acl \ - --dbname "${PGDATABASE}" "${archive}" + local manifest_database_bytes required_database_bytes + manifest_database_bytes=$(manifest_value "${backup_temp_dir}/extracted/manifest.env" database_bytes) + required_database_bytes=$((manifest_database_bytes + manifest_database_bytes / 4 + 67108864)) + [[ "${postgres_available}" -gt "${required_database_bytes}" ]] || + die "insufficient PostgreSQL tablespace capacity for a rollback-safe candidate restore" + + restore_candidate_database="sdw_restore_$(date -u +%Y%m%d%H%M%S)_$$" + local candidate_exists + candidate_exists=$(psql --no-psqlrc --dbname "${maintenance_database}" \ + --tuples-only --no-align --set=ON_ERROR_STOP=1 \ + --set=candidate="${restore_candidate_database}" <<'SQL' +SELECT EXISTS (SELECT 1 FROM pg_database WHERE datname = :'candidate'); +SQL + ) || + die "cannot inspect candidate database name" + [[ "${candidate_exists}" == f ]] || die "generated candidate database already exists" + + local -a create_options=( + --maintenance-db="${maintenance_database}" + --template=template0 + --owner="${database_owner}" + --encoding="${database_encoding}" + --lc-collate="${database_collation}" + --lc-ctype="${database_ctype}") + # Explicitly naming pg_default unnecessarily requires CREATE privilege on + # that tablespace. Custom target tablespaces do need to be preserved. + if [[ "${database_tablespace}" != pg_default ]]; then + create_options+=(--tablespace="${database_tablespace}") + fi + createdb "${create_options[@]}" -- "${restore_candidate_database}" || + die "cannot create restore candidate database" + pg_restore --exit-on-error --single-transaction --no-owner --no-acl \ + --role="${database_owner}" --dbname "${restore_candidate_database}" "${archive}" || + die "candidate database restore failed" + + [[ "$(database_schema_version "${restore_candidate_database}")" == "${backup_schema}" ]] || + die "candidate database schema does not match the backup" + + local wrong_owners invalid_indexes app_access + wrong_owners=$(psql --no-psqlrc --dbname "${restore_candidate_database}" \ + --tuples-only --no-align --set=ON_ERROR_STOP=1 --set=owner="${database_owner}" <<'SQL' + SELECT + (SELECT count(*) FROM pg_class AS object + JOIN pg_namespace AS namespace ON namespace.oid = object.relnamespace + WHERE namespace.nspname NOT IN ('pg_catalog', 'information_schema') + AND namespace.nspname !~ '^pg_toast' + AND object.relkind IN ('r','p','v','m','S','f') + AND pg_get_userbyid(object.relowner) <> :'owner') + + (SELECT count(*) FROM pg_proc AS object + JOIN pg_namespace AS namespace ON namespace.oid = object.pronamespace + WHERE namespace.nspname NOT IN ('pg_catalog', 'information_schema') + AND namespace.nspname !~ '^pg_toast' + AND pg_get_userbyid(object.proowner) <> :'owner') + + (SELECT count(*) FROM pg_type AS object + JOIN pg_namespace AS namespace ON namespace.oid = object.typnamespace + WHERE namespace.nspname NOT IN ('pg_catalog', 'information_schema') + AND namespace.nspname !~ '^pg_toast' + AND pg_get_userbyid(object.typowner) <> :'owner') + + (SELECT count(*) FROM pg_namespace AS object + WHERE object.nspname NOT IN ('public', 'pg_catalog', 'information_schema') + AND object.nspname !~ '^pg_toast' + AND pg_get_userbyid(object.nspowner) <> :'owner') + + (SELECT count(*) FROM pg_extension AS object + WHERE object.extname <> 'plpgsql' + AND pg_get_userbyid(object.extowner) <> :'owner'); +SQL + ) || + die "cannot validate restored object ownership" + [[ "${wrong_owners}" == 0 ]] || die "candidate database contains objects not owned by the application role" + + invalid_indexes=$(psql --no-psqlrc --dbname "${restore_candidate_database}" \ + --tuples-only --no-align --set=ON_ERROR_STOP=1 --command \ + 'SELECT count(*) FROM pg_index WHERE NOT indisvalid OR NOT indisready') || + die "cannot validate candidate indexes" + [[ "${invalid_indexes}" == 0 ]] || die "candidate database contains invalid indexes" + app_access=$(psql --no-psqlrc --dbname "${restore_candidate_database}" \ + --tuples-only --no-align --set=ON_ERROR_STOP=1 --set=owner="${database_owner}" <<'SQL' + SET ROLE :"owner"; + SELECT CASE WHEN has_schema_privilege(current_user, 'public', 'USAGE') + AND has_schema_privilege(current_user, 'public', 'CREATE') + AND NOT EXISTS ( + SELECT 1 FROM pg_class AS object + JOIN pg_namespace AS namespace ON namespace.oid = object.relnamespace + WHERE namespace.nspname = 'public' + AND object.relkind IN ('r','p','S') + AND NOT has_table_privilege(current_user, object.oid, 'SELECT')) + THEN 'yes' ELSE 'no' END; +SQL + ) || + die "application role cannot access the candidate database" + app_access=${app_access##*$'\n'} + [[ "${app_access}" == yes ]] || die "candidate database failed application-role readiness checks" +} + +switch_candidate_database() { + local maintenance_database=${PGMAINTENANCEDATABASE:-postgres} + local previous_database="sdw_previous_$(date -u +%Y%m%d%H%M%S)_$$" active_connections + active_connections=$(psql --no-psqlrc --dbname "${maintenance_database}" \ + --tuples-only --no-align --set=ON_ERROR_STOP=1 --set=target="${PGDATABASE}" \ + --set=candidate="${restore_candidate_database}" <<'SQL' +SELECT count(*) FROM pg_stat_activity + WHERE datname IN (:'target', :'candidate') AND pid <> pg_backend_pid(); +SQL + ) || + die "cannot inspect active database connections" + [[ "${active_connections}" == 0 ]] || + die "target database still has active connections; stop the application before restore" + + psql --no-psqlrc --dbname "${maintenance_database}" --set=ON_ERROR_STOP=1 \ + --set=target="${PGDATABASE}" --set=previous="${previous_database}" \ + >/dev/null <<'SQL' || +ALTER DATABASE :"target" RENAME TO :"previous"; +SQL + die "could not preserve the original database before switching" + if ! psql --no-psqlrc --dbname "${maintenance_database}" --set=ON_ERROR_STOP=1 \ + --set=candidate="${restore_candidate_database}" --set=target="${PGDATABASE}" \ + >/dev/null <<'SQL' +ALTER DATABASE :"candidate" RENAME TO :"target"; +SQL + then + psql --no-psqlrc --dbname "${maintenance_database}" --set=ON_ERROR_STOP=1 \ + --set=previous="${previous_database}" --set=target="${PGDATABASE}" \ + >/dev/null <<'SQL' || +ALTER DATABASE :"previous" RENAME TO :"target"; +SQL + die "candidate switch failed and automatic database-name rollback also failed" + die "candidate switch failed; original database name was restored" + fi + restore_candidate_database="" + restore_state_rollback_active=false + printf 'original database retained as %s for rollback\n' "${previous_database}" >&2 } restore_backup() { @@ -557,6 +876,7 @@ restore_backup() { local expected_schema="${SDW_EXPECTED_SCHEMA_VERSION:-}" local identity="${SDW_BACKUP_AGE_IDENTITY:-}" local safety_directory="${SDW_BACKUP_DIRECTORY:-/var/lib/sdw-redive/backups}" + local postgres_available="${SDW_POSTGRESQL_AVAILABLE_BYTES:-}" local confirmed=false allow_version_mismatch=false while [[ $# -gt 0 ]]; do @@ -569,6 +889,7 @@ restore_backup() { --expected-schema) expected_schema=$2; shift 2 ;; --age-identity) identity=$2; shift 2 ;; --safety-directory) safety_directory=$2; shift 2 ;; + --postgres-available-bytes) postgres_available=$2; shift 2 ;; --allow-version-mismatch) allow_version_mismatch=true; shift ;; --confirm-replace) confirmed=true; shift ;; *) die "unknown restore option: $1" ;; @@ -586,6 +907,10 @@ restore_backup() { prepare_database_replacement prepare_archive "${archive}" "${identity}" + local maintenance_database=${PGMAINTENANCEDATABASE:-postgres} + acquire_migration_lease "${maintenance_database}" false + assert_migration_lease_held "${maintenance_database}" + local extracted="${backup_temp_dir}/extracted" local manifest="${extracted}/manifest.env" local backup_version backup_schema @@ -594,13 +919,23 @@ restore_backup() { if [[ "${allow_version_mismatch}" != true ]]; then [[ -n "${expected_version}" ]] || die "--expected-version is required unless --allow-version-mismatch is explicit" - [[ "$(major_version "${backup_version}")" == "$(major_version "${expected_version}")" ]] || + local backup_major expected_major + backup_major=$(major_version "${backup_version}") || + die "backup application version is invalid" + expected_major=$(major_version "${expected_version}") || + die "expected application version is invalid" + [[ "${backup_major}" == "${expected_major}" ]] || die "backup application major version is incompatible" fi - if [[ -n "${expected_schema}" && "${backup_schema}" != "${expected_schema}" ]]; then + [[ -n "${expected_schema}" ]] || + die "--expected-schema is required for fail-closed application compatibility" + safe_token "${expected_schema}" || die "expected schema contains unsupported characters" + if [[ "${backup_schema}" != "${expected_schema}" ]]; then die "backup schema is incompatible with the requested schema" fi + postgres_available=$(postgresql_available_bytes "${PGDATABASE}" "${postgres_available}") + local restored_config required_bytes destination_parent probe restored_config=$(find "${extracted}/state/config" -maxdepth 1 -type f -print -quit) [[ -n "${restored_config}" && -f "${extracted}/state/password.json" && @@ -624,44 +959,80 @@ restore_backup() { local timestamp safety_dump timestamp=$(date -u +%Y%m%dT%H%M%SZ) safety_dump="${safety_directory}/pre-restore-${timestamp}-$$.dump" - pg_dump --format=custom --compress=6 --no-owner --no-acl --file "${safety_dump}.partial" + pg_dump --format=custom --compress=6 --no-owner --no-acl --no-tablespaces \ + --file "${safety_dump}.partial" chmod 0600 "${safety_dump}.partial" mv "${safety_dump}.partial" "${safety_dump}" - replace_database_from_archive "${extracted}/database.dump" - [[ "$(database_schema_version)" == "${backup_schema}" ]] || - die "restored database schema does not match the backup" + restore_candidate_from_archive \ + "${extracted}/database.dump" "${backup_schema}" "${postgres_available}" + + # Prepare every state artifact before the database-name switch. Each staged + # path lives beside its destination, so the final rename is same-filesystem. + local config_partial="${config_destination}.sdw-restore-$$.partial" + local password_partial="${password_destination}.sdw-restore-$$.partial" + local key_ring_partial="${key_ring_destination}.sdw-restore-$$.partial" + local plugin_partial="${plugin_destination}.sdw-restore-$$.partial" + [[ ! -e "${config_partial}" && ! -L "${config_partial}" && + ! -e "${password_partial}" && ! -L "${password_partial}" && + ! -e "${key_ring_partial}" && ! -L "${key_ring_partial}" && + ! -e "${plugin_partial}" && ! -L "${plugin_partial}" ]] || + die "restore staging path already exists" + restore_staging_paths=( + "${config_partial}" "${password_partial}" + "${key_ring_partial}" "${plugin_partial}") + install -m 0600 "${restored_config}" "${config_partial}" + install -m 0600 "${extracted}/state/password.json" "${password_partial}" + mkdir -m 0700 "${key_ring_partial}" + cp -a "${extracted}/state/data-protection-keys/." "${key_ring_partial}/" + find "${key_ring_partial}" -type d -exec chmod 0700 {} + + find "${key_ring_partial}" -type f -exec chmod 0600 {} + + mkdir -m 0700 "${plugin_partial}" + if [[ -d "${extracted}/state/plugin-manifests" && + ! -f "${extracted}/state/plugin-manifests/none" ]]; then + cp -a "${extracted}/state/plugin-manifests/." "${plugin_partial}/" + fi if [[ -f "${config_destination}" ]]; then + restore_config_had_original=true install -m 0600 "${config_destination}" "${config_destination}.pre-restore-${timestamp}" fi if [[ -f "${password_destination}" ]]; then + restore_password_had_original=true install -m 0600 "${password_destination}" "${password_destination}.pre-restore-${timestamp}" fi - install -m 0600 "${restored_config}" "${config_destination}.partial" - mv "${config_destination}.partial" "${config_destination}" - install -m 0600 "${extracted}/state/password.json" "${password_destination}.partial" - mv "${password_destination}.partial" "${password_destination}" + restore_config_destination="${config_destination}" + restore_password_destination="${password_destination}" + restore_key_ring_destination="${key_ring_destination}" + restore_plugin_destination="${plugin_destination}" + restore_state_timestamp="${timestamp}" + restore_state_rollback_active=true + mv "${config_partial}" "${config_destination}" + mv "${password_partial}" "${password_destination}" if [[ -d "${key_ring_destination}" ]]; then + restore_key_ring_had_original=true mv "${key_ring_destination}" "${key_ring_destination}.pre-restore-${timestamp}" fi - mkdir -p "${key_ring_destination}" - chmod 0700 "${key_ring_destination}" - cp -a "${extracted}/state/data-protection-keys/." "${key_ring_destination}/" - find "${key_ring_destination}" -type f -exec chmod 0600 {} + + mv "${key_ring_partial}" "${key_ring_destination}" - if [[ -d "${extracted}/state/plugin-manifests" && - ! -f "${extracted}/state/plugin-manifests/none" ]]; then - mkdir -p "${plugin_destination}" - cp -a "${extracted}/state/plugin-manifests/." "${plugin_destination}/" + if [[ -d "${plugin_destination}" ]]; then + restore_plugin_had_original=true + mv "${plugin_destination}" "${plugin_destination}.pre-restore-${timestamp}" fi + mv "${plugin_partial}" "${plugin_destination}" + assert_migration_lease_held "${maintenance_database}" + switch_candidate_database + restore_state_rollback_active=false + restore_staging_paths=() + release_migration_lease printf 'restore completed; restart the application and run its health check\n' } command=${1:-} [[ -n "${command}" ]] || { usage; exit 1; } shift +validate_failure_webhook case "${command}" in create) create_backup "$@" ;; list) list_backups "$@" ;; diff --git a/deployments/tests/backup-restore-smoke.sh b/deployments/tests/backup-restore-smoke.sh index 34d7059..e6a548c 100755 --- a/deployments/tests/backup-restore-smoke.sh +++ b/deployments/tests/backup-restore-smoke.sh @@ -15,33 +15,93 @@ trap cleanup EXIT mkdir -p \ "${drill_root}/keys" \ - "${drill_root}/plugins" \ + "${drill_root}/plugins/example" \ "${drill_root}/backups" \ "${drill_root}/restored" install -m 0600 "${repo_root}/VERSION" "${drill_root}/appsettings.yml" install -m 0600 "${repo_root}/VERSION" "${drill_root}/password.json" +printf '\n' >"${drill_root}/keys/key.xml" +printf '{"name":"example","version":"1"}\n' >"${drill_root}/plugins/example/manifest.json" app_version=$(tr -d '[:space:]' <"${repo_root}/VERSION") +# EXIT, rather than ERR alone, must notify explicit die/exit paths without +# exposing the configured URL or any environment secret in the diagnostic. +mkdir "${drill_root}/webhook-bin" +cat >"${drill_root}/webhook-bin/curl" <<'EOF' +#!/usr/bin/env bash +while [[ $# -gt 0 ]]; do + if [[ "$1" == --data ]]; then + printf '%s\n' "$2" >"${WEBHOOK_CAPTURE}" + exit 0 + fi + shift +done +exit 2 +EOF +chmod +x "${drill_root}/webhook-bin/curl" +webhook_log="${drill_root}/webhook.log" +if WEBHOOK_CAPTURE="${drill_root}/webhook-payload" \ + SDW_BACKUP_FAILURE_WEBHOOK='https://secret.invalid/opaque-token' \ + PATH="${drill_root}/webhook-bin:/usr/bin:/bin" \ + "${repo_root}/deployments/sdw-backup" invalid-command >"${webhook_log}" 2>&1; then + printf 'invalid command unexpectedly succeeded\n' >&2 + exit 1 +fi +test "$(cat "${drill_root}/webhook-payload")" = '{"event":"sdw_backup_failed"}' +! grep -q 'opaque-token' "${webhook_log}" +if SDW_BACKUP_FAILURE_WEBHOOK='https://secret.invalid/opaque-token' PATH="${drill_root}/empty-path" \ + /bin/bash "${repo_root}/deployments/sdw-backup" invalid-command >/dev/null 2>&1; then + printf 'configured webhook unexpectedly accepted a missing curl\n' >&2 + exit 1 +fi + podman run --rm --detach --name "${container_name}" \ --env POSTGRES_PASSWORD=postgres \ --env POSTGRES_USER=postgres \ - --env POSTGRES_DB=sdw_source \ + --env POSTGRES_DB=postgres \ --publish 127.0.0.1::5432 \ postgres:17-alpine >/dev/null port=$(podman port "${container_name}" 5432/tcp | sed -E 's/.*:([0-9]+)$/\1/') -export PGHOST=127.0.0.1 PGPORT="${port}" PGUSER=postgres PGPASSWORD=postgres PGDATABASE=sdw_source +export PGHOST=127.0.0.1 PGPORT="${port}" PGUSER=postgres PGPASSWORD=postgres PGDATABASE=postgres for _ in {1..30}; do pg_isready --quiet && break sleep 1 done pg_isready --quiet +psql --no-psqlrc --set=ON_ERROR_STOP=1 <<'SQL' >/dev/null +CREATE ROLE sdw_app LOGIN PASSWORD 'app-password'; +CREATE ROLE sdw_restore_admin LOGIN CREATEDB PASSWORD 'restore-password'; +GRANT sdw_app TO sdw_restore_admin; +CREATE DATABASE sdw_source OWNER sdw_app; +SQL + +export PGUSER=sdw_app PGPASSWORD=app-password PGDATABASE=sdw_source psql --no-psqlrc --set=ON_ERROR_STOP=1 --command \ "CREATE TABLE drill_items (id integer PRIMARY KEY, value text NOT NULL); INSERT INTO drill_items VALUES (1, 'round-trip');" \ >/dev/null -export ConnectionStrings__sdw="Host=127.0.0.1;Port=${port};User ID=postgres;Password=postgres;Database=sdw_source" -unset PGHOST PGPORT PGUSER PGPASSWORD PGDATABASE +# Simulate the application startup hook already owning SDWMIGR1. The explicit +# flag must avoid a child-process self-deadlock while the dump remains valid. +coproc HELD_LEASE { + psql --no-psqlrc --quiet --tuples-only --no-align --set=ON_ERROR_STOP=1 +} +printf 'SELECT pg_advisory_lock(6000016593017852465);\n\\echo HELD\n' >&"${HELD_LEASE[1]}" +while IFS= read -r lease_line <&"${HELD_LEASE[0]}"; do + [[ "${lease_line}" == HELD ]] && break +done +archive=$("${repo_root}/deployments/sdw-backup" create --migration-lock-held \ + --output "${drill_root}/backups" \ + --config "${drill_root}/appsettings.yml" \ + --password-file "${drill_root}/password.json" \ + --key-ring "${drill_root}/keys" \ + --plugin-dir "${drill_root}/plugins" \ + --retention-days 7 \ + --app-version "${app_version}") +printf '\\q\n' >&"${HELD_LEASE[1]}" +wait "${HELD_LEASE_PID}" + +# Exercise normal lease acquisition too; use this archive for the restore drill. archive=$("${repo_root}/deployments/sdw-backup" create \ --output "${drill_root}/backups" \ --config "${drill_root}/appsettings.yml" \ @@ -50,10 +110,8 @@ archive=$("${repo_root}/deployments/sdw-backup" create \ --plugin-dir "${drill_root}/plugins" \ --retention-days 7 \ --app-version "${app_version}") -unset ConnectionStrings__sdw -export PGHOST=127.0.0.1 PGPORT="${port}" PGUSER=postgres PGPASSWORD=postgres PGDATABASE=sdw_source -"${repo_root}/deployments/sdw-backup" verify "${archive}" +"${repo_root}/deployments/sdw-backup" verify "${archive}" cp -- "${archive}" "${drill_root}/corrupt.tar.gz" archive_size=$(stat --format=%s "${drill_root}/corrupt.tar.gz") printf CORRUPT | dd of="${drill_root}/corrupt.tar.gz" bs=1 \ @@ -63,23 +121,66 @@ if "${repo_root}/deployments/sdw-backup" verify "${drill_root}/corrupt.tar.gz" > exit 1 fi -createdb sdw_restore -export PGDATABASE=sdw_restore +export PGUSER=postgres PGPASSWORD=postgres PGDATABASE=postgres +createdb --template=sdw_source --owner=sdw_app sdw_restore +export PGUSER=sdw_restore_admin PGPASSWORD=restore-password PGDATABASE=sdw_restore +export PGMAINTENANCEDATABASE=postgres psql --no-psqlrc --set=ON_ERROR_STOP=1 --command \ - "CREATE TABLE destination_only (id integer); INSERT INTO destination_only VALUES (99);" \ + "CREATE TABLE restore_guard (id integer PRIMARY KEY); INSERT INTO restore_guard VALUES (99);" \ >/dev/null -"${repo_root}/deployments/sdw-backup" restore "${archive}" \ - --confirm-replace \ - --expected-version "${app_version}" \ - --config-destination "${drill_root}/restored/appsettings.yml" \ - --password-destination "${drill_root}/restored/password.json" \ - --key-ring-destination "${drill_root}/restored/keys" \ - --plugin-destination "${drill_root}/restored/plugins" \ - --safety-directory "${drill_root}/backups" +available_kib=$(podman exec "${container_name}" df -Pk /var/lib/postgresql/data | awk 'NR == 2 {print $4}') +available_bytes=$((available_kib * 1024)) +restore_options=( + --confirm-replace + --expected-version "${app_version}" + --expected-schema uninitialized + --postgres-available-bytes "${available_bytes}" + --config-destination "${drill_root}/restored/appsettings.yml" + --password-destination "${drill_root}/restored/password.json" + --key-ring-destination "${drill_root}/restored/keys" + --plugin-destination "${drill_root}/restored/plugins" + --safety-directory "${drill_root}/backups") + +if "${repo_root}/deployments/sdw-backup" restore "${archive}" \ + "${restore_options[@]}" --expected-schema incompatible >/dev/null 2>&1; then + printf 'schema mismatch unexpectedly restored\n' >&2 + exit 1 +fi +test "$(psql --no-psqlrc -Atc 'SELECT id FROM restore_guard')" = 99 +if "${repo_root}/deployments/sdw-backup" restore "${archive}" \ + "${restore_options[@]/${available_bytes}/1}" >/dev/null 2>&1; then + printf 'insufficient PostgreSQL capacity unexpectedly restored\n' >&2 + exit 1 +fi +test "$(psql --no-psqlrc -Atc 'SELECT id FROM restore_guard')" = 99 + +mkdir "${drill_root}/fault-bin" +real_pg_restore=$(command -v pg_restore) +cat >"${drill_root}/fault-bin/pg_restore" </dev/null 2>&1; then + printf 'injected candidate failure unexpectedly restored\n' >&2 + exit 1 +fi +test "$(psql --no-psqlrc -Atc 'SELECT id FROM restore_guard')" = 99 +test "$(psql --no-psqlrc -Atc "SELECT count(*) FROM pg_database WHERE datname LIKE 'sdw_restore_%'")" = 0 -test "$(psql --no-psqlrc --tuples-only --no-align --command \ - 'SELECT value FROM drill_items WHERE id = 1')" = round-trip -test "$(psql --no-psqlrc --tuples-only --no-align --command \ - "SELECT to_regclass('public.destination_only') IS NULL")" = t +"${repo_root}/deployments/sdw-backup" restore "${archive}" "${restore_options[@]}" +test "$(psql --no-psqlrc -Atc 'SELECT value FROM drill_items WHERE id = 1')" = round-trip +test "$(psql --no-psqlrc -Atc "SELECT to_regclass('public.restore_guard') IS NULL")" = t +test "$(psql --no-psqlrc -Atc \ + "SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND c.relkind IN ('r','p','v','m','S','f') AND pg_get_userbyid(c.relowner) <> 'sdw_app'")" = 0 cmp "${drill_root}/password.json" "${drill_root}/restored/password.json" +cmp "${drill_root}/keys/key.xml" "${drill_root}/restored/keys/key.xml" +cmp "${drill_root}/plugins/example/manifest.json" \ + "${drill_root}/restored/plugins/example/manifest.json" +export PGUSER=sdw_app PGPASSWORD=app-password +test "$(psql --no-psqlrc -Atc 'SELECT value FROM drill_items WHERE id = 1')" = round-trip printf 'backup restore smoke test passed\n' diff --git a/docs/backup-restore.md b/docs/backup-restore.md index 3858501..b2315d9 100644 --- a/docs/backup-restore.md +++ b/docs/backup-restore.md @@ -10,11 +10,11 @@ - Data Protection 密钥环是恢复数据库内加密运行时凭据的必要条件，必须与数据库来自同一个恢复点。 - Valkey 中的会话和短期状态不恢复；恢复后用户可能需要重新登录。 -每个归档包含时间戳、应用版本、最新 EF schema 版本、最低临时空间估算和 SHA-256 文件清单；旁边的同名 `.sha256` 文件校验整个压缩或加密归档。manifest 和旁路校验文件均不包含数据库口令、JWT、上游 API key 或连接字符串。未加密归档仍包含配置和密钥文件，因此必须按秘密处理。 +每个归档包含时间戳、应用版本、最新 EF schema 版本、数据库实际字节数、最低临时空间估算和 SHA-256 文件清单；旁边的同名 `.sha256` 文件校验整个压缩或加密归档。manifest 和旁路校验文件均不包含数据库口令、JWT、上游 API key 或连接字符串。未加密归档仍包含配置和密钥文件，因此必须按秘密处理。 ## 创建、列出和验证 -命令读取标准 `PGHOST`、`PGPORT`、`PGUSER`、`PGPASSWORD`、`PGDATABASE`；在容器内也可直接解析已有的 `ConnectionStrings__sdw` 环境变量。 +命令读取标准 `PGHOST`、`PGPORT`、`PGUSER`、`PGPASSWORD`、`PGDATABASE`；在容器内也可直接解析已有的 `ConnectionStrings__sdw` 环境变量。创建期间会获取与应用迁移相同的 `SDWMIGR1` PostgreSQL advisory lease，并在 lease 内依次读取 schema、完成一致性 dump、再次核对 schema，然后复制 Data Protection 密钥环。密钥在数据库快照之后复制，因此归档包含 dump 内所有加密值所需密钥的安全超集。 ```bash export PGHOST=localhost PGPORT=5432 PGUSER=sdw PGDATABASE=sdw @@ -41,7 +41,7 @@ sdw-backup create ... --age-recipient 'age1...' sdw-backup verify backup.tar.gz.age --age-identity /secure/backup-key.txt ``` -私钥不写入归档、manifest 或日志。若使用 webhook，只会发送固定的 `sdw_backup_failed` 事件，不发送错误文本、路径或凭据： +私钥不写入归档、manifest 或日志。若使用 webhook，只会发送固定的 `sdw_backup_failed` 事件，不发送错误文本、路径或凭据。显式参数错误等 `exit` 路径同样通知；配置了 webhook 但没有 `curl` 时命令会立即失败： ```bash export SDW_BACKUP_FAILURE_WEBHOOK=https://monitor.example/hooks/opaque-token @@ -51,7 +51,7 @@ export SDW_BACKUP_FAILURE_WEBHOOK=https://monitor.example/hooks/opaque-token ## systemd 定时执行 -系统包安装 `/etc/sdw-redive/backup.env`、`sdw-backup.service` 和 `sdw-backup.timer`，但不会在口令仍为占位符时自动启用。安装 PostgreSQL client，编辑并保护环境文件后启用： +系统包依赖 PostgreSQL client 与 `curl`，并安装 `/etc/sdw-redive/backup.env`、`sdw-backup.service` 和 `sdw-backup.timer`，但不会在口令仍为占位符时自动启用。编辑并保护环境文件后启用： ```bash sudoedit /etc/sdw-redive/backup.env @@ -62,7 +62,7 @@ sudo systemctl start sdw-backup.service sudo journalctl -u sdw-backup.service ``` -升级前可执行 `sudo systemctl start sdw-backup.service`，验证新归档后再升级。这样自动迁移数据库前已有明确恢复点。 +升级前可执行 `sudo systemctl start sdw-backup.service`，验证新归档后再升级。应用的 pre-migration backup hook 已经持有 `SDWMIGR1` lease 时，必须把 `create --migration-lock-held` 作为 hook 参数；该选项只供持锁父进程使用，普通定时任务不得设置，否则会绕过迁移串行化。 ## 容器部署 @@ -84,10 +84,11 @@ podman-compose exec sdw-redive sdw-backup verify /app/backups/sdw-backup-....tar 恢复是替换操作。先停止所有应用副本和后台任务；仅停止应用，不要停止 PostgreSQL。 -1. 把目标应用安装为与备份相同的 major 版本。恢复脚本会在生成 safety dump 后删除并重建目标数据库，数据库登录角色必须拥有目标数据库并具有 `CREATEDB`（或使用 PostgreSQL 超级用户）；不要把 `PGDATABASE` 指向 `postgres`、`template0` 或 `template1`。默认的低权限应用账号不应长期获得 `CREATEDB`，恢复时请临时提供独立的数据库管理员凭据。 -2. 预先挂载足够空间；脚本在任何数据库写入前验证路径、链接、所有 SHA-256、`pg_restore --list`、格式、major 版本、可选 schema 版本、临时空间和目标目录可写性。 -3. 执行恢复。命令先在 safety directory 创建现有数据库 dump，旧配置、密码和密钥环也会重命名为 `.pre-restore-*`，可人工回退。 -4. 修正文件所有者与权限，启动应用，检查 `/api/auth/allowRegister`、登录、订阅和文件浏览。 +1. 把目标应用安装为与备份相同的 major 版本。数据库恢复登录角色必须是超级用户，或同时具有 `CREATEDB` 且是目标数据库 owner 的成员；脚本通过 `pg_restore --role=` 保证恢复对象仍归应用 owner。不要把 `PGDATABASE` 指向 `postgres`、`template0` 或 `template1`，也不要长期提升低权限应用账号。 +2. 预先挂载足够空间。脚本先验证路径、链接、所有 SHA-256、`pg_restore --list`、格式、major、精确 schema、临时空间和目标目录。必须用 `--expected-schema`（或 `SDW_EXPECTED_SCHEMA_VERSION`）传入当前应用构建所要求的完整 migration id，不能只依赖同 major 或目标库碰巧已有的版本。对于远程 PostgreSQL，脚本无法从客户端可靠读取服务端文件系统剩余字节，必须用 `--postgres-available-bytes` 提供数据库主机或监控刚采集的 default tablespace 可用量；本地可见的 tablespace 会直接用 `df` 测量。 +3. 执行恢复。命令先创建 safety dump，再把归档以 `--single-transaction --exit-on-error --role=` 恢复到候选数据库；schema、对象 owner、索引和应用角色访问检查全部通过后，才用可回滚的数据库 rename 切换。任何候选恢复失败都删除候选库，目标库不变。成功后原数据库以 `sdw_previous_*` 名称保留，确认应用和备份后再由管理员删除。 +4. 配置、密码、密钥环和插件 manifest 在切换前同文件系统 staging；切换失败时自动恢复原状态。旧状态仍以 `.pre-restore-*` 保留。 +5. 修正文件所有者与权限，启动应用，检查 `/api/auth/allowRegister`、登录、加密运行时设置、订阅和文件浏览。 ```bash sudo systemctl stop sdw-redive @@ -98,6 +99,7 @@ sudo --preserve-env=PGHOST,PGPORT,PGUSER,PGPASSWORD,PGDATABASE,PGMAINTENANCEDATA --confirm-replace \ --expected-version 2.3.0 \ --expected-schema 20260801000000_ExpectedMigration \ + --postgres-available-bytes 21474836480 \ --config-destination /etc/sdw-redive/appsettings.yml \ --password-destination /var/lib/sdw-redive/password.json \ --key-ring-destination /var/lib/sdw-redive/data-protection-keys @@ -115,6 +117,8 @@ podman-compose stop sdw-redive podman-compose run --rm --no-deps --entrypoint sdw-backup sdw-redive \ restore /app/backups/sdw-backup-....tar.gz \ --confirm-replace --expected-version 2.3.0 \ + --expected-schema 20260801000000_ExpectedMigration \ + --postgres-available-bytes "$MEASURED_DATABASE_FREE_BYTES" \ --config-destination /app/backups/restored/podman-compose.yml \ --password-destination /app/data/password.json \ --key-ring-destination /app/data/data-protection-keys \ @@ -124,7 +128,7 @@ podman-compose up -d curl --fail http://127.0.0.1:5097/api/auth/allowRegister ``` -如果版本、schema 或空间检查失败，数据库不会被写入。只有在经过人工评估后才能使用 `--allow-version-mismatch`；该开关仍不会跳过格式、校验和、dump 与空间验证。 +如果版本、schema 或空间检查失败，候选库也不会创建。有效归档在 `pg_restore`、owner 或可用性检查中失败时，正式目标库仍保持原名和原内容。只有在经过人工评估后才能使用 `--allow-version-mismatch`；该开关不会跳过精确 schema、格式、校验和、dump 与空间验证。 ## 逻辑 JSON 导出与导入 @@ -141,6 +145,8 @@ curl --fail -H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json https://sdw.example/api/data-transfer/import ``` -导出 envelope 内的 SHA-256 在任何写入前验证。`skip` 可安全重复导入；`overwrite` 更新稳定键冲突项；`fail` 在首个冲突处返回 409，事务不会提交。订阅以 URL、规则以 TMDB id + pattern、人工修正以 release URL + title + publish time、播放进度以虚拟路径匹配。目标实例缺少对应 release 或虚拟文件时会明确计入 skipped，不会制造指向不存在媒体的记录。人工修正迁移当前元数据和审计操作，但不会覆盖物理路径；映射仍由目标实例的文件映射流程负责。 +导出在一个 repeatable-read 快照内读取所有类别；导出与导入共同限制为每类 10,000 条、完整导入请求 10 MiB，因此不会生成自身无法重新导入的文件。envelope 内的 SHA-256 在任何写入前验证。`skip` 可安全重复导入；`overwrite` 更新稳定键冲突项；`fail` 在首个冲突处返回 409，事务不会提交。生产 Npgsql 重试的每个 attempt 都使用全新 scope、DbContext 和 mapper 状态。 + +订阅以 URL、规则以 TMDB id + pattern、人工修正以 release URL + title + publish time、播放进度以虚拟路径匹配。目标实例缺少对应 release 或虚拟文件时会明确计入 skipped，不会制造指向不存在媒体的记录。人工修正不会覆盖目标中已有同 TMDB Animation 的全局名称、原名或海报，也不会改动共享该 Animation 的其他 release；物理路径由目标实例的映射预览与事务性替换流程处理。 逻辑导出不含 JWT、登录密码、WebDAV token、Data Protection key、AI/qBittorrent 凭据、聊天内容或媒体文件。跨 major 格式、不匹配校验和、未知类别、非法数值与超大类别会在事务开始前拒绝。 diff --git a/packaging/backup.env b/packaging/backup.env index 400ebdf..75a6e02 100644 --- a/packaging/backup.env +++ b/packaging/backup.env @@ -4,10 +4,13 @@ PGPORT=5432 PGUSER=sdw PGDATABASE=sdw PGPASSWORD=CHANGE_ME -# Replacement restore additionally requires a temporary database administrator -# that owns PGDATABASE and has CREATEDB. Do not grant that privilege permanently -# to this runtime/backup account; override PGUSER/PGPASSWORD for the manual restore. +# Replacement restore additionally requires a temporary database administrator: +# superuser, or CREATEDB plus membership in PGDATABASE's owner role. Do not grant +# that privilege permanently to the runtime account; override credentials manually. # PGMAINTENANCEDATABASE=postgres +# For remote PostgreSQL, set this only from a fresh database-host/tablespace +# measurement. Local visible tablespaces are measured automatically. +# SDW_POSTGRESQL_AVAILABLE_BYTES=21474836480 SDW_BACKUP_DIRECTORY=/var/lib/sdw-redive/backups SDW_BACKUP_RETENTION_DAYS=14 diff --git a/packaging/nfpm.yaml b/packaging/nfpm.yaml index d3ce403..88f6757 100644 --- a/packaging/nfpm.yaml +++ b/packaging/nfpm.yaml @@ -11,14 +11,23 @@ license: "Apache-2.0" depends: - aspnetcore-runtime-10.0 + - postgresql-client + - curl recommends: - valkey overrides: + rpm: + depends: + - aspnetcore-runtime-10.0 + - postgresql + - curl archlinux: depends: - aspnet-runtime-10.0 + - postgresql + - curl recommends: - valkey @@ -62,9 +71,6 @@ contents: owner: root group: sdw-redive - - src: ./VERSION - dst: /usr/lib/sdw-redive/VERSION - - src: ./LICENSE dst: /usr/share/doc/sdw-redive/LICENSE diff --git a/packaging/postinstall.sh b/packaging/postinstall.sh index 10cd22a..d580dbd 100755 --- a/packaging/postinstall.sh +++ b/packaging/postinstall.sh @@ -57,5 +57,8 @@ find /var/lib/sdw-redive/data-protection-keys -type f \ -exec chown sdw-redive:sdw-redive {} + \ -exec chmod 0600 {} + -# Reload systemd -systemctl daemon-reload +# Package-image roots do not necessarily run systemd. On a real host, malformed +# units and daemon failures must still fail package installation. +if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then + systemctl daemon-reload +fi diff --git a/packaging/preremove.sh b/packaging/preremove.sh index 9da2e22..dadcf43 100755 --- a/packaging/preremove.sh +++ b/packaging/preremove.sh @@ -1,11 +1,18 @@ #!/bin/bash set -e -# Stop and disable the service before removal -if systemctl is-active --quiet sdw-redive; then - systemctl stop sdw-redive -fi +# Debian passes "upgrade" and RPM passes 1 for replacement upgrades. Preserve +# the administrator's enabled/running state in those cases; this hook owns only +# final package removal. +case "${1:-remove}" in + upgrade|1) exit 0 ;; +esac -if systemctl is-enabled --quiet sdw-redive 2>/dev/null; then - systemctl disable sdw-redive +# Package-image roots do not necessarily run systemd. On a real host, stop the +# timer before its service and propagate every genuine systemctl failure. +if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then + # stop/disable are idempotent for inactive/disabled installed units. Calling + # them directly keeps D-Bus and unit errors visible to the package manager. + systemctl stop sdw-backup.timer sdw-backup.service sdw-redive.service + systemctl disable sdw-backup.timer sdw-redive.service fi From 81903d5444ba77a1b59e9a8e7b2457484f9c38d0 Mon Sep 17 00:00:00 2001 From: mahoshojoHCG Date: Mon, 31 Aug 2026 15:20:01 +0800 Subject: [PATCH 17/45] Fix staged release activation and search cursors --- .../DataRepository/ReleaseUpgrade.cs | 4 + ...4_StageInactiveReleaseMappings.Designer.cs | 1642 +++++++++++++++++ ...0831071234_StageInactiveReleaseMappings.cs | 102 + .../ApplicationContextModelSnapshot.cs | 43 +- .../Models/ApplicationContext.cs | 20 + .../Models/StagedFileMapping.cs | 14 + .../Repositories/AnimationInfoRepository.cs | 125 +- .../Repositories/FileMappingRepository.cs | 68 +- .../Repositories/LibrarySearchRepository.cs | 24 +- .../Repositories/MetadataReviewRepository.cs | 120 +- .../Repositories/ReleaseUpgradeRepository.cs | 66 +- .../ReleaseUpgradeCoordinator.cs | 2 +- 12 files changed, 2164 insertions(+), 66 deletions(-) create mode 100644 SecondDimensionWatcherReDive/Migrations/20260831071234_StageInactiveReleaseMappings.Designer.cs create mode 100644 SecondDimensionWatcherReDive/Migrations/20260831071234_StageInactiveReleaseMappings.cs create mode 100644 SecondDimensionWatcherReDive/Models/StagedFileMapping.cs diff --git a/SecondDimensionWatcherReDive.Framework/DataRepository/ReleaseUpgrade.cs b/SecondDimensionWatcherReDive.Framework/DataRepository/ReleaseUpgrade.cs index a01bc05..2b97331 100644 --- a/SecondDimensionWatcherReDive.Framework/DataRepository/ReleaseUpgrade.cs +++ b/SecondDimensionWatcherReDive.Framework/DataRepository/ReleaseUpgrade.cs @@ -79,6 +79,10 @@ Task> GetReadyCandidateIdsAsync( Guid candidateReleaseId, CancellationToken cancellationToken); + Task> GetCandidateMappingsAsync( + Guid candidateReleaseId, + CancellationToken cancellationToken); + Task GetRollbackAsync( Guid operationId, CancellationToken cancellationToken); diff --git a/SecondDimensionWatcherReDive/Migrations/20260831071234_StageInactiveReleaseMappings.Designer.cs b/SecondDimensionWatcherReDive/Migrations/20260831071234_StageInactiveReleaseMappings.Designer.cs new file mode 100644 index 0000000..7723a0c --- /dev/null +++ b/SecondDimensionWatcherReDive/Migrations/20260831071234_StageInactiveReleaseMappings.Designer.cs @@ -0,0 +1,1642 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using SecondDimensionWatcherReDive.Models; + +#nullable disable + +namespace SecondDimensionWatcherReDive.Migrations +{ + [DbContext(typeof(ApplicationContext))] + [Migration("20260831071234_StageInactiveReleaseMappings")] + partial class StageInactiveReleaseMappings + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.Animation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("OriginalName") + .IsRequired() + .HasColumnType("text"); + + b.Property("PosterPath") + .HasColumnType("text"); + + b.Property("TmdbId") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("TmdbId") + .IsUnique(); + + b.ToTable("Animations"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.AnimationCatalogEntry", b => + { + b.Property("AnimationId") + .HasColumnType("uuid"); + + b.Property("AutomationAttentionCount") + .HasColumnType("integer"); + + b.Property("EpisodeCount") + .HasColumnType("integer"); + + b.Property("LatestPublishTime") + .HasColumnType("timestamp with time zone"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("OriginalName") + .IsRequired() + .HasColumnType("text"); + + b.Property("PosterPath") + .HasColumnType("text"); + + b.Property("ReleaseCount") + .HasColumnType("integer"); + + b.Property("TmdbId") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("AnimationId"); + + b.HasIndex("TmdbId") + .IsUnique(); + + b.HasIndex("LatestPublishTime", "TmdbId") + .IsDescending(); + + b.ToTable("AnimationCatalogEntries", t => + { + t.HasCheckConstraint("CK_AnimationCatalogEntries_Counts", "\"EpisodeCount\" >= 0 AND \"ReleaseCount\" > 0 AND \"AutomationAttentionCount\" >= 0"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.AnimationCatalogState", b => + { + b.Property("Id") + .HasColumnType("integer"); + + b.Property("Revision") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.ToTable("AnimationCatalogStates", t => + { + t.HasCheckConstraint("CK_AnimationCatalogStates_Revision_Positive", "\"Revision\" > 0"); + + t.HasCheckConstraint("CK_AnimationCatalogStates_Singleton", "\"Id\" = 1"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.AnimationGroup", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("AnimationGroups"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.AnimationInfo", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AdditionalDownloadInfo") + .IsRequired() + .HasColumnType("text"); + + b.Property("AiRetryCount") + .HasColumnType("integer"); + + b.Property("AnimationId") + .HasColumnType("uuid"); + + b.Property("AutomationDisposition") + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("AutomationExplanationJson") + .HasColumnType("text"); + + b.Property("CachedDownloadData") + .IsRequired() + .HasColumnType("bytea"); + + b.Property("CurrentMetadataReviewOperationId") + .HasColumnType("uuid"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text"); + + b.Property("DownloadAttemptId") + .HasColumnType("uuid"); + + b.Property("DownloadCancellationId") + .HasColumnType("uuid"); + + b.Property("DownloadEndTime") + .HasColumnType("timestamp with time zone"); + + b.Property("DownloadStartTime") + .HasColumnType("timestamp with time zone"); + + b.Property("DownloadType") + .IsRequired() + .HasColumnType("text"); + + b.Property("DownloadUrl") + .IsRequired() + .HasColumnType("text"); + + b.Property("EnclosureId") + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("Episode") + .HasColumnType("integer"); + + b.Property("ExpectedEpisodeCount") + .HasColumnType("integer"); + + b.Property("FeedItemGuid") + .HasMaxLength(1024) + .HasColumnType("character varying(1024)"); + + b.Property("FileStore") + .HasColumnType("text"); + + b.Property("GroupId") + .HasColumnType("uuid"); + + b.Property("IngestedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("IsActiveRelease") + .HasColumnType("boolean"); + + b.Property("IsAiProcessed") + .HasColumnType("boolean"); + + b.Property("IsDownloadFinished") + .HasColumnType("boolean"); + + b.Property("IsDownloadTracked") + .HasColumnType("boolean"); + + b.Property("MediaLibraryMissingSince") + .HasColumnType("timestamp with time zone"); + + b.Property("MediaLibrarySourceId") + .HasColumnType("uuid"); + + b.Property("MetadataConfidence") + .HasColumnType("double precision"); + + b.Property("MetadataLastError") + .HasMaxLength(1024) + .HasColumnType("character varying(1024)"); + + b.Property("MetadataReviewedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("MetadataStatus") + .HasColumnType("integer"); + + b.Property("PublishTime") + .HasColumnType("timestamp with time zone"); + + b.Property("ReleaseCodec") + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("ReleaseIdentity") + .HasMaxLength(192) + .HasColumnType("character varying(192)"); + + b.PrimitiveCollection("ReleaseLanguages") + .IsRequired() + .HasColumnType("text[]"); + + b.Property("ReleaseResolution") + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("ReleaseScore") + .HasColumnType("integer"); + + b.Property("ReleaseScoreReasonsJson") + .HasColumnType("text"); + + b.Property("ReleaseSizeBytes") + .HasColumnType("bigint"); + + b.Property("ReleaseSubtitleGroup") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("Season") + .HasColumnType("integer"); + + b.Property("SourceFeedId") + .HasColumnType("uuid"); + + b.Property("StateVersion") + .IsConcurrencyToken() + .HasColumnType("bigint"); + + b.Property("StorePath") + .HasColumnType("text"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text"); + + b.Property("TorrentInfoHash") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.HasKey("Id"); + + b.HasIndex("CurrentMetadataReviewOperationId") + .IsUnique(); + + b.HasIndex("GroupId"); + + b.HasIndex("MediaLibrarySourceId"); + + b.HasIndex("ReleaseCodec"); + + b.HasIndex("ReleaseIdentity") + .IsUnique() + .HasDatabaseName("UX_AnimationInfo_ReleaseIdentity") + .HasFilter("\"ReleaseIdentity\" IS NOT NULL"); + + b.HasIndex("ReleaseResolution"); + + b.HasIndex("ReleaseSubtitleGroup"); + + b.HasIndex("SourceFeedId"); + + b.HasIndex("AutomationDisposition", "PublishTime"); + + b.HasIndex("FileStore", "StorePath") + .IsUnique() + .HasFilter("\"DownloadType\" = 'http://schemas.hcgstudio.com/ws/2023/06/sdw/downloadtype/media-library-import'"); + + b.HasIndex("MetadataStatus", "PublishTime"); + + b.HasIndex("AnimationId", "PublishTime", "Id"); + + b.HasIndex("AnimationId", "Season", "Episode") + .IsUnique() + .HasDatabaseName("UX_AnimationInfo_ActiveEpisodeRelease") + .HasFilter("\"IsActiveRelease\" = TRUE AND \"AnimationId\" IS NOT NULL AND \"Season\" IS NOT NULL AND \"Episode\" IS NOT NULL"); + + b.HasIndex("DownloadType", "IsDownloadFinished", "IsDownloadTracked"); + + b.HasIndex("MediaLibraryMissingSince", "PublishTime", "Id"); + + b.HasIndex("Season", "Episode", "ReleaseScore"); + + b.ToTable("AnimationInfo", t => + { + t.HasCheckConstraint("CK_AnimationInfo_ExpectedEpisodeCount_Positive", "\"ExpectedEpisodeCount\" IS NULL OR \"ExpectedEpisodeCount\" > 0"); + + t.HasCheckConstraint("CK_AnimationInfo_MetadataConfidence_Range", "\"MetadataConfidence\" IS NULL OR (\"MetadataConfidence\" >= 0 AND \"MetadataConfidence\" <= 1)"); + + t.HasCheckConstraint("CK_AnimationInfo_ReleaseScore_NonNegative", "\"ReleaseScore\" >= 0"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ApplicationSettings", b => + { + b.Property("Id") + .HasColumnType("integer"); + + b.Property("ProtectedSecrets") + .HasColumnType("text"); + + b.Property("Revision") + .IsConcurrencyToken() + .HasColumnType("bigint"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("ValuesJson") + .IsRequired() + .HasColumnType("jsonb"); + + b.HasKey("Id"); + + b.ToTable("ApplicationSettings", t => + { + t.HasCheckConstraint("CK_ApplicationSettings_Revision_Positive", "\"Revision\" > 0"); + + t.HasCheckConstraint("CK_ApplicationSettings_Singleton", "\"Id\" = 1"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.AuthenticationState", b => + { + b.Property("Id") + .HasColumnType("integer"); + + b.Property("ClaimId") + .HasColumnType("uuid"); + + b.Property("PasswordHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("RegisteredAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.ToTable("AuthenticationStates", t => + { + t.HasCheckConstraint("CK_AuthenticationStates_Singleton", "\"Id\" = 1"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.BangumiSubgroup", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("MikanSubgroupId") + .HasColumnType("integer"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("ScrapedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("SeasonBangumiId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("SeasonBangumiId", "MikanSubgroupId") + .IsUnique(); + + b.ToTable("BangumiSubgroups"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ChatConversation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Title") + .HasColumnType("text"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.ToTable("ChatConversations"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ChatMessage", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Content") + .HasColumnType("text"); + + b.Property("ConversationId") + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Order") + .HasColumnType("integer"); + + b.Property("Role") + .IsRequired() + .HasColumnType("text"); + + b.Property("ToolCallId") + .HasColumnType("text"); + + b.Property("ToolCallsJson") + .HasColumnType("text"); + + b.Property("ToolName") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("ConversationId"); + + b.ToTable("ChatMessages"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.Feed", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Name") + .HasColumnType("text"); + + b.Property("Url") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("Feeds"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.FileMapping", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AnimationInfoId") + .HasColumnType("uuid"); + + b.Property("FileStore") + .IsRequired() + .HasColumnType("text"); + + b.Property("PhysicalPath") + .IsRequired() + .HasColumnType("text"); + + b.Property("VirtualPath") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("AnimationInfoId"); + + b.HasIndex("VirtualPath") + .IsUnique(); + + b.ToTable("FileMappings", t => + { + t.HasCheckConstraint("CK_FileMappings_VirtualPath_Canonical", "\"VirtualPath\" ~ '^/[^/]+(?:/[^/]+)*$' AND \"VirtualPath\" !~ '(^|/)\\.\\.?($|/)'"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.FileNameRegexRule", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AnimationId") + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("Pattern") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("character varying(512)"); + + b.HasKey("Id"); + + b.HasIndex("AnimationId", "CreatedAt"); + + b.HasIndex("AnimationId", "Pattern") + .IsUnique(); + + b.ToTable("FileNameRegexRules"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.FileSystemDirectoryState", b => + { + b.Property("Path") + .HasColumnType("text"); + + b.Property("Generation") + .HasColumnType("bigint"); + + b.HasKey("Path"); + + b.ToTable("FileSystemDirectoryStates", t => + { + t.HasCheckConstraint("CK_FileSystemDirectoryStates_Generation_Positive", "\"Generation\" > 0"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.FileSystemEntry", b => + { + b.Property("Path") + .HasColumnType("text"); + + b.Property("Cookie") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasDefaultValueSql("nextval('sdw_file_system_entry_cookie_seq')"); + + b.Property("DescendantFileCount") + .HasColumnType("integer"); + + b.Property("EntryId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasDefaultValueSql("gen_random_uuid()"); + + b.Property("FileMappingId") + .HasColumnType("uuid"); + + b.Property("IsDirectory") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("ParentPath") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Path"); + + b.HasIndex("Cookie") + .IsUnique(); + + b.HasIndex("EntryId") + .IsUnique(); + + b.HasIndex("FileMappingId") + .IsUnique() + .HasFilter("\"FileMappingId\" IS NOT NULL"); + + b.HasIndex("ParentPath", "Cookie"); + + b.HasIndex("ParentPath", "IsDirectory", "Name") + .IsDescending(false, true, false); + + b.ToTable("FileSystemEntries", t => + { + t.HasCheckConstraint("CK_FileSystemEntries_NodeShape", "(\"IsDirectory\" AND \"FileMappingId\" IS NULL AND \"DescendantFileCount\" > 0) OR (NOT \"IsDirectory\" AND \"FileMappingId\" IS NOT NULL AND \"DescendantFileCount\" = 1)"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.Incident", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Detail") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("DetectedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Fingerprint") + .IsRequired() + .HasMaxLength(96) + .HasColumnType("character varying(96)"); + + b.Property("LastRetryAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastRetryError") + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("Occurrence") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(1); + + b.Property("ResolvedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("RetryCount") + .HasColumnType("integer"); + + b.Property("Severity") + .HasColumnType("integer"); + + b.Property("SourceId") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("Title") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("Type") + .HasColumnType("integer"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("Fingerprint") + .IsUnique(); + + b.HasIndex("ResolvedAt", "Type", "UpdatedAt"); + + b.ToTable("Incidents", t => + { + t.HasCheckConstraint("CK_Incidents_Occurrence_Positive", "\"Occurrence\" > 0"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MediaLibrarySource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("IsMonitoring") + .HasColumnType("boolean"); + + b.Property("LastError") + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("LastImportedCount") + .HasColumnType("integer"); + + b.Property("LastRemovedCount") + .HasColumnType("integer"); + + b.Property("LastScanAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastSkippedCount") + .HasColumnType("integer"); + + b.Property("LastUpdatedCount") + .HasColumnType("integer"); + + b.Property("Path") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Path") + .IsUnique(); + + b.ToTable("MediaLibrarySources"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewMappingSnapshot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("FileStore") + .IsRequired() + .HasColumnType("text"); + + b.Property("Kind") + .HasColumnType("integer"); + + b.Property("OperationId") + .HasColumnType("uuid"); + + b.Property("PhysicalPath") + .IsRequired() + .HasColumnType("text"); + + b.Property("VirtualPath") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("OperationId", "Kind", "VirtualPath") + .IsUnique(); + + b.ToTable("MetadataReviewMappingSnapshots", t => + { + t.HasCheckConstraint("CK_MetadataReviewMappingSnapshots_VirtualPath_Canonical", "\"VirtualPath\" ~ '^/[^/]+(?:/[^/]+)*$' AND \"VirtualPath\" !~ '(^|/)\\.\\.?($|/)'"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AnimationInfoId") + .HasColumnType("uuid"); + + b.Property("AppliedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("AppliedVersion") + .HasColumnType("bigint"); + + b.Property("BaseFileStore") + .HasColumnType("text"); + + b.Property("BaseIsDownloadFinished") + .HasColumnType("boolean"); + + b.Property("BaseStorePath") + .HasColumnType("text"); + + b.Property("BaseVersion") + .HasColumnType("bigint"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("ExpiresAt") + .HasColumnType("timestamp with time zone"); + + b.Property("PreviousAiRetryCount") + .HasColumnType("integer"); + + b.Property("PreviousAnimationId") + .HasColumnType("uuid"); + + b.Property("PreviousConfidence") + .HasColumnType("double precision"); + + b.Property("PreviousCurrentOperationId") + .HasColumnType("uuid"); + + b.Property("PreviousDescription") + .HasColumnType("text"); + + b.Property("PreviousEpisode") + .HasColumnType("integer"); + + b.Property("PreviousGroupId") + .HasColumnType("uuid"); + + b.Property("PreviousIsAiProcessed") + .HasColumnType("boolean"); + + b.Property("PreviousLastError") + .HasColumnType("text"); + + b.Property("PreviousMetadataStatus") + .HasColumnType("integer"); + + b.Property("PreviousReviewedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("PreviousSeason") + .HasColumnType("integer"); + + b.Property("ProposedAnimationName") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProposedAnimationOriginalName") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProposedAnimationPosterPath") + .HasColumnType("text"); + + b.Property("ProposedAnimationTmdbId") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProposedDescription") + .IsRequired() + .HasColumnType("text"); + + b.Property("ProposedEpisode") + .HasColumnType("integer"); + + b.Property("ProposedGroupName") + .HasColumnType("text"); + + b.Property("ProposedSeason") + .HasColumnType("integer"); + + b.Property("State") + .HasColumnType("integer"); + + b.Property("UndoneAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("AnimationInfoId", "AppliedVersion") + .IsUnique(); + + b.HasIndex("AnimationInfoId", "State"); + + b.HasIndex("State", "ExpiresAt"); + + b.ToTable("MetadataReviewOperations", t => + { + t.HasCheckConstraint("CK_MetadataReviewOperations_Expiry", "\"ExpiresAt\" > \"CreatedAt\""); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MigrationExecutionState", b => + { + b.Property("Key") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("Version") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(1); + + b.Property("AttemptCount") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(1); + + b.Property("Checkpoint") + .HasMaxLength(4096) + .HasColumnType("character varying(4096)"); + + b.Property("FinishedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastErrorSummary") + .HasMaxLength(4096) + .HasColumnType("character varying(4096)"); + + b.Property("StartedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Status") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(3); + + b.Property("UpdatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.HasKey("Key", "Version"); + + b.ToTable("MigrationMarkers", null, t => + { + t.HasCheckConstraint("CK_MigrationMarkers_AttemptCount_NonNegative", "\"AttemptCount\" >= 0"); + + t.HasCheckConstraint("CK_MigrationMarkers_Status_Range", "\"Status\" BETWEEN 0 AND 3"); + + t.HasCheckConstraint("CK_MigrationMarkers_Version_Positive", "\"Version\" > 0"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.NotificationOutboxMessage", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AttemptCount") + .HasColumnType("integer"); + + b.Property("Body") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("Channel") + .IsRequired() + .HasMaxLength(24) + .HasColumnType("character varying(24)"); + + b.Property("DeduplicationKey") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("DeepLink") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("DeliveredAt") + .HasColumnType("timestamp with time zone"); + + b.Property("EventId") + .HasColumnType("uuid"); + + b.Property("LastAttemptAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastError") + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("NextAttemptAt") + .HasColumnType("timestamp with time zone"); + + b.Property("OccurredAt") + .HasColumnType("timestamp with time zone"); + + b.Property("PayloadJson") + .HasColumnType("jsonb"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(24) + .HasColumnType("character varying(24)"); + + b.Property("Title") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(48) + .HasColumnType("character varying(48)"); + + b.Property("WebPushSubscriptionId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("DeduplicationKey") + .IsUnique(); + + b.HasIndex("WebPushSubscriptionId"); + + b.HasIndex("Status", "NextAttemptAt"); + + b.ToTable("NotificationOutboxMessages"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.PlaybackPreference", b => + { + b.Property("UserId") + .HasColumnType("uuid"); + + b.Property("AudioLanguage") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("AudioTrackLabel") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("AutoPlayNext") + .HasColumnType("boolean"); + + b.Property("SubtitleLanguage") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("SubtitleTrackLabel") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("UserId"); + + b.ToTable("PlaybackPreferences"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.PlaybackProgress", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AnimationInfoId") + .HasColumnType("uuid"); + + b.Property("DurationSeconds") + .HasColumnType("double precision"); + + b.Property("IsWatched") + .HasColumnType("boolean"); + + b.Property("PositionSeconds") + .HasColumnType("double precision"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.Property("VirtualPath") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("WatchedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("AnimationInfoId"); + + b.HasIndex("UserId", "AnimationInfoId", "VirtualPath") + .IsUnique(); + + b.HasIndex("UserId", "IsWatched", "UpdatedAt"); + + b.ToTable("PlaybackProgresses", t => + { + t.HasCheckConstraint("CK_PlaybackProgresses_Duration_NonNegative", "\"DurationSeconds\" >= 0"); + + t.HasCheckConstraint("CK_PlaybackProgresses_Position_NonNegative", "\"PositionSeconds\" >= 0"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ReleaseUpgradeMappingSnapshot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AnimationInfoId") + .HasColumnType("uuid"); + + b.Property("FileStore") + .IsRequired() + .HasColumnType("text"); + + b.Property("Kind") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("OperationId") + .HasColumnType("uuid"); + + b.Property("OriginalMappingId") + .HasColumnType("uuid"); + + b.Property("PhysicalPath") + .IsRequired() + .HasColumnType("text"); + + b.Property("VirtualPath") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.HasKey("Id"); + + b.HasIndex("OperationId", "Kind", "OriginalMappingId") + .IsUnique(); + + b.ToTable("ReleaseUpgradeMappingSnapshots", t => + { + t.HasCheckConstraint("CK_ReleaseUpgradeMappingSnapshots_VirtualPath_Canonical", "\"VirtualPath\" ~ '^/[^/]+(?:/[^/]+)*$' AND \"VirtualPath\" !~ '(^|/)\\.\\.?($|/)'"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ReleaseUpgradeOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AppliedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CandidateReleaseId") + .HasColumnType("uuid"); + + b.Property("CandidateScore") + .HasColumnType("integer"); + + b.Property("CompletedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CurrentReleaseId") + .HasColumnType("uuid"); + + b.Property("CurrentScore") + .HasColumnType("integer"); + + b.Property("FailureSummary") + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("RollbackUntil") + .HasColumnType("timestamp with time zone"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("VerifiedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("CandidateReleaseId") + .IsUnique() + .HasFilter("\"Status\" <> 'Failed'"); + + b.HasIndex("CurrentReleaseId") + .IsUnique() + .HasDatabaseName("UX_ReleaseUpgradeOperations_ActiveCurrentRelease") + .HasFilter("\"Status\" IN ('Downloading', 'Verifying', 'Applied')"); + + b.HasIndex("Status", "CreatedAt"); + + b.ToTable("ReleaseUpgradeOperations", t => + { + t.HasCheckConstraint("CK_ReleaseUpgradeOperations_ScoreIncrease", "\"CandidateScore\" > \"CurrentScore\""); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SeasonBangumi", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("DayOfWeek") + .HasColumnType("integer"); + + b.Property("ImageUrl") + .HasColumnType("text"); + + b.Property("MikanId") + .HasColumnType("integer"); + + b.Property("ScrapedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("MikanId") + .IsUnique(); + + b.ToTable("SeasonBangumis"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.StagedFileMapping", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AnimationInfoId") + .HasColumnType("uuid"); + + b.Property("FileStore") + .IsRequired() + .HasColumnType("text"); + + b.Property("PhysicalPath") + .IsRequired() + .HasColumnType("text"); + + b.Property("VirtualPath") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.HasKey("Id"); + + b.HasIndex("AnimationInfoId", "VirtualPath") + .IsUnique(); + + b.ToTable("StagedFileMappings", t => + { + t.HasCheckConstraint("CK_StagedFileMappings_VirtualPath_Canonical", "\"VirtualPath\" ~ '^/[^/]+(?:/[^/]+)*$' AND \"VirtualPath\" !~ '(^|/)\\.\\.?($|/)'"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SubscriptionAutomationPolicy", b => + { + b.Property("FeedId") + .HasColumnType("uuid"); + + b.PrimitiveCollection("Codecs") + .IsRequired() + .HasColumnType("text[]"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("EnableVersionUpgrade") + .HasColumnType("boolean"); + + b.PrimitiveCollection("ExcludedKeywords") + .IsRequired() + .HasColumnType("text[]"); + + b.PrimitiveCollection("Languages") + .IsRequired() + .HasColumnType("text[]"); + + b.Property("MaxSizeBytes") + .HasColumnType("bigint"); + + b.Property("MinSizeBytes") + .HasColumnType("bigint"); + + b.Property("MinimumUpgradeScore") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(25); + + b.Property("Mode") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.PrimitiveCollection("Resolutions") + .IsRequired() + .HasColumnType("text[]"); + + b.PrimitiveCollection("SubtitleGroups") + .IsRequired() + .HasColumnType("text[]"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("UpgradeRollbackHours") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(72); + + b.HasKey("FeedId"); + + b.HasIndex("UpdatedAt"); + + b.ToTable("SubscriptionAutomationPolicies", t => + { + t.HasCheckConstraint("CK_SubscriptionAutomationPolicies_MinimumUpgradeScore", "\"MinimumUpgradeScore\" >= 1 AND \"MinimumUpgradeScore\" <= 1000"); + + t.HasCheckConstraint("CK_SubscriptionAutomationPolicies_UpgradeRollbackHours", "\"UpgradeRollbackHours\" >= 1 AND \"UpgradeRollbackHours\" <= 720"); + }); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.TodoItemState", b => + { + b.Property("Key") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("ReadAt") + .HasColumnType("timestamp with time zone"); + + b.Property("SnoozedUntil") + .HasColumnType("timestamp with time zone"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Key"); + + b.ToTable("TodoItemStates"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.WebDavToken", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("TokenHash") + .IsRequired() + .HasColumnType("text"); + + b.Property("Username") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Username") + .IsUnique(); + + b.ToTable("WebDavTokens"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.WebPushSubscription", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("EndpointHash") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("LastError") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("LastFailureAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastSuccessAt") + .HasColumnType("timestamp with time zone"); + + b.Property("ProtectedAuth") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("character varying(1024)"); + + b.Property("ProtectedEndpoint") + .IsRequired() + .HasMaxLength(4096) + .HasColumnType("character varying(4096)"); + + b.Property("ProtectedP256Dh") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("character varying(1024)"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("EndpointHash") + .IsUnique(); + + b.ToTable("WebPushSubscriptions"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.AnimationCatalogEntry", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.Animation", "Animation") + .WithOne() + .HasForeignKey("SecondDimensionWatcherReDive.Models.AnimationCatalogEntry", "AnimationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Animation"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.AnimationInfo", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.Animation", "Animation") + .WithMany() + .HasForeignKey("AnimationId"); + + b.HasOne("SecondDimensionWatcherReDive.Models.AnimationGroup", "Group") + .WithMany() + .HasForeignKey("GroupId"); + + b.HasOne("SecondDimensionWatcherReDive.Models.MediaLibrarySource", null) + .WithMany() + .HasForeignKey("MediaLibrarySourceId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("SecondDimensionWatcherReDive.Models.Feed", null) + .WithMany() + .HasForeignKey("SourceFeedId") + .OnDelete(DeleteBehavior.SetNull); + + b.Navigation("Animation"); + + b.Navigation("Group"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.BangumiSubgroup", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.SeasonBangumi", "SeasonBangumi") + .WithMany("Subgroups") + .HasForeignKey("SeasonBangumiId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("SeasonBangumi"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ChatMessage", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.ChatConversation", "Conversation") + .WithMany("Messages") + .HasForeignKey("ConversationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Conversation"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.FileNameRegexRule", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.Animation", null) + .WithMany() + .HasForeignKey("AnimationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.FileSystemEntry", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.FileMapping", "FileMapping") + .WithOne() + .HasForeignKey("SecondDimensionWatcherReDive.Models.FileSystemEntry", "FileMappingId") + .OnDelete(DeleteBehavior.Cascade); + + b.Navigation("FileMapping"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewMappingSnapshot", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.MetadataReviewOperation", "Operation") + .WithMany("MappingSnapshots") + .HasForeignKey("OperationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Operation"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewOperation", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.AnimationInfo", "AnimationInfo") + .WithMany() + .HasForeignKey("AnimationInfoId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AnimationInfo"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.PlaybackProgress", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.AnimationInfo", "AnimationInfo") + .WithMany() + .HasForeignKey("AnimationInfoId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AnimationInfo"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ReleaseUpgradeMappingSnapshot", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.ReleaseUpgradeOperation", "Operation") + .WithMany("MappingSnapshots") + .HasForeignKey("OperationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Operation"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ReleaseUpgradeOperation", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.AnimationInfo", "CandidateRelease") + .WithMany() + .HasForeignKey("CandidateReleaseId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("SecondDimensionWatcherReDive.Models.AnimationInfo", "CurrentRelease") + .WithMany() + .HasForeignKey("CurrentReleaseId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("CandidateRelease"); + + b.Navigation("CurrentRelease"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.StagedFileMapping", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.AnimationInfo", null) + .WithMany() + .HasForeignKey("AnimationInfoId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SubscriptionAutomationPolicy", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.Feed", "Feed") + .WithOne() + .HasForeignKey("SecondDimensionWatcherReDive.Models.SubscriptionAutomationPolicy", "FeedId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Feed"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ChatConversation", b => + { + b.Navigation("Messages"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.MetadataReviewOperation", b => + { + b.Navigation("MappingSnapshots"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.ReleaseUpgradeOperation", b => + { + b.Navigation("MappingSnapshots"); + }); + + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SeasonBangumi", b => + { + b.Navigation("Subgroups"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/SecondDimensionWatcherReDive/Migrations/20260831071234_StageInactiveReleaseMappings.cs b/SecondDimensionWatcherReDive/Migrations/20260831071234_StageInactiveReleaseMappings.cs new file mode 100644 index 0000000..5eba90c --- /dev/null +++ b/SecondDimensionWatcherReDive/Migrations/20260831071234_StageInactiveReleaseMappings.cs @@ -0,0 +1,102 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace SecondDimensionWatcherReDive.Migrations +{ + /// + public partial class StageInactiveReleaseMappings : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "StagedFileMappings", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + AnimationInfoId = table.Column(type: "uuid", nullable: false), + VirtualPath = table.Column(type: "character varying(2048)", maxLength: 2048, nullable: false), + PhysicalPath = table.Column(type: "text", nullable: false), + FileStore = table.Column(type: "text", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_StagedFileMappings", x => x.Id); + table.CheckConstraint("CK_StagedFileMappings_VirtualPath_Canonical", "\"VirtualPath\" ~ '^/[^/]+(?:/[^/]+)*$' AND \"VirtualPath\" !~ '(^|/)\\.\\.?($|/)'"); + table.ForeignKey( + name: "FK_StagedFileMappings_AnimationInfo_AnimationInfoId", + column: x => x.AnimationInfoId, + principalTable: "AnimationInfo", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "IX_StagedFileMappings_AnimationInfoId_VirtualPath", + table: "StagedFileMappings", + columns: new[] { "AnimationInfoId", "VirtualPath" }, + unique: true); + + // Alternative releases used to share the live FileMappings namespace. + // Move only known-episode alternatives that have a different active + // release; unknown/unmatched downloads remain ordinary live content. + migrationBuilder.Sql( + """ + INSERT INTO "StagedFileMappings" + ("Id", "AnimationInfoId", "VirtualPath", "PhysicalPath", "FileStore") + SELECT mapping."Id", mapping."AnimationInfoId", mapping."VirtualPath", + mapping."PhysicalPath", mapping."FileStore" + FROM "FileMappings" AS mapping + JOIN "AnimationInfo" AS candidate + ON candidate."Id" = mapping."AnimationInfoId" + WHERE NOT candidate."IsActiveRelease" + AND candidate."AnimationId" IS NOT NULL + AND candidate."Season" IS NOT NULL + AND candidate."Episode" IS NOT NULL + AND EXISTS ( + SELECT 1 + FROM "AnimationInfo" AS active + WHERE active."Id" <> candidate."Id" + AND active."IsActiveRelease" + AND active."AnimationId" = candidate."AnimationId" + AND active."Season" = candidate."Season" + AND active."Episode" = candidate."Episode"); + + DELETE FROM "FileMappings" AS mapping + USING "AnimationInfo" AS candidate + WHERE candidate."Id" = mapping."AnimationInfoId" + AND NOT candidate."IsActiveRelease" + AND candidate."AnimationId" IS NOT NULL + AND candidate."Season" IS NOT NULL + AND candidate."Episode" IS NOT NULL + AND EXISTS ( + SELECT 1 + FROM "AnimationInfo" AS active + WHERE active."Id" <> candidate."Id" + AND active."IsActiveRelease" + AND active."AnimationId" = candidate."AnimationId" + AND active."Season" = candidate."Season" + AND active."Episode" = candidate."Episode"); + """); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + // Fail the downgrade transaction if the live namespace changed in a + // way that prevents restoring every staged mapping; never discard it. + migrationBuilder.Sql( + """ + INSERT INTO "FileMappings" + ("Id", "AnimationInfoId", "VirtualPath", "PhysicalPath", "FileStore") + SELECT "Id", "AnimationInfoId", "VirtualPath", "PhysicalPath", "FileStore" + FROM "StagedFileMappings"; + """); + + migrationBuilder.DropTable( + name: "StagedFileMappings"); + } + } +} diff --git a/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs b/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs index 9a7cc2a..81f2aff 100644 --- a/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs +++ b/SecondDimensionWatcherReDive/Migrations/ApplicationContextModelSnapshot.cs @@ -1044,7 +1044,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) modelBuilder.Entity("SecondDimensionWatcherReDive.Models.PlaybackPreference", b => { b.Property("UserId") - .ValueGeneratedNever() .HasColumnType("uuid"); b.Property("AudioLanguage") @@ -1258,6 +1257,39 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("SeasonBangumis"); }); + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.StagedFileMapping", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AnimationInfoId") + .HasColumnType("uuid"); + + b.Property("FileStore") + .IsRequired() + .HasColumnType("text"); + + b.Property("PhysicalPath") + .IsRequired() + .HasColumnType("text"); + + b.Property("VirtualPath") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.HasKey("Id"); + + b.HasIndex("AnimationInfoId", "VirtualPath") + .IsUnique(); + + b.ToTable("StagedFileMappings", t => + { + t.HasCheckConstraint("CK_StagedFileMappings_VirtualPath_Canonical", "\"VirtualPath\" ~ '^/[^/]+(?:/[^/]+)*$' AND \"VirtualPath\" !~ '(^|/)\\.\\.?($|/)'"); + }); + }); + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SubscriptionAutomationPolicy", b => { b.Property("FeedId") @@ -1562,6 +1594,15 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Navigation("CurrentRelease"); }); + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.StagedFileMapping", b => + { + b.HasOne("SecondDimensionWatcherReDive.Models.AnimationInfo", null) + .WithMany() + .HasForeignKey("AnimationInfoId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + modelBuilder.Entity("SecondDimensionWatcherReDive.Models.SubscriptionAutomationPolicy", b => { b.HasOne("SecondDimensionWatcherReDive.Models.Feed", "Feed") diff --git a/SecondDimensionWatcherReDive/Models/ApplicationContext.cs b/SecondDimensionWatcherReDive/Models/ApplicationContext.cs index d36ffcf..fd6d01c 100644 --- a/SecondDimensionWatcherReDive/Models/ApplicationContext.cs +++ b/SecondDimensionWatcherReDive/Models/ApplicationContext.cs @@ -25,6 +25,7 @@ public ApplicationContext(DbContextOptions options) public DbSet ChatConversations { get; set; } public DbSet ChatMessages { get; set; } public DbSet FileMappings { get; set; } + public DbSet StagedFileMappings { get; set; } public DbSet FileSystemEntries { get; set; } public DbSet FileSystemDirectoryStates { get; set; } public DbSet FileNameRegexRules { get; set; } @@ -504,6 +505,25 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) "CK_FileMappings_VirtualPath_Canonical", "\"VirtualPath\" ~ '^/[^/]+(?:/[^/]+)*$' AND \"VirtualPath\" !~ '(^|/)\\.\\.?($|/)'")); + modelBuilder.Entity() + .HasIndex(mapping => new { mapping.AnimationInfoId, mapping.VirtualPath }) + .IsUnique(); + + modelBuilder.Entity() + .Property(mapping => mapping.VirtualPath) + .HasMaxLength(2048); + + modelBuilder.Entity() + .HasOne() + .WithMany() + .HasForeignKey(mapping => mapping.AnimationInfoId) + .OnDelete(DeleteBehavior.Cascade); + + modelBuilder.Entity() + .ToTable(table => table.HasCheckConstraint( + "CK_StagedFileMappings_VirtualPath_Canonical", + "\"VirtualPath\" ~ '^/[^/]+(?:/[^/]+)*$' AND \"VirtualPath\" !~ '(^|/)\\.\\.?($|/)'")); + modelBuilder.Entity() .ToTable(table => table.HasCheckConstraint( "CK_MetadataReviewMappingSnapshots_VirtualPath_Canonical", diff --git a/SecondDimensionWatcherReDive/Models/StagedFileMapping.cs b/SecondDimensionWatcherReDive/Models/StagedFileMapping.cs new file mode 100644 index 0000000..57fe3e7 --- /dev/null +++ b/SecondDimensionWatcherReDive/Models/StagedFileMapping.cs @@ -0,0 +1,14 @@ +namespace SecondDimensionWatcherReDive.Models; + +/// +/// A completed alternative release's files before the release is validated and +/// atomically promoted into the live virtual-file-system namespace. +/// +public sealed class StagedFileMapping +{ + public Guid Id { get; set; } + public Guid AnimationInfoId { get; set; } + public string VirtualPath { get; set; } = string.Empty; + public string PhysicalPath { get; set; } = string.Empty; + public string FileStore { get; set; } = string.Empty; +} diff --git a/SecondDimensionWatcherReDive/Repositories/AnimationInfoRepository.cs b/SecondDimensionWatcherReDive/Repositories/AnimationInfoRepository.cs index 07ef042..a1fbc17 100644 --- a/SecondDimensionWatcherReDive/Repositories/AnimationInfoRepository.cs +++ b/SecondDimensionWatcherReDive/Repositories/AnimationInfoRepository.cs @@ -516,7 +516,8 @@ public async Task> GetDownloadedWithoutFileMappings && info.MediaLibraryMissingSince == null && info.FileStore != null && info.StorePath != null - && !context.FileMappings.Any(mapping => mapping.AnimationInfoId == info.Id)) + && !context.FileMappings.Any(mapping => mapping.AnimationInfoId == info.Id) + && !context.StagedFileMappings.Any(mapping => mapping.AnimationInfoId == info.Id)) .OrderBy(info => info.DownloadEndTime) .ToListAsync(cancellationToken); return entities.Select(entity => entity.ToRecord()).ToList(); @@ -610,6 +611,10 @@ await SetEpisodeReleaseActivityAsync( entity, willHaveMappings: false, cancellationToken); + await ReconcileMappingVisibilityAfterMetadataChangeAsync( + writeContext, + entity, + cancellationToken); var currentEpisodeIdentity = entity.IsActiveRelease ? GetEpisodeIdentity(writeContext, entity) : null; @@ -948,6 +953,10 @@ await SetEpisodeReleaseActivityAsync( entity, willHaveMappings: false, cancellationToken); + await ReconcileMappingVisibilityAfterMetadataChangeAsync( + writeContext, + entity, + cancellationToken); var currentEpisodeIdentity = entity.IsActiveRelease ? GetEpisodeIdentity(writeContext, entity) : null; @@ -975,9 +984,11 @@ internal static async Task SetEpisodeReleaseActivityAsync( CancellationToken cancellationToken) { var identity = GetEpisodeIdentity(writeContext, entity); - var hasMappings = willHaveMappings || await writeContext.FileMappings - .AsNoTracking() - .AnyAsync(mapping => mapping.AnimationInfoId == entity.Id, cancellationToken); + var hasMappings = willHaveMappings || + await writeContext.FileMappings.AsNoTracking() + .AnyAsync(mapping => mapping.AnimationInfoId == entity.Id, cancellationToken) || + await writeContext.StagedFileMappings.AsNoTracking() + .AnyAsync(mapping => mapping.AnimationInfoId == entity.Id, cancellationToken); if (identity is null || entity.MediaLibraryMissingSince is not null || !entity.IsDownloadFinished || !hasMappings) { @@ -1023,6 +1034,68 @@ await activeOthers.ExecuteUpdateAsync( : null; } + private static async Task ReconcileMappingVisibilityAfterMetadataChangeAsync( + Models.ApplicationContext writeContext, + Models.AnimationInfo entity, + CancellationToken cancellationToken) + { + var liveMappings = await writeContext.FileMappings + .Where(mapping => mapping.AnimationInfoId == entity.Id) + .ToListAsync(cancellationToken); + var stagedMappings = await writeContext.StagedFileMappings + .Where(mapping => mapping.AnimationInfoId == entity.Id) + .ToListAsync(cancellationToken); + var shouldStage = GetEpisodeIdentity(writeContext, entity) is not null && + !entity.IsActiveRelease; + + if (shouldStage) + { + if (liveMappings.Count == 0) return; + + writeContext.StagedFileMappings.RemoveRange(stagedMappings); + await writeContext.StagedFileMappings.AddRangeAsync( + liveMappings.Select(mapping => new Models.StagedFileMapping + { + Id = mapping.Id, + AnimationInfoId = mapping.AnimationInfoId, + VirtualPath = mapping.VirtualPath, + PhysicalPath = mapping.PhysicalPath, + FileStore = mapping.FileStore + }), + cancellationToken); + writeContext.FileMappings.RemoveRange(liveMappings); + return; + } + + if (liveMappings.Count > 0) + { + writeContext.StagedFileMappings.RemoveRange(stagedMappings); + return; + } + + if (stagedMappings.Count == 0) return; + var conflicts = await VirtualPathNamespaceGuard.FindConflictsAsync( + writeContext, + entity.Id, + stagedMappings.Select(mapping => mapping.VirtualPath).ToArray(), + cancellationToken); + if (conflicts.Count > 0) + throw new InvalidOperationException( + $"Cannot publish the remapped release because '{conflicts[0].OccupiedPath}' is occupied."); + + await writeContext.FileMappings.AddRangeAsync( + stagedMappings.Select(mapping => new Models.FileMapping + { + Id = mapping.Id, + AnimationInfoId = mapping.AnimationInfoId, + VirtualPath = mapping.VirtualPath, + PhysicalPath = mapping.PhysicalPath, + FileStore = mapping.FileStore + }), + cancellationToken); + writeContext.StagedFileMappings.RemoveRange(stagedMappings); + } + internal static async Task PromotePreviousEpisodeSuccessorAsync( Models.ApplicationContext writeContext, Guid changedReleaseId, @@ -1042,29 +1115,63 @@ internal static async Task PromotePreviousEpisodeSuccessorAsync( cancellationToken)) return; - var successorId = await writeContext.AnimationInfo + var successor = await writeContext.AnimationInfo .AsNoTracking() .Where(info => info.Id != changedReleaseId && info.MediaLibraryMissingSince == null && info.IsDownloadFinished && - writeContext.FileMappings.Any(mapping => mapping.AnimationInfoId == info.Id) && + (writeContext.FileMappings.Any(mapping => mapping.AnimationInfoId == info.Id) || + writeContext.StagedFileMappings.Any(mapping => mapping.AnimationInfoId == info.Id)) && EF.Property(info, "AnimationId") == previous.AnimationId && info.Season == previous.Season && info.Episode == previous.Episode) .OrderByDescending(info => info.ReleaseScore) .ThenByDescending(info => info.PublishTime) .ThenBy(info => info.Id) - .Select(info => (Guid?)info.Id) + .Select(info => new + { + info.Id, + HasLiveMappings = writeContext.FileMappings.Any(mapping => mapping.AnimationInfoId == info.Id) + }) .FirstOrDefaultAsync(cancellationToken); - if (successorId is null) return; + if (successor is null) return; + + if (!successor.HasLiveMappings) + { + var stagedMappings = await writeContext.StagedFileMappings + .Where(mapping => mapping.AnimationInfoId == successor.Id) + .OrderBy(mapping => mapping.VirtualPath) + .ToListAsync(cancellationToken); + if (stagedMappings.Count == 0) return; + + var conflicts = await VirtualPathNamespaceGuard.FindConflictsAsync( + writeContext, + successor.Id, + stagedMappings.Select(mapping => mapping.VirtualPath).ToArray(), + cancellationToken); + if (conflicts.Count > 0) return; + + await writeContext.FileMappings.AddRangeAsync( + stagedMappings.Select(mapping => new Models.FileMapping + { + Id = mapping.Id, + AnimationInfoId = mapping.AnimationInfoId, + VirtualPath = mapping.VirtualPath, + PhysicalPath = mapping.PhysicalPath, + FileStore = mapping.FileStore + }), + cancellationToken); + writeContext.StagedFileMappings.RemoveRange(stagedMappings); + } await writeContext.AnimationInfo - .Where(info => info.Id == successorId.Value) + .Where(info => info.Id == successor.Id) .ExecuteUpdateAsync( setters => setters .SetProperty(info => info.IsActiveRelease, true) .SetProperty(info => info.StateVersion, info => info.StateVersion + 1), cancellationToken); + await writeContext.SaveChangesAsync(cancellationToken); } internal readonly record struct EpisodeReleaseIdentity(Guid AnimationId, int Season, int Episode); diff --git a/SecondDimensionWatcherReDive/Repositories/FileMappingRepository.cs b/SecondDimensionWatcherReDive/Repositories/FileMappingRepository.cs index 6423776..51f9af5 100644 --- a/SecondDimensionWatcherReDive/Repositories/FileMappingRepository.cs +++ b/SecondDimensionWatcherReDive/Repositories/FileMappingRepository.cs @@ -128,20 +128,13 @@ public async Task ReplaceForAnimationInfoAsync( .AsNoTracking() .Where(mapping => mapping.AnimationInfoId == animationInfoId) .ToListAsync(cancellationToken); + var existingStagedMappings = await replaceContext.StagedFileMappings + .AsNoTracking() + .Where(mapping => mapping.AnimationInfoId == animationInfoId) + .ToListAsync(cancellationToken); var desiredMappings = mappings .Select(mapping => mapping.ToEntity()) .ToList(); - await PlaybackProgressMappingMigrator.MigrateAsync( - replaceContext, - animationInfoId, - existingMappings, - desiredMappings, - cancellationToken); - var reconciliation = await FileMappingSetReconciler.ReconcileAsync( - replaceContext, - animationInfoId, - desiredMappings, - cancellationToken); var previousEpisodeIdentity = AnimationInfoRepository.GetEpisodeIdentity( replaceContext, current); @@ -151,6 +144,28 @@ await AnimationInfoRepository.SetEpisodeReleaseActivityAsync( current, willHaveMappings: desiredMappings.Count > 0, cancellationToken); + var shouldStage = previousEpisodeIdentity is not null && !current.IsActiveRelease; + var previousMappings = existingMappings.Count > 0 + ? existingMappings + : existingStagedMappings.Select(ToFileMapping).ToList(); + await PlaybackProgressMappingMigrator.MigrateAsync( + replaceContext, + animationInfoId, + previousMappings, + desiredMappings, + cancellationToken); + var reconciliation = await FileMappingSetReconciler.ReconcileAsync( + replaceContext, + animationInfoId, + shouldStage ? [] : desiredMappings, + cancellationToken); + replaceContext.StagedFileMappings.RemoveRange(existingStagedMappings); + if (shouldStage) + { + await replaceContext.StagedFileMappings.AddRangeAsync( + desiredMappings.Select(ToStagedFileMapping), + cancellationToken); + } var currentEpisodeIdentity = current.IsActiveRelease ? AnimationInfoRepository.GetEpisodeIdentity(replaceContext, current) : null; @@ -396,9 +411,10 @@ private static IQueryable ProjectFileSystemEntries( public async Task ExistsForAnimationInfoAsync(Guid animationInfoId, CancellationToken cancellationToken) { - return await context.FileMappings - .AsNoTracking() - .AnyAsync(m => m.AnimationInfoId == animationInfoId, cancellationToken); + return await context.FileMappings.AsNoTracking() + .AnyAsync(m => m.AnimationInfoId == animationInfoId, cancellationToken) + || await context.StagedFileMappings.AsNoTracking() + .AnyAsync(m => m.AnimationInfoId == animationInfoId, cancellationToken); } public async Task TryFinalizeDownloadCancellationAsync( @@ -446,6 +462,9 @@ await finalizeContext.PlaybackProgresses await finalizeContext.FileMappings .Where(mapping => mapping.AnimationInfoId == animationInfoId) .ExecuteDeleteAsync(cancellationToken); + await finalizeContext.StagedFileMappings + .Where(mapping => mapping.AnimationInfoId == animationInfoId) + .ExecuteDeleteAsync(cancellationToken); animationInfo.IsDownloadTracked = false; animationInfo.IsDownloadFinished = false; animationInfo.IsActiveRelease = false; @@ -508,6 +527,9 @@ await strategy.ExecuteAsync(async () => await removeContext.FileMappings .Where(mapping => mapping.AnimationInfoId == animationInfoId) .ExecuteDeleteAsync(cancellationToken); + await removeContext.StagedFileMappings + .Where(mapping => mapping.AnimationInfoId == animationInfoId) + .ExecuteDeleteAsync(cancellationToken); if (animationInfo is not null) { animationInfo.IsActiveRelease = false; @@ -525,4 +547,22 @@ await AnimationInfoRepository.PromotePreviousEpisodeSuccessorAsync( await transaction.CommitAsync(cancellationToken); }); } + + private static Models.FileMapping ToFileMapping(Models.StagedFileMapping mapping) => new() + { + Id = mapping.Id, + AnimationInfoId = mapping.AnimationInfoId, + VirtualPath = mapping.VirtualPath, + PhysicalPath = mapping.PhysicalPath, + FileStore = mapping.FileStore + }; + + private static Models.StagedFileMapping ToStagedFileMapping(Models.FileMapping mapping) => new() + { + Id = mapping.Id, + AnimationInfoId = mapping.AnimationInfoId, + VirtualPath = mapping.VirtualPath, + PhysicalPath = mapping.PhysicalPath, + FileStore = mapping.FileStore + }; } diff --git a/SecondDimensionWatcherReDive/Repositories/LibrarySearchRepository.cs b/SecondDimensionWatcherReDive/Repositories/LibrarySearchRepository.cs index b1f1cf2..5c46a2c 100644 --- a/SecondDimensionWatcherReDive/Repositories/LibrarySearchRepository.cs +++ b/SecondDimensionWatcherReDive/Repositories/LibrarySearchRepository.cs @@ -15,6 +15,7 @@ public sealed class LibrarySearchRepository(Models.ApplicationContext context) private sealed record SearchCursor( DateTimeOffset SnapshotUtc, string Signature, + long Revision, Guid LastId, DateTimeOffset? PublishedAt, int? Score, @@ -75,6 +76,9 @@ public async Task SearchAsync( var cursor = DecodeCursor(request.Cursor); if (cursor is not null && !string.Equals(cursor.Signature, signature, StringComparison.Ordinal)) throw new ArgumentException("The search cursor does not match the active filters.", nameof(request)); + var revision = await ReadLibraryRevisionAsync(cancellationToken); + if (cursor is not null && cursor.Revision != revision) + throw new ArgumentException("The library changed; restart search pagination.", nameof(request)); var snapshot = cursor?.SnapshotUtc ?? DateTimeOffset.UtcNow; if (cursor is not null && cursor.LastId == Guid.Empty) throw new ArgumentException("The search cursor is invalid.", nameof(request)); @@ -102,7 +106,10 @@ public async Task SearchAsync( if (request.Season is { } season) query = query.Where(info => info.Season == season); if (request.Episode is { } episode) query = query.Where(info => info.Episode == episode); if (!string.IsNullOrWhiteSpace(request.SubtitleGroup)) - query = query.Where(info => info.ReleaseSubtitleGroup == request.SubtitleGroup); + query = query.Where(info => + info.ReleaseSubtitleGroup == request.SubtitleGroup || + (info.ReleaseSubtitleGroup == null && info.Group != null && + info.Group.Name == request.SubtitleGroup)); if (!string.IsNullOrWhiteSpace(request.Resolution)) query = query.Where(info => info.ReleaseResolution == request.Resolution); if (!string.IsNullOrWhiteSpace(request.Codec)) @@ -258,8 +265,11 @@ PARTITION BY mapping."AnimationInfoId" info.PublishTime); }).ToList(); + if (await ReadLibraryRevisionAsync(cancellationToken) != revision) + throw new ArgumentException("The library changed; restart search pagination.", nameof(request)); + var nextCursor = hasMore - ? EncodeCursor(CreateCursor(page[^1], request.Sort, snapshot, signature)) + ? EncodeCursor(CreateCursor(page[^1], request.Sort, snapshot, signature, revision)) : null; return new LibrarySearchResult(items, nextCursor); } @@ -316,10 +326,12 @@ private static SearchCursor CreateCursor( SearchRow row, LibrarySearchSort sort, DateTimeOffset snapshot, - string signature) => + string signature, + long revision) => new( snapshot, signature, + revision, row.Id, sort is LibrarySearchSort.PublishedDescending or LibrarySearchSort.ScoreDescending ? row.PublishTime @@ -335,6 +347,12 @@ sort is LibrarySearchSort.TitleAscending or LibrarySearchSort.EpisodeAscending ? row.Episode : null); + private async Task ReadLibraryRevisionAsync(CancellationToken cancellationToken) => + await context.AnimationCatalogStates.AsNoTracking() + .Where(state => state.Id == 1) + .Select(state => state.Revision) + .SingleAsync(cancellationToken); + public async Task> GetIntegrityAsync( string? tmdbId, int? season, diff --git a/SecondDimensionWatcherReDive/Repositories/MetadataReviewRepository.cs b/SecondDimensionWatcherReDive/Repositories/MetadataReviewRepository.cs index 19e0c48..f539420 100644 --- a/SecondDimensionWatcherReDive/Repositories/MetadataReviewRepository.cs +++ b/SecondDimensionWatcherReDive/Repositories/MetadataReviewRepository.cs @@ -52,6 +52,17 @@ public async Task GetQueueAsync( .GroupBy(mapping => mapping.AnimationInfoId) .Select(group => new { AnimationInfoId = group.Key, Count = group.Count() }) .ToDictionaryAsync(row => row.AnimationInfoId, row => row.Count, cancellationToken); + if (animationInfoIds.Length > 0) + { + var stagedCounts = await context.StagedFileMappings + .AsNoTracking() + .Where(mapping => animationInfoIds.Contains(mapping.AnimationInfoId)) + .GroupBy(mapping => mapping.AnimationInfoId) + .Select(group => new { AnimationInfoId = group.Key, Count = group.Count() }) + .ToListAsync(cancellationToken); + foreach (var row in stagedCounts) + mappingCounts[row.AnimationInfoId] = mappingCounts.GetValueOrDefault(row.AnimationInfoId) + row.Count; + } var operationIds = entities .Where(info => info.CurrentMetadataReviewOperationId.HasValue) @@ -227,11 +238,10 @@ public async Task ApplyPreviewAsync( return Failure(MetadataReviewMutationOutcome.NotFound, operationId); if (operation.State == MetadataReviewOperationState.Applied) { - var currentMappings = await applyContext.FileMappings - .AsNoTracking() - .Where(mapping => mapping.AnimationInfoId == animationInfo.Id) - .OrderBy(mapping => mapping.VirtualPath) - .ToListAsync(cancellationToken); + var currentMappings = await LoadOwnedMappingsAsync( + applyContext, + animationInfo.Id, + cancellationToken); var appliedSnapshots = operation.MappingSnapshots .Where(snapshot => snapshot.Kind == MetadataReviewMappingKind.Proposed) .ToList(); @@ -295,11 +305,10 @@ public async Task ApplyPreviewAsync( operationId, animationInfo.Id); - var existingMappings = await applyContext.FileMappings - .AsNoTracking() - .Where(mapping => mapping.AnimationInfoId == animationInfo.Id) - .OrderBy(mapping => mapping.VirtualPath) - .ToListAsync(cancellationToken); + var existingMappings = await LoadOwnedMappingsAsync( + applyContext, + animationInfo.Id, + cancellationToken); var mappingsBefore = existingMappings.Select(mapping => mapping.ToRecord()).ToList(); var animationInfoEntry = applyContext.Entry(animationInfo); @@ -374,6 +383,8 @@ await AnimationInfoRepository.SetEpisodeReleaseActivityAsync( var currentEpisodeIdentity = animationInfo.IsActiveRelease ? AnimationInfoRepository.GetEpisodeIdentity(applyContext, animationInfo) : null; + var shouldStage = currentEpisodeIdentity is null && + AnimationInfoRepository.GetEpisodeIdentity(applyContext, animationInfo) is not null; animationInfo.StateVersion = checked(animationInfo.StateVersion + 1); var desiredMappings = proposedSnapshots @@ -395,9 +406,14 @@ await PlaybackProgressMappingMigrator.MigrateAsync( var reconciliation = await FileMappingSetReconciler.ReconcileAsync( applyContext, animationInfo.Id, - desiredMappings, + shouldStage ? [] : desiredMappings, + cancellationToken); + await ReplaceStagedMappingsAsync( + applyContext, + animationInfo.Id, + shouldStage ? desiredMappings : [], cancellationToken); - var replacementMappings = reconciliation.Mappings; + var replacementMappings = desiredMappings; operation.State = MetadataReviewOperationState.Applied; operation.AppliedAt = appliedAt; @@ -468,11 +484,10 @@ public async Task UndoAsync( return Failure(MetadataReviewMutationOutcome.NotFound, operationId); if (operation.State == MetadataReviewOperationState.Undone) { - var idempotentCurrentMappings = await undoContext.FileMappings - .AsNoTracking() - .Where(mapping => mapping.AnimationInfoId == animationInfo.Id) - .OrderBy(mapping => mapping.VirtualPath) - .ToListAsync(cancellationToken); + var idempotentCurrentMappings = await LoadOwnedMappingsAsync( + undoContext, + animationInfo.Id, + cancellationToken); var restoredSnapshots = operation.MappingSnapshots .Where(snapshot => snapshot.Kind == MetadataReviewMappingKind.Previous) .ToList(); @@ -513,11 +528,10 @@ public async Task UndoAsync( operationId, animationInfo.Id); - var currentMappings = await undoContext.FileMappings - .AsNoTracking() - .Where(mapping => mapping.AnimationInfoId == animationInfo.Id) - .OrderBy(mapping => mapping.VirtualPath) - .ToListAsync(cancellationToken); + var currentMappings = await LoadOwnedMappingsAsync( + undoContext, + animationInfo.Id, + cancellationToken); var proposedSnapshots = operation.MappingSnapshots .Where(snapshot => snapshot.Kind == MetadataReviewMappingKind.Proposed) .ToList(); @@ -619,6 +633,8 @@ await AnimationInfoRepository.SetEpisodeReleaseActivityAsync( var currentEpisodeIdentity = animationInfo.IsActiveRelease ? AnimationInfoRepository.GetEpisodeIdentity(undoContext, animationInfo) : null; + var shouldStage = currentEpisodeIdentity is null && + AnimationInfoRepository.GetEpisodeIdentity(undoContext, animationInfo) is not null; animationInfo.StateVersion = checked(animationInfo.StateVersion + 1); var desiredMappings = previousSnapshots @@ -640,9 +656,14 @@ await PlaybackProgressMappingMigrator.MigrateAsync( var reconciliation = await FileMappingSetReconciler.ReconcileAsync( undoContext, animationInfo.Id, - desiredMappings, + shouldStage ? [] : desiredMappings, cancellationToken); - var restoredMappings = reconciliation.Mappings; + await ReplaceStagedMappingsAsync( + undoContext, + animationInfo.Id, + shouldStage ? desiredMappings : [], + cancellationToken); + var restoredMappings = desiredMappings; var undoneAt = DateTimeOffset.UtcNow; operation.State = MetadataReviewOperationState.Undone; @@ -735,6 +756,57 @@ ON CONFLICT ("Name") DO NOTHING .SingleAsync(group => group.Name == proposedGroupName, cancellationToken); } + private static async Task> LoadOwnedMappingsAsync( + Models.ApplicationContext operationContext, + Guid animationInfoId, + CancellationToken cancellationToken) + { + var liveMappings = await operationContext.FileMappings + .AsNoTracking() + .Where(mapping => mapping.AnimationInfoId == animationInfoId) + .ToListAsync(cancellationToken); + var stagedMappings = await operationContext.StagedFileMappings + .AsNoTracking() + .Where(mapping => mapping.AnimationInfoId == animationInfoId) + .Select(mapping => new Models.FileMapping + { + Id = mapping.Id, + AnimationInfoId = mapping.AnimationInfoId, + VirtualPath = mapping.VirtualPath, + PhysicalPath = mapping.PhysicalPath, + FileStore = mapping.FileStore + }) + .ToListAsync(cancellationToken); + return liveMappings + .Concat(stagedMappings) + .OrderBy(mapping => mapping.VirtualPath, StringComparer.Ordinal) + .ToList(); + } + + private static async Task ReplaceStagedMappingsAsync( + Models.ApplicationContext operationContext, + Guid animationInfoId, + IReadOnlyList desiredMappings, + CancellationToken cancellationToken) + { + var existingMappings = await operationContext.StagedFileMappings + .Where(mapping => mapping.AnimationInfoId == animationInfoId) + .ToListAsync(cancellationToken); + operationContext.StagedFileMappings.RemoveRange(existingMappings); + if (desiredMappings.Count == 0) return; + + await operationContext.StagedFileMappings.AddRangeAsync( + desiredMappings.Select(mapping => new Models.StagedFileMapping + { + Id = mapping.Id, + AnimationInfoId = mapping.AnimationInfoId, + VirtualPath = mapping.VirtualPath, + PhysicalPath = mapping.PhysicalPath, + FileStore = mapping.FileStore + }), + cancellationToken); + } + private static bool MappingSetsMatch( IReadOnlyList mappings, IReadOnlyList snapshots) diff --git a/SecondDimensionWatcherReDive/Repositories/ReleaseUpgradeRepository.cs b/SecondDimensionWatcherReDive/Repositories/ReleaseUpgradeRepository.cs index bc67c77..072434a 100644 --- a/SecondDimensionWatcherReDive/Repositories/ReleaseUpgradeRepository.cs +++ b/SecondDimensionWatcherReDive/Repositories/ReleaseUpgradeRepository.cs @@ -2,6 +2,7 @@ using Microsoft.EntityFrameworkCore; using Npgsql; using SecondDimensionWatcherReDive.Framework.DataRepository; +using SecondDimensionWatcherReDive.Utils.FileStore; namespace SecondDimensionWatcherReDive.Repositories; @@ -307,7 +308,7 @@ public async Task> GetReadyCandidateIdsAsync( (operation.Status == ReleaseUpgradeStatus.Downloading || operation.Status == ReleaseUpgradeStatus.Verifying) && operation.CandidateRelease.IsDownloadFinished && - context.FileMappings.Any(mapping => + context.StagedFileMappings.Any(mapping => mapping.AnimationInfoId == operation.CandidateReleaseId)) .OrderBy(operation => operation.CreatedAt) .Select(operation => operation.CandidateReleaseId) @@ -326,19 +327,34 @@ public async Task> GetReadyCandidateIdsAsync( cancellationToken); if (operation is null) return null; - var mappings = await context.FileMappings.AsNoTracking() - .Where(mapping => mapping.AnimationInfoId == operation.CurrentReleaseId || - mapping.AnimationInfoId == operation.CandidateReleaseId) + var previousMappings = await context.FileMappings.AsNoTracking() + .Where(mapping => mapping.AnimationInfoId == operation.CurrentReleaseId) + .OrderBy(mapping => mapping.VirtualPath) + .ToListAsync(cancellationToken); + var candidateMappings = await context.StagedFileMappings.AsNoTracking() + .Where(mapping => mapping.AnimationInfoId == operation.CandidateReleaseId) .OrderBy(mapping => mapping.VirtualPath) .ToListAsync(cancellationToken); return new ReleaseUpgradeActivation( operation.ToRecord(), - mappings.Where(mapping => mapping.AnimationInfoId == operation.CurrentReleaseId) - .Select(mapping => mapping.ToRecord()).ToList(), - mappings.Where(mapping => mapping.AnimationInfoId == operation.CandidateReleaseId) - .Select(mapping => mapping.ToRecord()).ToList()); + previousMappings.Select(mapping => mapping.ToRecord()).ToList(), + candidateMappings.Select(ToRecord).ToList()); } + public async Task> GetCandidateMappingsAsync( + Guid candidateReleaseId, + CancellationToken cancellationToken) => + await context.StagedFileMappings.AsNoTracking() + .Where(mapping => mapping.AnimationInfoId == candidateReleaseId) + .OrderBy(mapping => mapping.VirtualPath) + .Select(mapping => new FileMapping( + mapping.Id, + mapping.AnimationInfoId, + mapping.VirtualPath, + mapping.PhysicalPath, + mapping.FileStore)) + .ToListAsync(cancellationToken); + public async Task GetRollbackAsync( Guid operationId, CancellationToken cancellationToken) @@ -407,13 +423,15 @@ public async Task ActivateAsync( candidate.ReleaseScore <= current.ReleaseScore) return new ReleaseUpgradeMutationResult(false, "release_changed", operation.ToRecord()); - var mappings = await writeContext.FileMappings - .Where(mapping => mapping.AnimationInfoId == current.Id || - mapping.AnimationInfoId == candidate.Id) + var previous = await writeContext.FileMappings + .Where(mapping => mapping.AnimationInfoId == current.Id) + .OrderBy(mapping => mapping.VirtualPath) + .ToListAsync(cancellationToken); + var stagedNext = await writeContext.StagedFileMappings + .Where(mapping => mapping.AnimationInfoId == candidate.Id) .OrderBy(mapping => mapping.VirtualPath) .ToListAsync(cancellationToken); - var previous = mappings.Where(mapping => mapping.AnimationInfoId == current.Id).ToList(); - var next = mappings.Where(mapping => mapping.AnimationInfoId == candidate.Id).ToList(); + var next = stagedNext.Select(ToFileMapping).ToList(); if (previous.Count == 0 || next.Count == 0) return new ReleaseUpgradeMutationResult(false, "mapping_missing", operation.ToRecord()); if (!MappingSetsMatch(previous, expectedPreviousMappings) || @@ -445,6 +463,7 @@ await SnapshotEntryIdentitiesAsync( [current.Id, candidate.Id], replacement.Mappings, cancellationToken); + writeContext.StagedFileMappings.RemoveRange(stagedNext); await TransferPlaybackProgressAsync( writeContext, BuildActivationPlaybackTransfers(current.Id, candidate.Id, replacement), @@ -648,6 +667,22 @@ private static Models.ReleaseUpgradeMappingSnapshot ToSnapshot( FileStore = mapping.FileStore }; + private static FileMapping ToRecord(Models.StagedFileMapping mapping) => new( + mapping.Id, + mapping.AnimationInfoId, + mapping.VirtualPath, + mapping.PhysicalPath, + mapping.FileStore); + + private static Models.FileMapping ToFileMapping(Models.StagedFileMapping mapping) => new() + { + Id = mapping.Id, + AnimationInfoId = mapping.AnimationInfoId, + VirtualPath = mapping.VirtualPath, + PhysicalPath = mapping.PhysicalPath, + FileStore = mapping.FileStore + }; + private static Task SnapshotEntryIdentitiesAsync( Models.ApplicationContext writeContext, Guid operationId, @@ -741,7 +776,10 @@ private static string GetStableFileRole(string virtualPath) var fileName = virtualPath[(virtualPath.LastIndexOf('/') + 1)..]; var extension = Path.GetExtension(fileName); var stem = extension.Length == 0 ? fileName : fileName[..^extension.Length]; - return CollisionSuffixRegex().Replace(stem, string.Empty) + extension; + var roleExtension = MediaFileTypes.VideoExtensions.Contains(extension) + ? "