From b4a5f8c1891d364d1647656ec04ef0ce09c0e040 Mon Sep 17 00:00:00 2001 From: OneHotTake Date: Sun, 4 Oct 2026 09:25:35 -0500 Subject: [PATCH 1/2] Preserve diversity deadlines and settle rejected lookup leases --- InfiniteDrive.csproj | 6 +- Models/ImportDiversityEntry.cs | 13 +++- README.md | 6 +- Services/ImportReconciliationService.cs | 23 ++++--- Tests/ImportDiversityTests.cs | 24 ++++++- Tests/ImportReconciliationTests.cs | 83 +++++++++++++++++++++++++ docs/README.md | 2 +- docs/provider-diversity-queue.md | 16 +++-- plugin.json | 2 +- 9 files changed, 152 insertions(+), 23 deletions(-) diff --git a/InfiniteDrive.csproj b/InfiniteDrive.csproj index 0231cc6..03e4174 100755 --- a/InfiniteDrive.csproj +++ b/InfiniteDrive.csproj @@ -2,9 +2,9 @@ net8.0 - 0.42.14.0 - 0.42.14.0 - 0.42.14.0 + 0.42.15.0 + 0.42.15.0 + 0.42.15.0 enable InfiniteDrive InfiniteDrive diff --git a/Models/ImportDiversityEntry.cs b/Models/ImportDiversityEntry.cs index 61d3b35..d149dc9 100644 --- a/Models/ImportDiversityEntry.cs +++ b/Models/ImportDiversityEntry.cs @@ -49,8 +49,16 @@ public static void Reconcile(ImportCoverage coverage, ImportEpisode ep, string p if (!Eligible(coverage, ep)) { if (ep.Diversity != null) - { ep.Diversity.Status = providers?.Count > 1 ? "resolved" : "retired"; - ep.Diversity.Reason = providers?.Count > 1 ? "multiple_sources" : "ineligible"; } + { + // A successful ordinary refresh waits for Emby indexing. It is not a + // new diversity generation and must not keep restarting the six-hour wait. + var awaitingIndex = coverage.Exclusion.Length == 0 && coverage.SnapshotStatus == "success" && + ep.Expected && ep.Eligible && ep.Failure.Length == 0 && ep.State == "awaiting_indexing" && + providers?.Count == 1 && ep.Diversity.Profile == profile && + ep.Diversity.Generation == coverage.Generation; + ep.Diversity.Status = awaitingIndex ? "waiting" : providers?.Count > 1 ? "resolved" : "retired"; + ep.Diversity.Reason = awaitingIndex ? "awaiting_indexing" : providers?.Count > 1 ? "multiple_sources" : "ineligible"; + } return; } if (ep.Diversity == null || ep.Diversity.Profile != profile || ep.Diversity.Generation != coverage.Generation) @@ -59,6 +67,7 @@ public static void Reconcile(ImportCoverage coverage, ImportEpisode ep, string p else if (ep.Diversity.Status is "retired" or "resolved") { ep.Diversity.Status = "waiting"; ep.Diversity.Reason = "single_source"; ep.Diversity.NextAttempt = now.AddHours(6); } + else if (ep.Diversity.Reason == "awaiting_indexing") ep.Diversity.Reason = "single_source"; if (ep.Diversity.Status == "in_flight" && (ep.LeaseUntil == null || ep.LeaseUntil <= now)) { ep.Diversity.Status = "waiting"; ep.Diversity.Reason = "interrupted"; ep.Diversity.NextAttempt = now.AddMinutes(5); } diff --git a/README.md b/README.md index 1fba9c3..3f3e966 100755 --- a/README.md +++ b/README.md @@ -9,9 +9,9 @@ imports titles and repairs missing files. A stream will appear. Probably. -**Current version: 0.42.14** +**Current version: 0.42.15** -Release 0.42.14 adds eventual provider diversity while retaining the administrator whole-title blocks and recoverable managed-stream cleanup from 0.42.13. It targets **Emby 4.10.0.40**. +Release 0.42.15 fixes diversity deadlines across routine refresh indexing and immediately releases rejected diversity leases. It retains the eventual provider diversity queue from 0.42.14 and administrator whole-title blocks from 0.42.13. It targets **Emby 4.10.0.40**. ## How we run it @@ -240,7 +240,7 @@ With **Repair & refresh** enabled, Marvin gradually revisits indexed movies and The queue defaults on. **Pause source diversity** stops its lookups/additions while essential repair continues; **Enable eventual source diversity** resumes normal scheduled processing. Entries, backoff and attempt history persist. Neither button triggers a pass or resets retries. There is no force-drain action. Unknown labels, blocked/removed/owned titles, unconfirmed identities and essential failures are excluded. -New entries wait six hours. Missing items and ordinary refreshes have priority; diversity borrows at most two checks per normal pass or eight per catch-up pass from the existing shared allowance, serially. With stale health it can make one shared probe only if essential work has not already reserved a check. Same-source/empty diversity results escalate through 6h, 1d, 2d, 3d and 7d, plus up to 20% jitter and longer cooldowns. The queue persists beyond catch-up expiry; gradual processing does not guarantee a second source exists. +New entries wait six hours. Routine refresh and re-indexing preserve an existing diversity deadline and backoff; they do not restart that wait. Final guard rejection clears only the completed lookup’s own lease. Missing items and ordinary refreshes have priority; diversity borrows at most two checks per normal pass or eight per catch-up pass from the existing shared allowance, serially. With stale health it can make one shared probe only if essential work has not already reserved a check. Same-source/empty diversity results escalate through 6h, 1d, 2d, 3d and 7d, plus up to 20% jitter and longer cooldowns. The queue persists beyond catch-up expiry; gradual processing does not guarantee a second source exists. A successful diversity check **adds** one validated variant and preserves every working file. New variants require a known different label and positive exact upstream size no greater than 40GB. An item with all eight version slots occupied waits for capacity; Marvin does not delete a working choice to force diversity. A saved addition does not prove playback or independent provider infrastructure. Ordinary refresh retains its existing replacement behavior and can change coverage later. diff --git a/Services/ImportReconciliationService.cs b/Services/ImportReconciliationService.cs index 598e422..1870760 100644 --- a/Services/ImportReconciliationService.cs +++ b/Services/ImportReconciliationService.cs @@ -117,19 +117,27 @@ async Task CompleteAsync(ResolutionWork work) { var live = await _db.GetImportCoverageAsync(work.Coverage.Identity); var episode = live?.Items.FirstOrDefault(x => x.Key == work.Episode.Key); - if (_mode() != ImportMode.Repair || live == null || episode == null || - live.Generation != work.Generation || episode.Lease != work.Lease || - !episode.Eligible || !episode.Expected || - !await IsAuthorizedAsync(live, work.Item, token) || _inventory.IsOwned(work.Item)) return; + // Only the owner of this lease may settle it, even when other final + // guards changed. A stale result must never clear a replacement lease. + if (live == null || episode == null || episode.Lease != work.Lease) return; + var authorized = episode.Eligible && episode.Expected && + await IsAuthorizedAsync(live, work.Item, token) && !_inventory.IsOwned(work.Item); + var finalGuards = authorized && _mode() == ImportMode.Repair && live.Generation == work.Generation; var now = _clock(); if (work.Diversity) { var entry = episode.Diversity; - if (!_diversityEnabled() || entry == null || entry.Profile != _inventory.ProfileFingerprint || + if (!finalGuards || !_diversityEnabled() || entry == null || entry.Profile != _inventory.ProfileFingerprint || entry.Generation != live.Generation || !ImportDiversityPolicy.Eligible(live, episode) || DiversitySignature(episode) != work.Signature) { - if (entry != null) { entry.Status = "waiting"; entry.Reason = "guard_changed"; entry.NextAttempt = now.AddMinutes(5); } + if (entry != null) + { + var retired = !authorized || live.Exclusion.Length > 0 || episode.Failure.Length > 0; + entry.Status = retired ? "retired" : "waiting"; + entry.Reason = retired ? "ineligible" : "guard_changed"; + if (!retired) entry.NextAttempt = now.AddMinutes(5); + } episode.Lease = null; episode.LeaseUntil = null; await _db.SaveImportCoverageAsync(live, token); return; } @@ -168,6 +176,7 @@ async Task CompleteAsync(ResolutionWork work) if (ix >= 0) work.Coverage.Items[ix] = episode; return; } + if (!finalGuards) return; var failure = result.Failure; sourceHealth.Record(failure, now); await _db.SaveImportSourceHealthAsync(sourceHealth, token); @@ -204,8 +213,6 @@ async Task CompleteAsync(ResolutionWork work) } episode.Lease = null; episode.LeaseUntil = null; ImportDiversityPolicy.Reconcile(live, episode, _inventory.ProfileFingerprint, now); - if (failure.Length == 0 && episode.Diversity?.Status == "waiting") - episode.Diversity.NextAttempt = now.AddHours(6); await _db.SaveImportCoverageAsync(live, token); if (work.Reprocess) await _db.SetImportReprocessStatusAsync(live.Identity, episode.Key, failure.Length == 0 ? "published" : "failed", now, token); diff --git a/Tests/ImportDiversityTests.cs b/Tests/ImportDiversityTests.cs index b94595b..a5eaf7d 100644 --- a/Tests/ImportDiversityTests.cs +++ b/Tests/ImportDiversityTests.cs @@ -44,13 +44,35 @@ [Fact] public void QueueIsIdempotentAndProfileGenerationChangesRestartItsOwnWait var second = ep.Diversity; ImportDiversityPolicy.Reconcile(c, ep, "profile2", Now.AddHours(2)); Assert.NotSame(second, ep.Diversity); Assert.Equal("profile2", ep.Diversity!.Profile); } - [Theory] [InlineData("excluded")] [InlineData("index_mismatch")] [InlineData("retrying")] [InlineData("awaiting_indexing")] + [Theory] [InlineData("excluded")] [InlineData("index_mismatch")] [InlineData("retrying")] public void NonIndexedStatesRetireInsteadOfConsumingDiversityCredits(string state) { var ep = Single(); var c = Coverage(ep); ImportDiversityPolicy.Reconcile(c, ep, "p", Now); ep.State = state; ImportDiversityPolicy.Reconcile(c, ep, "p", Now); Assert.Equal("retired", ep.Diversity!.Status); } + [Fact] public void TransientRefreshIndexingPreservesPersistedDueTimeAndBackoff() + { + var ep = Single(); var c = Coverage(ep); ImportDiversityPolicy.Reconcile(c, ep, "p", Now); + ImportDiversityPolicy.Defer(ep.Diversity!, "same_source", Now, 0, DateTimeOffset.MinValue); + var due = ep.Diversity!.NextAttempt; + for (var hour = 1; hour <= 8; hour++) + { + ep.State = "awaiting_indexing"; + ImportDiversityPolicy.Reconcile(c, ep, "p", Now.AddHours(hour)); + Assert.Equal("waiting", ep.Diversity.Status); + Assert.Equal("awaiting_indexing", ep.Diversity.Reason); + Assert.False(ImportDiversityPolicy.Eligible(c, ep)); + Assert.Equal(due, ep.Diversity.NextAttempt); Assert.Equal(1, ep.Diversity.Streak); + ep.State = "indexed"; + ImportDiversityPolicy.Reconcile(c, ep, "p", Now.AddHours(hour)); + Assert.Equal("single_source", ep.Diversity.Reason); + Assert.Equal(due, ep.Diversity.NextAttempt); Assert.Equal(1, ep.Diversity.Streak); + } + ep.State = "awaiting_indexing"; c.Exclusion = "not_authorized"; + ImportDiversityPolicy.Reconcile(c, ep, "p", Now.AddHours(9)); + Assert.Equal("retired", ep.Diversity.Status); + } [Fact] public void ActualFailuresAndBlocksWinOverDiversity() { var ep = Single(); var c = Coverage(ep); ImportDiversityPolicy.Reconcile(c, ep, "p", Now); diff --git a/Tests/ImportReconciliationTests.cs b/Tests/ImportReconciliationTests.cs index 1475cad..f0a340e 100644 --- a/Tests/ImportReconciliationTests.cs +++ b/Tests/ImportReconciliationTests.cs @@ -794,6 +794,89 @@ [Fact] public async Task DiversityEssentialAttemptWinsAndStaleHealthAllowsOnlyOn Assert.Equal(new[]{2},h.Inventory.ResolvedEpisodes); } + [Fact] public async Task HourlyRefreshAndReindexingPreserveDiversityDueTimeUntilItCanDispatch() + { + using var h = new Harness(); await SeedDiversity(h); + h.Config.ImportCatchUpStartedAt = ""; h.Config.ImportCatchUpUntil = ""; + var c = await h.State(); var due = h.Now.AddHours(6); + c.Items[0].Diversity!.NextAttempt = due; + c.Items[0].Diversity!.Streak = 2; + await h.Db.SaveImportCoverageAsync(c); + h.Inventory.Service = "TorBox"; + for (var hour = 1; hour <= 6; hour++) + { + h.Now = Now.AddHours(hour).AddMinutes(hour); + await h.Run(); // Successful ordinary refresh is temporarily awaiting indexing. + Assert.Equal("awaiting_indexing", (await h.State()).Items[0].State); + await h.Run(ImportMode.Observe); // Emby indexes the unchanged single-provider choices. + var episode = (await h.State()).Items[0]; + Assert.Equal(due, episode.Diversity!.NextAttempt); + Assert.Equal(2, episode.Diversity.Streak); + } + h.Db = new DatabaseManager(h.Directory.Path, NullLogger.Instance); h.Db.Initialise(); + h.Inventory.Service = "Usenet"; + await h.Run(); // The latest refresh is fresh; the original diversity due time has elapsed. + var final = (await h.State()).Items[0]; + Assert.Equal("resolved", final.Diversity!.Status); + Assert.Equal(1, h.Inventory.Added); + Assert.Equal(6, h.Inventory.Published); + Assert.Equal(7, await h.Db.GetRecentImportAttemptsAsync(h.Now)); + } + + [Theory] + [InlineData("observe", "waiting")] + [InlineData("generation", "waiting")] + [InlineData("blocked", "retired")] + [InlineData("owned", "retired")] + [InlineData("ineligible", "retired")] + [InlineData("unexpected", "retired")] + public async Task RejectedDiversityCompletionImmediatelyClearsItsLease(string change, string status) + { + using var h = new Harness(); await SeedDiversity(h); h.Inventory.Service = "Usenet"; + var mode = ImportMode.Repair; + h.Inventory.OnResolve = async () => + { + if (change == "observe") mode = ImportMode.Observe; + else if (change == "owned") h.Inventory.Owned = true; + else if (change == "blocked") + await h.Db.UpsertBlockedItemAsync(h.Item.AioId, null, null, "fixture", "series", "test"); + else + { + var live = await h.State(); + if (change == "generation") live.Generation++; + if (change == "ineligible") live.Items[0].Eligible = false; + if (change == "unexpected") live.Items[0].Expected = false; + await h.Db.SaveImportCoverageAsync(live); + } + }; + await new ImportReconciliationService(h.Db, h.Inventory, () => mode, TimeZoneInfo.Utc, + () => h.Now, () => h.Cooldown, () => ImportWorkBudget.For(h.Config, h.Now), + () => h.Config.ImportProviderDiversityEnabled).RunAsync(default, new[] { h.Item }); + var e = (await h.State()).Items[0]; + Assert.Null(e.Lease); Assert.Null(e.LeaseUntil); + Assert.Equal(status, e.Diversity!.Status); + Assert.Equal(0, e.Diversity.Streak); + Assert.Equal(1, e.Diversity.Attempts); + Assert.Equal(0, h.Inventory.Added); Assert.Single(e.Paths); + Assert.Equal("", e.Failure); + Assert.Equal(1, await h.Db.GetRecentImportAttemptsAsync(h.Now)); + } + + [Fact] public async Task StaleDiversityCompletionCannotClearAReplacementLease() + { + using var h = new Harness(); await SeedDiversity(h); h.Inventory.Service = "Usenet"; + var expiry = h.Now.AddMinutes(20); + h.Inventory.OnResolve = async () => + { + var live = await h.State(); + live.Items[0].Lease = "replacement"; live.Items[0].LeaseUntil = expiry; + await h.Db.SaveImportCoverageAsync(live); + }; + await h.Run(); var e = (await h.State()).Items[0]; + Assert.Equal("replacement", e.Lease); Assert.Equal(expiry, e.LeaseUntil); + Assert.Equal("in_flight", e.Diversity!.Status); Assert.Equal(0, h.Inventory.Added); + } + private sealed class Harness : IDisposable { public TempDir Directory = new(); public DatabaseManager Db; public FakeInventory Inventory = new(); diff --git a/docs/README.md b/docs/README.md index 58600d7..18edec5 100644 --- a/docs/README.md +++ b/docs/README.md @@ -28,4 +28,4 @@ old test plans and dated reports. These are not current configuration instructions. The implementation and current guides take precedence over historical claims. -Updated October 3, 2026 for the 0.42.14 source tree. +Updated October 4, 2026 for the 0.42.15 source tree. diff --git a/docs/provider-diversity-queue.md b/docs/provider-diversity-queue.md index d62466b..c97bc46 100644 --- a/docs/provider-diversity-queue.md +++ b/docs/provider-diversity-queue.md @@ -1,6 +1,6 @@ # Eventual source diversity — Marvin's durable queue -Implemented for 0.42.14, targeting Emby 4.10.0.40. Deployment and dated QA evidence are recorded separately in the release notes and stack handoff. This queue improves already indexed items with one known delivery source. It does not guarantee a second source exists, represent a download, or establish playback. +Implemented in 0.42.14; the 0.42.15 source fixes refresh/indexing deadlines and final-guard lease cleanup, targeting Emby 4.10.0.40. Deployment and dated QA evidence are recorded separately in the release notes and stack handoff. This queue improves already indexed items with one known delivery source. It does not guarantee a second source exists, represent a download, or establish playback. ## Enable, pause and ownership @@ -36,7 +36,9 @@ Before an outbound diversity lookup, production also verifies the retained file Creation, endpoint/profile fingerprint change or coverage generation change creates one new entry with a **six-hour** initial wait and zero diversity streak/attempts. It does not reset the essential failure state, existing attempt ledger or provider cooldown. The fingerprint detects configured endpoint changes, not opaque AIO server-side policy edits at the same endpoint; those do not automatically shorten due times. -Dispatch claims the existing episode lease with a random ID and **ten-minute** expiry, persists entry `in_flight`, increments its Attempts and reserves the shared attempt credit before starting transport. Publication rechecks live generation, lease, enabled mode, authorization/ownership, current profile and a signature of retained paths/hashes/labels. A changed guard prevents addition. Every completed path clears its lease; budget cancellation leaves queue work waiting five minutes without penalizing the repair item. After a crash, a missing/expired lease on `in_flight` becomes waiting with a five-minute interruption delay when reconciled. Active leases cannot be stolen. +A successful ordinary single-source refresh with the same profile/generation temporarily sets the episode to `awaiting_indexing`. An existing diversity entry remains `waiting / awaiting_indexing`, preserving its CreatedAt, NextAttempt, Streak, Attempts and CheckedAt. Dispatch remains forbidden until the episode is indexed. Re-indexing restores `single_source` without postponing the deadline; repeated hourly refreshes do not restart the six-hour window. Hard failures, exclusions, missing/unconfirmed identities or changed profile/generation are not transient indexing and retain their existing retirement/new-entry rules. Entries already delayed by 0.42.14 are not bulk-reset or backdated: 0.42.15 prevents future drift, while their persisted deadline remains authoritative. + +Dispatch claims the existing episode lease with a random ID and **ten-minute** expiry, persists entry `in_flight`, increments its Attempts and reserves the shared attempt credit before starting transport. Publication rechecks live generation, lease, enabled mode, authorization/ownership, current profile and a signature of retained paths/hashes/labels. A changed guard prevents addition. Final guard rejection immediately clears the completed diversity lookup’s own Lease and LeaseUntil before returning. Mode, disable, profile/generation or signature changes defer it as `waiting / guard_changed` for five minutes; blocked, owned, unexpected/ineligible, excluded or essential-failed work becomes `retired / ineligible`. These guard outcomes preserve reserved attempts and streak, do not record source health failures, and publish nothing. Missing coverage/episode means there is no current entry to clean up. A different current lease belongs to newer work and must never be cleared or changed. Every completed path still owning its lease clears it; budget cancellation leaves queue work waiting five minutes without penalizing the repair item. After a crash, a missing/expired lease on `in_flight` becomes waiting with a five-minute interruption delay when reconciled. Active leases cannot be stolen. Markers `import_diversity_schema=1` and `import_diversity_enabled` are written by native passes for read-only reporting. Their timestamps/snapshots are not instantaneous UI control acknowledgements. Reading administrator coverage/status or the public collector never creates entries. @@ -66,6 +68,8 @@ Slice cancellation, interrupted lease and changed publication guard wait five mi | --- | --- | --- | --- | | New eligible single-source item | waiting / single_source | none | creation +6h | | Same identity/profile/generation census | retains entry/streak/due time | none | unchanged | +| Ordinary single-source refresh awaiting Emby indexing | waiting / awaiting_indexing; no diversity dispatch | none for reconciliation; ordinary refresh retains its own reservation | unchanged | +| Re-indexed after that refresh | waiting / single_source; retained counters | none | unchanged | | Eight current variants | capacity / version_limit | none | no diversity dispatch until room exists | | Capacity becomes <8 | waiting / single_source | none | retained due time | | Due + enabled Repair + guards + spare budget | in_flight; lease; Attempts+1 | one shared reservation | lookup deadline applies | @@ -73,7 +77,7 @@ Slice cancellation, interrupted lease and changed publication guard wait five mi | Transport/deadline/429/config/publication failure | waiting; distinct reason; Streak+1 | retains reservation | error ladder + jitter/cooldown | | Valid second source saved/evidence verified | resolved / multiple_sources; streak0 | retains reservation | null | | Source/authorization/generation changes in flight | no addition; guard_changed if still live | retains reservation | +5m, then reconciliation | -| Block/ownership/ineligible snapshot | retired / ineligible on observation; dispatch/publication forbidden immediately | none beyond prior reservation | no dispatch | +| Block/ownership/ineligible snapshot | retired / ineligible; completed diversity lease cleared immediately if still owned | none beyond prior reservation | no dispatch | | Multiple saved sources found by census | resolved / multiple_sources | none | no dispatch | | Retired/resolved item later eligible with one source | waiting / single_source; no force reset of old streak | none | now +6h | | Process/slice interruption | waiting / interrupted or slice_cancelled | prior reservation retained | +5m after reconciliation/cancellation | @@ -100,8 +104,12 @@ Examples: - HTTP429: retain TorBox and store the actual reason; apply the current error rung plus jitter and any longer global cooldown. Native circuit also closes; no queue-wide reset. - Outage: no calls while health/cooldown pauses. Afterwards one bounded shared probe while readiness is unproven; full 2/8 cap only with readiness and spare credits. - Restart: waiting state/due/streak persist. An expired in-flight lease waits five more minutes and revalidates before another reservation. -- Block during lookup: final alias/ownership/generation guards prevent publication. A later observation retires the entry; unblocking is never automatic. +- Block during lookup: final alias/ownership guards prevent publication, retire the diversity entry and immediately clear only its own lease; unblocking is never automatic. - Endpoint change: a new six-hour entry for the new fingerprint; an old result cannot publish. Server-side edits at the same endpoint do not reset waiting times. - Catch-up expires with 1,000 entries: they remain and contend only for normal remaining capacity. No window extension, quota increase or completion promise. Before upgrade, take private current configuration and stopped-database backups. Prefer pausing diversity through the native control for behavior rollback; this preserves the queue and newer state. A binary downgrade never requires restoring an older database or deleting ledger/coverage/archives. Older binaries can ignore/drop unknown Diversity JSON members when rewriting coverage; preserve the current backup for audit and expect conservative rediscovery after re-upgrade, rather than restoring historical user state. Protect active playback/recordings during any approved Emby restart. The beta/isolated QA plugin must never be reused as a production artifact. + +## October 4 lifecycle regression verification + +The release-0.42.14 code failed seven new runtime regression cases: six guard changes retained their completed lease, and the first hourly refresh/re-index cycle postponed its diversity deadline. The patched coordinator passes six consecutive normal hourly refresh/re-index cycles, SQLite reopen and a due additive lookup; it immediately settles Observe/generation/block/ownership/Expected/Eligible changes, while leaving a replacement lease untouched. A policy regression also preserves due/streak through eight transient indexing cycles and confirms a hard exclusion still retires work. Full pinned-ABI test/build and production deployment evidence are recorded separately; synthetic URLs establish state and publication behavior, not playback. diff --git a/plugin.json b/plugin.json index bca739c..9bd15dd 100755 --- a/plugin.json +++ b/plugin.json @@ -5,7 +5,7 @@ "overview": "InfiniteDrive discovers your AIOStreams catalog, writes .strm files, and resolves debrid URLs on demand. Like the Infinite Improbability Drive: a stream will appear. Probably. No external processes needed.", "owner": "InfiniteDrive", "category": "General", - "version": "0.42.14.0", + "version": "0.42.15.0", "targetAbi": "4.10.0.40", "framework": "net8.0", "imageUrl": "" From 1505701a3dafd720ee8a186a8e3f3ac057815033 Mon Sep 17 00:00:00 2001 From: OneHotTake Date: Sun, 4 Oct 2026 09:33:06 -0500 Subject: [PATCH 2/2] Rearm diversity after resolved provider coverage collapses --- Models/ImportDiversityEntry.cs | 5 +++++ Tests/ImportDiversityTests.cs | 20 ++++++++++++++++++++ docs/provider-diversity-queue.md | 4 ++-- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/Models/ImportDiversityEntry.cs b/Models/ImportDiversityEntry.cs index d149dc9..4404b65 100644 --- a/Models/ImportDiversityEntry.cs +++ b/Models/ImportDiversityEntry.cs @@ -56,6 +56,11 @@ public static void Reconcile(ImportCoverage coverage, ImportEpisode ep, string p ep.Expected && ep.Eligible && ep.Failure.Length == 0 && ep.State == "awaiting_indexing" && providers?.Count == 1 && ep.Diversity.Profile == profile && ep.Diversity.Generation == coverage.Generation; + // Resolved/retired entries are entering a new single-source wait. A + // resolved entry may have no deadline; do not strand it by preserving null. + if (awaitingIndex && (ep.Diversity.Status is "resolved" or "retired" || + ep.Diversity.NextAttempt == null)) + ep.Diversity.NextAttempt = now.AddHours(6); ep.Diversity.Status = awaitingIndex ? "waiting" : providers?.Count > 1 ? "resolved" : "retired"; ep.Diversity.Reason = awaitingIndex ? "awaiting_indexing" : providers?.Count > 1 ? "multiple_sources" : "ineligible"; } diff --git a/Tests/ImportDiversityTests.cs b/Tests/ImportDiversityTests.cs index a5eaf7d..086faba 100644 --- a/Tests/ImportDiversityTests.cs +++ b/Tests/ImportDiversityTests.cs @@ -73,6 +73,26 @@ [Fact] public void TransientRefreshIndexingPreservesPersistedDueTimeAndBackoff() ImportDiversityPolicy.Reconcile(c, ep, "p", Now.AddHours(9)); Assert.Equal("retired", ep.Diversity.Status); } + [Theory] [InlineData("resolved", true)] [InlineData("resolved", false)] + [InlineData("retired", true)] [InlineData("retired", false)] + public void ReenteringSingleSourceAfterRefreshGetsANewDeadline(string status, bool noDeadline) + { + var ep = Single(); var c = Coverage(ep); ImportDiversityPolicy.Reconcile(c, ep, "p", Now); + ep.Diversity!.Status = status; + ep.Diversity.NextAttempt = noDeadline ? null : Now.AddHours(-1); + ep.Diversity.Attempts = 3; ep.Diversity.Streak = 2; + ep.State = "awaiting_indexing"; + ImportDiversityPolicy.Reconcile(c, ep, "p", Now.AddHours(1)); + Assert.Equal("waiting", ep.Diversity.Status); + Assert.Equal("awaiting_indexing", ep.Diversity.Reason); + Assert.Equal(Now.AddHours(7), ep.Diversity.NextAttempt); + Assert.False(ImportDiversityPolicy.Eligible(c, ep)); + ep.State = "indexed"; + ImportDiversityPolicy.Reconcile(c, ep, "p", Now.AddHours(2)); + Assert.Equal("single_source", ep.Diversity.Reason); + Assert.Equal(Now.AddHours(7), ep.Diversity.NextAttempt); + Assert.Equal(3, ep.Diversity.Attempts); Assert.Equal(2, ep.Diversity.Streak); + } [Fact] public void ActualFailuresAndBlocksWinOverDiversity() { var ep = Single(); var c = Coverage(ep); ImportDiversityPolicy.Reconcile(c, ep, "p", Now); diff --git a/docs/provider-diversity-queue.md b/docs/provider-diversity-queue.md index c97bc46..ae08d2b 100644 --- a/docs/provider-diversity-queue.md +++ b/docs/provider-diversity-queue.md @@ -36,7 +36,7 @@ Before an outbound diversity lookup, production also verifies the retained file Creation, endpoint/profile fingerprint change or coverage generation change creates one new entry with a **six-hour** initial wait and zero diversity streak/attempts. It does not reset the essential failure state, existing attempt ledger or provider cooldown. The fingerprint detects configured endpoint changes, not opaque AIO server-side policy edits at the same endpoint; those do not automatically shorten due times. -A successful ordinary single-source refresh with the same profile/generation temporarily sets the episode to `awaiting_indexing`. An existing diversity entry remains `waiting / awaiting_indexing`, preserving its CreatedAt, NextAttempt, Streak, Attempts and CheckedAt. Dispatch remains forbidden until the episode is indexed. Re-indexing restores `single_source` without postponing the deadline; repeated hourly refreshes do not restart the six-hour window. Hard failures, exclusions, missing/unconfirmed identities or changed profile/generation are not transient indexing and retain their existing retirement/new-entry rules. Entries already delayed by 0.42.14 are not bulk-reset or backdated: 0.42.15 prevents future drift, while their persisted deadline remains authoritative. +A successful ordinary single-source refresh with the same profile/generation temporarily sets the episode to `awaiting_indexing`. An already-pending diversity entry remains `waiting / awaiting_indexing`, preserving its CreatedAt, NextAttempt, Streak, Attempts and CheckedAt. A resolved/retired entry that becomes single-source again instead receives a fresh six-hour deadline, including when the old deadline is null; attempts and streak remain intact. A missing deadline is also repaired on this transition. Dispatch remains forbidden until the episode is indexed. Re-indexing restores `single_source` without postponing the deadline; repeated hourly refreshes do not restart the six-hour window. Hard failures, exclusions, missing/unconfirmed identities or changed profile/generation are not transient indexing and retain their existing retirement/new-entry rules. Entries already delayed by 0.42.14 are not bulk-reset or backdated: 0.42.15 prevents future drift, while their persisted deadline remains authoritative. Dispatch claims the existing episode lease with a random ID and **ten-minute** expiry, persists entry `in_flight`, increments its Attempts and reserves the shared attempt credit before starting transport. Publication rechecks live generation, lease, enabled mode, authorization/ownership, current profile and a signature of retained paths/hashes/labels. A changed guard prevents addition. Final guard rejection immediately clears the completed diversity lookup’s own Lease and LeaseUntil before returning. Mode, disable, profile/generation or signature changes defer it as `waiting / guard_changed` for five minutes; blocked, owned, unexpected/ineligible, excluded or essential-failed work becomes `retired / ineligible`. These guard outcomes preserve reserved attempts and streak, do not record source health failures, and publish nothing. Missing coverage/episode means there is no current entry to clean up. A different current lease belongs to newer work and must never be cleared or changed. Every completed path still owning its lease clears it; budget cancellation leaves queue work waiting five minutes without penalizing the repair item. After a crash, a missing/expired lease on `in_flight` becomes waiting with a five-minute interruption delay when reconciled. Active leases cannot be stolen. @@ -112,4 +112,4 @@ Before upgrade, take private current configuration and stopped-database backups. ## October 4 lifecycle regression verification -The release-0.42.14 code failed seven new runtime regression cases: six guard changes retained their completed lease, and the first hourly refresh/re-index cycle postponed its diversity deadline. The patched coordinator passes six consecutive normal hourly refresh/re-index cycles, SQLite reopen and a due additive lookup; it immediately settles Observe/generation/block/ownership/Expected/Eligible changes, while leaving a replacement lease untouched. A policy regression also preserves due/streak through eight transient indexing cycles and confirms a hard exclusion still retires work. Full pinned-ABI test/build and production deployment evidence are recorded separately; synthetic URLs establish state and publication behavior, not playback. +The release-0.42.14 code failed seven new runtime regression cases: six guard changes retained their completed lease, and the first hourly refresh/re-index cycle postponed its diversity deadline. The patched coordinator passes six consecutive normal hourly refresh/re-index cycles, SQLite reopen and a due additive lookup; it immediately settles Observe/generation/block/ownership/Expected/Eligible changes, while leaving a replacement lease untouched. Policy regressions also preserve due/streak through eight transient indexing cycles, confirm a hard exclusion still retires work, and verify that resolved/retired entries with null or expired deadlines regain a fresh six-hour deadline after provider collapse. The provider-collapse tests were reproduced failing against the initial fix before correction. Full pinned-ABI test/build and production deployment evidence are recorded separately; synthetic URLs establish state and publication behavior, not playback.