diff --git a/src/Chaptarr.Core.Test/ImportLists/HardcoverLibraryImportListCursorFixture.cs b/src/Chaptarr.Core.Test/ImportLists/HardcoverLibraryImportListCursorFixture.cs new file mode 100644 index 00000000..fbfbacee --- /dev/null +++ b/src/Chaptarr.Core.Test/ImportLists/HardcoverLibraryImportListCursorFixture.cs @@ -0,0 +1,197 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using NLog; +using NUnit.Framework; +using NzbDrone.Common.Http; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.ImportLists; +using NzbDrone.Core.ImportLists.Hardcover.Library; +using NzbDrone.Core.Profiles.Metadata; +using NzbDrone.Core.Profiles.Qualities; +using NzbDrone.Core.Tags; + +namespace Chaptarr.Core.Test.ImportLists +{ + [TestFixture] + public class HardcoverLibraryImportListCursorFixture + { + private class HttpClientProxy : DispatchProxy + { + public List Requests { get; } = new(); + + protected override object Invoke(MethodInfo targetMethod, object[] args) + { + if (targetMethod?.Name == nameof(IHttpClient.Execute) && args?.Length >= 1 && args[0] is HttpRequest request) + { + Requests.Add(request); + + var query = request.ContentSummary ?? (request.ContentData != null ? System.Text.Encoding.UTF8.GetString(request.ContentData) : request.Url?.FullUri ?? ""); + + if (query.Contains("query Me")) + { + var json = "{\"data\":{\"me\":[{\"id\":42,\"username\":\"testuser\"}]}}"; + return new HttpResponse(request, new HttpHeader(), json); + } + + if (query.Contains("query OwnedListBooks")) + { + var json = "{\"data\":{\"list_books\":[{\"id\":20,\"book\":{\"id\":502,\"canonical_id\":502,\"title\":\"Owned Book\",\"contributions\":[{\"author_id\":202,\"contribution\":\"Author\",\"author\":{\"id\":202,\"name\":\"Owned Author\",\"canonical_id\":202,\"identifiers\":\"\"}}]}}]}}"; + return new HttpResponse(request, new HttpHeader(), json); + } + + if (query.Contains("query OwnedList(")) + { + var json = "{\"data\":{\"lists\":[{\"id\":100}]}}"; + return new HttpResponse(request, new HttpHeader(), json); + } + + if (query.Contains("query UserBooks")) + { + var json = "{\"data\":{\"user_books\":[{\"id\":10,\"updated_at\":\"2026-08-19T20:00:00Z\",\"book\":{\"id\":501,\"canonical_id\":501,\"title\":\"Test Book\",\"contributions\":[{\"author_id\":201,\"contribution\":\"Author\",\"author\":{\"id\":201,\"name\":\"Test Author\",\"canonical_id\":201,\"identifiers\":\"\"}}]}}]}}"; + return new HttpResponse(request, new HttpHeader(), json); + } + + return new HttpResponse(request, new HttpHeader(), "{\"data\":{}}"); + } + + return null; + } + } + + private class StateRepositoryProxy : DispatchProxy + { + public HardcoverLibraryImportListState State { get; set; } + public List Inserted { get; } = new(); + public List Updated { get; } = new(); + + protected override object Invoke(MethodInfo targetMethod, object[] args) + { + if (targetMethod?.Name == nameof(IHardcoverLibraryImportListStateRepository.GetByImportListId)) + { + return State; + } + + if (targetMethod?.Name == nameof(IHardcoverLibraryImportListStateRepository.Insert) && args?.Length >= 1 && args[0] is HardcoverLibraryImportListState insertModel) + { + insertModel.Id = 1; + Inserted.Add(insertModel); + State = insertModel; + return insertModel; + } + + if (targetMethod?.Name == nameof(IHardcoverLibraryImportListStateRepository.Update) && args?.Length >= 1 && args[0] is HardcoverLibraryImportListState updateModel) + { + Updated.Add(updateModel); + State = updateModel; + return updateModel; + } + + return null; + } + } + + private class StatusServiceProxy : DispatchProxy + { + public List SuccessRecorded { get; } = new(); + public List FailureRecorded { get; } = new(); + + protected override object Invoke(MethodInfo targetMethod, object[] args) + { + if (targetMethod?.Name == nameof(IImportListStatusService.RecordSuccess) && args?.Length >= 1 && args[0] is int successId) + { + SuccessRecorded.Add(successId); + return null; + } + + if (targetMethod?.Name == nameof(IImportListStatusService.RecordFailure) && args?.Length >= 1 && args[0] is int failureId) + { + FailureRecorded.Add(failureId); + return null; + } + + return null; + } + } + + private class ConfigProxy : DispatchProxy + { + public bool HardcoverEnabled { get; set; } = true; + public string HardcoverApiKey { get; set; } = "test-global-key"; + + protected override object Invoke(MethodInfo targetMethod, object[] args) + { + if (targetMethod?.Name == "get_HardcoverEnabled") return HardcoverEnabled; + if (targetMethod?.Name == "get_HardcoverApiKey") return HardcoverApiKey; + return null; + } + } + + [Test] + public void should_not_persist_cursors_in_fetch_until_commit_state_is_called() + { + var httpClient = DispatchProxy.Create(); + var stateRepo = DispatchProxy.Create(); + var statusService = DispatchProxy.Create(); + var configService = DispatchProxy.Create(); + + var stateRepoProxy = (StateRepositoryProxy)(object)stateRepo; + var statusServiceProxy = (StatusServiceProxy)(object)statusService; + + var definition = new ImportListDefinition + { + Id = 1, + Name = "Test Hardcover Library", + Implementation = nameof(HardcoverLibraryImportList), + Settings = new HardcoverLibraryImportListSettings + { + ApiToken = "test-token", + ImportWantToRead = true, + ImportOwned = true, + MonitorAudiobooks = true, + MonitorEbooks = true, + AudiobookQualityProfileId = 1, + EbookQualityProfileId = 1, + AudiobookMetadataProfileId = 1, + EbookMetadataProfileId = 1, + AudiobookRootFolderPath = "/audiobooks", + EbookRootFolderPath = "/ebooks" + } + }; + + var importList = new HardcoverLibraryImportList( + httpClient, + new Lazy(() => null), + new Lazy(() => null), + new Lazy(() => null), + null, + null, + stateRepo, + statusService, + configService, + null, + LogManager.GetCurrentClassLogger()) + { + Definition = definition + }; + + // 1. Run Fetch() + var items = importList.Fetch(); + + Assert.That(items, Has.Count.GreaterThan(0)); + // Verify state is NOT yet saved to repository + Assert.That(stateRepoProxy.Inserted, Is.Empty, "Cursor should not be inserted during Fetch()"); + Assert.That(stateRepoProxy.Updated, Is.Empty, "Cursor should not be updated during Fetch()"); + Assert.That(statusServiceProxy.SuccessRecorded, Is.Empty, "Success should not be recorded during Fetch()"); + + // 2. Run CommitState() (simulating successful ProcessListItems) + importList.CommitState(); + + Assert.That(stateRepoProxy.Inserted, Has.Count.EqualTo(1), "Cursor should be inserted after CommitState()"); + Assert.That(stateRepoProxy.State.CursorUserBookId, Is.EqualTo(10)); + Assert.That(stateRepoProxy.State.OwnedCursorListBookId, Is.EqualTo(20)); + Assert.That(statusServiceProxy.SuccessRecorded, Contains.Item(definition.Id)); + } + } +} diff --git a/src/Chaptarr.Core.Test/ImportLists/ImportListSyncServiceSearchOnAddFixture.cs b/src/Chaptarr.Core.Test/ImportLists/ImportListSyncServiceSearchOnAddFixture.cs index 8264582e..4ac7a4df 100644 --- a/src/Chaptarr.Core.Test/ImportLists/ImportListSyncServiceSearchOnAddFixture.cs +++ b/src/Chaptarr.Core.Test/ImportLists/ImportListSyncServiceSearchOnAddFixture.cs @@ -149,6 +149,16 @@ args[0] is int id && return Definition; } + if (targetMethod?.Name == nameof(IImportListFactory.GetInstance)) + { + return null; + } + + if (targetMethod?.Name == nameof(IImportListFactory.AutomaticAddEnabled)) + { + return new List(); + } + throw new NotImplementedException($"Test proxy does not implement IImportListFactory.{targetMethod?.Name}"); } } diff --git a/src/NzbDrone.Core/ImportLists/Hardcover/Library/HardcoverLibraryImportList.cs b/src/NzbDrone.Core/ImportLists/Hardcover/Library/HardcoverLibraryImportList.cs index f4f22bcb..f85228f3 100644 --- a/src/NzbDrone.Core/ImportLists/Hardcover/Library/HardcoverLibraryImportList.cs +++ b/src/NzbDrone.Core/ImportLists/Hardcover/Library/HardcoverLibraryImportList.cs @@ -408,6 +408,7 @@ query UserBooksDelta($userId: Int!, $limit: Int!, $updatedAfter: timestamptz!, $ private readonly IRootFolderService _rootFolderService; private readonly IRootFolderSettingsResolver _rootFolderSettingsResolver; private readonly IHardcoverLibraryImportListStateRepository _stateRepository; + private HardcoverLibraryImportListState _pendingState; private bool _useCachedContributorsFallback; public override string Name => "Hardcover Library"; @@ -700,28 +701,42 @@ public override IList Fetch() state.HardcoverUserId = user.Id; state.SettingsSignature = settingsSignature; state.UpdatedAt = now; - - if (state.Id == 0) - { - _stateRepository.Insert(state); - } - else - { - _stateRepository.Update(state); - } + _pendingState = state; + } + else + { + _pendingState = null; } - - _importListStatusService.RecordSuccess(Definition.Id); } catch (Exception ex) { _logger.Warn(ex, "Hardcover library import failed"); + _pendingState = null; _importListStatusService.RecordFailure(Definition.Id); } return CleanupListItems(results); } + public override void CommitState() + { + if (_pendingState != null) + { + if (_pendingState.Id == 0) + { + _stateRepository.Insert(_pendingState); + } + else + { + _stateRepository.Update(_pendingState); + } + + _pendingState = null; + } + + _importListStatusService.RecordSuccess(Definition.Id); + } + protected override IList CleanupListItems(IEnumerable releases) { // Hardcover library sync can include multiple editions of the same book; dedupe by provider IDs, not title. diff --git a/src/NzbDrone.Core/ImportLists/IImportList.cs b/src/NzbDrone.Core/ImportLists/IImportList.cs index 4970c526..9bfd6e4f 100644 --- a/src/NzbDrone.Core/ImportLists/IImportList.cs +++ b/src/NzbDrone.Core/ImportLists/IImportList.cs @@ -10,5 +10,6 @@ public interface IImportList : IProvider ImportListType ListType { get; } TimeSpan MinRefreshInterval { get; } IList Fetch(); + void CommitState(); } } diff --git a/src/NzbDrone.Core/ImportLists/ImportListBase.cs b/src/NzbDrone.Core/ImportLists/ImportListBase.cs index 9d882c2a..30bfe054 100644 --- a/src/NzbDrone.Core/ImportLists/ImportListBase.cs +++ b/src/NzbDrone.Core/ImportLists/ImportListBase.cs @@ -64,6 +64,10 @@ public virtual object RequestAction(string action, IDictionary q public abstract IList Fetch(); + public virtual void CommitState() + { + } + private static object BuildDedupeKey(ImportListItemInfo item) { if (item == null) diff --git a/src/NzbDrone.Core/ImportLists/ImportListSyncService.cs b/src/NzbDrone.Core/ImportLists/ImportListSyncService.cs index 11e27e39..a133c4d2 100644 --- a/src/NzbDrone.Core/ImportLists/ImportListSyncService.cs +++ b/src/NzbDrone.Core/ImportLists/ImportListSyncService.cs @@ -650,18 +650,64 @@ public void MarkExistingAuthor(string providerId) } } - private ImportListLocalLookup BuildImportListLocalLookup() + private ImportListLocalLookup BuildImportListLocalLookup(IEnumerable items = null) { var lookup = new ImportListLocalLookup(); - foreach (var author in _authorService.GetAllAuthors() ?? new List()) + if (items == null) { - lookup.AddAuthor(author); + return lookup; } - foreach (var book in _bookService.GetAllBooks() ?? new List()) + var authorProviderIds = new HashSet(StringComparer.OrdinalIgnoreCase); + var bookProviderIds = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var item in items) + { + if (item == null) + { + continue; + } + + var authorId = GetAuthorProviderId(item); + if (authorId.IsNotNullOrWhiteSpace()) + { + authorProviderIds.Add(authorId.Trim()); + } + + var bookId = GetBookProviderId(item); + if (bookId.IsNotNullOrWhiteSpace()) + { + bookProviderIds.Add(bookId.Trim()); + } + } + + foreach (var authorId in authorProviderIds) + { + var (prefix, rawId) = SplitProviderId(authorId); + prefix ??= "gr"; + + var author = _authorService.FindByProviderId(prefix, authorId) + ?? (rawId.IsNotNullOrWhiteSpace() ? _authorService.FindByProviderId(prefix, rawId) : null); + + if (author != null) + { + lookup.AddAuthor(author); + } + } + + foreach (var bookId in bookProviderIds) { - lookup.AddBook(book); + var (prefix, rawId) = SplitProviderId(bookId); + prefix ??= "gr"; + + var book = _bookService.FindByProviderId(prefix, bookId) + ?? (rawId.IsNotNullOrWhiteSpace() ? _bookService.FindByProviderId(prefix, rawId) : null); + + if (book != null) + { + lookup.AddBook(book); + } } return lookup; @@ -1337,7 +1383,21 @@ private List SyncAll() var listItems = _listFetcherAndParser.Fetch().ToList(); - return ProcessListItems(listItems); + var processed = ProcessListItems(listItems); + + foreach (var importList in enabledImportLists) + { + try + { + importList.CommitState(); + } + catch (Exception e) + { + _logger.Error(e, "Error committing state for Import List {0} ({1})", importList.Name, importList.Definition.Name); + } + } + + return processed; } private List SyncList(ImportListDefinition definition) @@ -1346,7 +1406,19 @@ private List SyncList(ImportListDefinition definition) var listItems = _listFetcherAndParser.FetchSingleList(definition).ToList(); - return ProcessListItems(listItems); + var processed = ProcessListItems(listItems); + + try + { + var importList = _importListFactory.GetInstance(definition); + importList?.CommitState(); + } + catch (Exception e) + { + _logger.Error(e, "Error committing state for Import List {0} ({1})", definition.Name, definition.Id); + } + + return processed; } private List ProcessListItems(List items) @@ -1374,7 +1446,7 @@ private List ProcessListItems(List items) var liveBookLookupHitCache = new Dictionary(StringComparer.OrdinalIgnoreCase); _logger.ProgressInfo("Import list sync: fetched {0} list items; indexing local library", items.Count); - var localLookup = BuildImportListLocalLookup(); + var localLookup = BuildImportListLocalLookup(items); _logger.ProgressInfo("Import list sync: local match index ready ({0} authors, {1} books)", localLookup.AuthorCount, localLookup.BookCount); @@ -1583,7 +1655,7 @@ ImportListDefinition GetImportListDefinition(int importListId) if (pendingAuthorsQueuedEarly.Any()) { _logger.ProgressInfo("Import list sync: refreshing local match index after queueing {0} authors", pendingAuthorsQueuedEarly.Count); - localLookup = BuildImportListLocalLookup(); + localLookup = BuildImportListLocalLookup(items); _logger.ProgressInfo("Import list sync: refreshed local match index ({0} authors, {1} books)", localLookup.AuthorCount, localLookup.BookCount); } @@ -2744,7 +2816,6 @@ private List SyncHardcoverLibrary(bool filterBlockedImportLists) { var hardcoverImportLists = _importListFactory.AutomaticAddEnabled(filterBlockedImportLists) .Where(l => l.Definition.Implementation.EqualsIgnoreCase(nameof(HardcoverLibraryImportList))) - .Select(l => (ImportListDefinition)l.Definition) .ToList(); if (hardcoverImportLists.Empty()) @@ -2756,12 +2827,33 @@ private List SyncHardcoverLibrary(bool filterBlockedImportLists) _logger.ProgressInfo("Starting Hardcover Library Sync"); var listItems = new List(); - foreach (var definition in hardcoverImportLists) + foreach (var importList in hardcoverImportLists) { - listItems.AddRange(_listFetcherAndParser.FetchSingleList(definition)); + try + { + listItems.AddRange(importList.Fetch()); + } + catch (Exception ex) + { + _logger.Error(ex, "Failed to fetch list items for {0}", importList.Definition.Name); + } + } + + var processed = ProcessListItems(listItems); + + foreach (var importList in hardcoverImportLists) + { + try + { + importList.CommitState(); + } + catch (Exception ex) + { + _logger.Error(ex, "Failed to commit state for {0}", importList.Definition.Name); + } } - return ProcessListItems(listItems); + return processed; } } }