From 45d5b1f19ab70fd0421a20e9ed3e69600cc0ecc2 Mon Sep 17 00:00:00 2001 From: Benjamin Tobalt Date: Wed, 9 Sep 2026 00:07:04 -0400 Subject: [PATCH 01/18] Add Grimmory connect with per-media-type library refresh Queues affected Grimmory libraries on import, rename, delete and retag events and drains them via ProcessQueue after files land on disk, replacing the archived branch's in-handler sleep and all-libraries sync task. Book deletes now publish DeleteCompletedEvent after disk cleanup so queued notification work (Grimmory, Plex) drains at the right time. --- .../Notifications/Grimmory/GrimmoryFixture.cs | 238 ++++++++++++++++ .../Grimmory/GrimmoryProxyFixture.cs | 201 ++++++++++++++ .../MediaFiles/MediaFileDeletionService.cs | 4 + .../Notifications/Grimmory/Grimmory.cs | 255 ++++++++++++++++++ .../Notifications/Grimmory/GrimmoryProxy.cs | 196 ++++++++++++++ .../Grimmory/GrimmorySettings.cs | 48 ++++ 6 files changed, 942 insertions(+) create mode 100644 src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs create mode 100644 src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryProxyFixture.cs create mode 100644 src/NzbDrone.Core/Notifications/Grimmory/Grimmory.cs create mode 100644 src/NzbDrone.Core/Notifications/Grimmory/GrimmoryProxy.cs create mode 100644 src/NzbDrone.Core/Notifications/Grimmory/GrimmorySettings.cs diff --git a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs new file mode 100644 index 00000000..4ffc04d3 --- /dev/null +++ b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs @@ -0,0 +1,238 @@ +using System; +using System.Collections.Generic; +using FluentValidation.Results; +using NLog; +using NUnit.Framework; +using NzbDrone.Common.Cache; +using NzbDrone.Core.Books; +using NzbDrone.Core.MediaFiles; +using NzbDrone.Core.Notifications; +using NzbDrone.Core.Notifications.Grimmory; +using NzbDrone.Core.Qualities; + +namespace Chaptarr.Core.Test.Notifications.Grimmory +{ + [TestFixture] + public class GrimmoryFixture + { + private const long EbookLibraryId = 10; + private const long AudiobookLibraryId = 20; + + [Test] + public void should_not_refresh_at_event_time_and_refresh_on_process_queue() + { + var proxy = new FakeGrimmoryProxy(); + var subject = CreateSubject(proxy); + + subject.OnReleaseImport(BuildImport(BookMediaType.Ebook)); + + Assert.That(proxy.RefreshedLibraryIds, Is.Empty); + Assert.That(subject.HasPendingQueue, Is.True); + + subject.ProcessQueue(); + + Assert.That(proxy.RefreshedLibraryIds, Is.EqualTo(new List { EbookLibraryId })); + Assert.That(subject.HasPendingQueue, Is.False); + } + + [Test] + public void should_dedupe_multiple_events_into_single_refresh() + { + var proxy = new FakeGrimmoryProxy(); + var subject = CreateSubject(proxy); + + subject.OnReleaseImport(BuildImport(BookMediaType.Ebook)); + subject.OnReleaseImport(BuildImport(BookMediaType.Ebook)); + subject.OnBookFileDelete(new BookFileDeleteMessage + { + Book = new Book { MediaType = BookMediaType.Ebook }, + BookFile = new BookFile { MediaType = "ebook" } + }); + + subject.ProcessQueue(); + + Assert.That(proxy.RefreshedLibraryIds, Is.EqualTo(new List { EbookLibraryId })); + } + + [Test] + public void should_route_audiobook_events_to_audiobook_library() + { + var proxy = new FakeGrimmoryProxy(); + var subject = CreateSubject(proxy); + + subject.OnReleaseImport(BuildImport(BookMediaType.Audiobook)); + subject.ProcessQueue(); + + Assert.That(proxy.RefreshedLibraryIds, Is.EqualTo(new List { AudiobookLibraryId })); + } + + [Test] + public void should_skip_events_for_unconfigured_library() + { + var proxy = new FakeGrimmoryProxy(); + var subject = CreateSubject(proxy, audiobookLibraryId: 0); + + subject.OnReleaseImport(BuildImport(BookMediaType.Audiobook)); + + Assert.That(subject.HasPendingQueue, Is.False); + + subject.ProcessQueue(); + + Assert.That(proxy.RefreshedLibraryIds, Is.Empty); + } + + [Test] + public void should_refresh_both_libraries_for_mixed_renames() + { + var proxy = new FakeGrimmoryProxy(); + var subject = CreateSubject(proxy); + + subject.OnRename(new Author { Name = "Robin Hobb" }, new List + { + new RenamedBookFile { BookFile = new BookFile { MediaType = "ebook" } }, + new RenamedBookFile { BookFile = new BookFile { MediaType = "audiobook" } } + }); + + subject.ProcessQueue(); + + Assert.That(proxy.RefreshedLibraryIds, Is.EquivalentTo(new List { EbookLibraryId, AudiobookLibraryId })); + } + + [Test] + public void should_determine_media_type_from_quality_when_not_set_on_file() + { + var proxy = new FakeGrimmoryProxy(); + var subject = CreateSubject(proxy); + + subject.OnRename(new Author { Name = "Robin Hobb" }, new List + { + new RenamedBookFile { BookFile = new BookFile { MediaType = null, Quality = new QualityModel(Quality.EPUB) } } + }); + + subject.ProcessQueue(); + + Assert.That(proxy.RefreshedLibraryIds, Is.EqualTo(new List { EbookLibraryId })); + } + + [Test] + public void should_not_queue_book_delete_without_deleted_files() + { + var proxy = new FakeGrimmoryProxy(); + var subject = CreateSubject(proxy); + + subject.OnBookDelete(new BookDeleteMessage(new Book { MediaType = BookMediaType.Ebook }, false)); + + Assert.That(subject.HasPendingQueue, Is.False); + } + + [Test] + public void should_queue_both_configured_libraries_on_author_delete() + { + var proxy = new FakeGrimmoryProxy(); + var subject = CreateSubject(proxy); + + subject.OnAuthorDelete(new AuthorDeleteMessage(new Author { Name = "Robin Hobb" }, true)); + subject.ProcessQueue(); + + Assert.That(proxy.RefreshedLibraryIds, Is.EquivalentTo(new List { EbookLibraryId, AudiobookLibraryId })); + } + + [Test] + public void should_throw_and_still_refresh_remaining_when_one_library_fails() + { + var proxy = new FakeGrimmoryProxy(); + proxy.FailingLibraryIds.Add(EbookLibraryId); + + var subject = CreateSubject(proxy); + + subject.OnReleaseImport(BuildImport(BookMediaType.Ebook)); + subject.OnReleaseImport(BuildImport(BookMediaType.Audiobook)); + + Assert.Throws(() => subject.ProcessQueue()); + Assert.That(proxy.RefreshedLibraryIds, Is.EqualTo(new List { AudiobookLibraryId })); + } + + [Test] + public void should_queue_again_after_process_queue_failure() + { + var proxy = new FakeGrimmoryProxy(); + proxy.FailingLibraryIds.Add(EbookLibraryId); + + var subject = CreateSubject(proxy); + + subject.OnReleaseImport(BuildImport(BookMediaType.Ebook)); + Assert.Throws(() => subject.ProcessQueue()); + + proxy.FailingLibraryIds.Clear(); + + subject.OnReleaseImport(BuildImport(BookMediaType.Ebook)); + subject.ProcessQueue(); + + Assert.That(proxy.RefreshedLibraryIds, Is.EqualTo(new List { EbookLibraryId })); + } + + private static BookDownloadMessage BuildImport(BookMediaType mediaType) + { + var fileMediaType = mediaType == BookMediaType.Ebook ? "ebook" : "audiobook"; + + return new BookDownloadMessage + { + Author = new Author { Name = "Robin Hobb" }, + Book = new Book { Title = "Assassin's Apprentice", MediaType = mediaType }, + BookFiles = new List + { + new BookFile { MediaType = fileMediaType } + } + }; + } + + private static NzbDrone.Core.Notifications.Grimmory.Grimmory CreateSubject(FakeGrimmoryProxy proxy, + long ebookLibraryId = EbookLibraryId, + long audiobookLibraryId = AudiobookLibraryId) + { + var settings = new GrimmorySettings + { + Url = "http://grimmory:6060", + Username = "chaptarr", + Password = "secret", + EbookLibraryId = ebookLibraryId, + AudiobookLibraryId = audiobookLibraryId + }; + + return new NzbDrone.Core.Notifications.Grimmory.Grimmory( + proxy, + new CacheManager(), + LogManager.GetLogger("GrimmoryFixture")) + { + Definition = new NotificationDefinition { Settings = settings } + }; + } + + private class FakeGrimmoryProxy : IGrimmoryProxy + { + public List Libraries { get; set; } = new List(); + public List RefreshedLibraryIds { get; } = new List(); + public HashSet FailingLibraryIds { get; } = new HashSet(); + + public List GetLibraries(GrimmorySettings settings) + { + return Libraries; + } + + public void RefreshLibrary(GrimmorySettings settings, long libraryId) + { + if (FailingLibraryIds.Contains(libraryId)) + { + throw new InvalidOperationException($"Refresh failed for library {libraryId}"); + } + + RefreshedLibraryIds.Add(libraryId); + } + + public ValidationFailure Test(GrimmorySettings settings) + { + return null; + } + } + } +} diff --git a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryProxyFixture.cs b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryProxyFixture.cs new file mode 100644 index 00000000..ed6ab938 --- /dev/null +++ b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryProxyFixture.cs @@ -0,0 +1,201 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using NLog; +using NUnit.Framework; +using NzbDrone.Common.Cache; +using NzbDrone.Common.Http; +using NzbDrone.Core.Notifications.Grimmory; + +namespace Chaptarr.Core.Test.Notifications.Grimmory +{ + [TestFixture] + public class GrimmoryProxyFixture + { + private const string LibrariesJson = "[{\"id\":10,\"name\":\"Ebooks\",\"allowedFormats\":[\"EPUB\",\"PDF\"]},{\"id\":20,\"name\":\"Audiobooks\",\"allowedFormats\":[\"AUDIOBOOK\"]}]"; + + [Test] + public void should_login_and_fetch_libraries_with_bearer_token() + { + var httpClient = new ScriptedHttpClient { ValidTokens = { "token1" } }; + var proxy = CreateProxy(httpClient); + + var libraries = proxy.GetLibraries(BuildSettings()); + + Assert.That(libraries, Has.Count.EqualTo(2)); + Assert.That(libraries[0].Id, Is.EqualTo(10)); + Assert.That(libraries[0].Name, Is.EqualTo("Ebooks")); + Assert.That(libraries[0].AllowedFormats, Is.EqualTo(new List { "EPUB", "PDF" })); + Assert.That(httpClient.LoginCount, Is.EqualTo(1)); + + var libraryRequest = httpClient.Requests.Last(); + Assert.That(libraryRequest.Url.ToString(), Does.EndWith("/api/v1/libraries")); + Assert.That(libraryRequest.Headers["Authorization"], Is.EqualTo("Bearer token1")); + } + + [Test] + public void should_reuse_cached_token_across_calls() + { + var httpClient = new ScriptedHttpClient { ValidTokens = { "token1" } }; + var proxy = CreateProxy(httpClient); + var settings = BuildSettings(); + + proxy.GetLibraries(settings); + proxy.RefreshLibrary(settings, 10); + + Assert.That(httpClient.LoginCount, Is.EqualTo(1)); + } + + [Test] + public void should_relogin_once_when_token_is_rejected() + { + var httpClient = new ScriptedHttpClient { ValidTokens = { "token2" } }; + var proxy = CreateProxy(httpClient); + + var libraries = proxy.GetLibraries(BuildSettings()); + + Assert.That(libraries, Has.Count.EqualTo(2)); + Assert.That(httpClient.LoginCount, Is.EqualTo(2)); + } + + [Test] + public void should_throw_authentication_exception_when_relogin_still_rejected() + { + var httpClient = new ScriptedHttpClient(); + var proxy = CreateProxy(httpClient); + + Assert.Throws(() => proxy.GetLibraries(BuildSettings())); + Assert.That(httpClient.LoginCount, Is.EqualTo(2)); + } + + [Test] + public void should_throw_authentication_exception_when_login_is_rejected() + { + var httpClient = new ScriptedHttpClient { RejectLogin = true }; + var proxy = CreateProxy(httpClient); + + Assert.Throws(() => proxy.GetLibraries(BuildSettings())); + } + + [Test] + public void should_send_put_to_refresh_endpoint() + { + var httpClient = new ScriptedHttpClient { ValidTokens = { "token1" } }; + var proxy = CreateProxy(httpClient); + + proxy.RefreshLibrary(BuildSettings(), 20); + + var refreshRequest = httpClient.Requests.Last(); + Assert.That(refreshRequest.Method, Is.EqualTo(HttpMethod.Put)); + Assert.That(refreshRequest.Url.ToString(), Does.EndWith("/api/v1/libraries/20/refresh")); + } + + [Test] + public void test_should_fail_when_configured_library_is_missing() + { + var httpClient = new ScriptedHttpClient { ValidTokens = { "token1" } }; + var proxy = CreateProxy(httpClient); + + var settings = BuildSettings(); + settings.EbookLibraryId = 99; + + var failure = proxy.Test(settings); + + Assert.That(failure, Is.Not.Null); + Assert.That(failure.PropertyName, Is.EqualTo(nameof(GrimmorySettings.EbookLibraryId))); + } + + [Test] + public void test_should_pass_when_configured_libraries_exist() + { + var httpClient = new ScriptedHttpClient { ValidTokens = { "token1" } }; + var proxy = CreateProxy(httpClient); + + Assert.That(proxy.Test(BuildSettings()), Is.Null); + } + + private static GrimmoryProxy CreateProxy(ScriptedHttpClient httpClient) + { + return new GrimmoryProxy(httpClient, new CacheManager(), LogManager.GetLogger("GrimmoryProxyFixture")); + } + + private static GrimmorySettings BuildSettings() + { + return new GrimmorySettings + { + Url = "http://grimmory:6060", + Username = "chaptarr", + Password = "secret", + EbookLibraryId = 10, + AudiobookLibraryId = 20 + }; + } + + private class ScriptedHttpClient : IHttpClient + { + public List Requests { get; } = new List(); + public HashSet ValidTokens { get; } = new HashSet(); + public bool RejectLogin { get; set; } + public int LoginCount { get; private set; } + + public HttpResponse Execute(HttpRequest request) + { + Requests.Add(request); + + var url = request.Url.ToString(); + var headers = new HttpHeader { ContentType = "application/json" }; + + if (url.EndsWith("/api/v1/auth/login")) + { + LoginCount++; + + if (RejectLogin) + { + return new HttpResponse(request, headers, string.Empty, HttpStatusCode.Unauthorized); + } + + // The first login hands out token1, the second token2, and so on. Which of + // them the server still accepts is controlled per-test via ValidTokens. + return new HttpResponse(request, headers, $"{{\"accessToken\":\"token{LoginCount}\"}}"); + } + + var authorization = request.Headers["Authorization"]; + + if (authorization == null || !ValidTokens.Contains(authorization.Replace("Bearer ", string.Empty))) + { + return new HttpResponse(request, headers, string.Empty, HttpStatusCode.Unauthorized); + } + + if (url.EndsWith("/api/v1/libraries")) + { + return new HttpResponse(request, headers, LibrariesJson); + } + + if (url.Contains("/api/v1/libraries/") && url.EndsWith("/refresh")) + { + return new HttpResponse(request, headers, string.Empty, HttpStatusCode.NoContent); + } + + return new HttpResponse(request, headers, string.Empty, HttpStatusCode.NotFound); + } + + public HttpResponse Get(HttpRequest request) => Execute(request); + + public void DownloadFile(string url, string fileName, string userAgent = null) => throw new NotImplementedException(); + public HttpResponse Get(HttpRequest request) where T : new() => throw new NotImplementedException(); + public HttpResponse Head(HttpRequest request) => throw new NotImplementedException(); + public HttpResponse Post(HttpRequest request) => throw new NotImplementedException(); + public HttpResponse Post(HttpRequest request) where T : new() => throw new NotImplementedException(); + public Task ExecuteAsync(HttpRequest request) => throw new NotImplementedException(); + public Task DownloadFileAsync(string url, string fileName, string userAgent = null) => throw new NotImplementedException(); + public Task GetAsync(HttpRequest request) => throw new NotImplementedException(); + public Task> GetAsync(HttpRequest request) where T : new() => throw new NotImplementedException(); + public Task HeadAsync(HttpRequest request) => throw new NotImplementedException(); + public Task PostAsync(HttpRequest request) => throw new NotImplementedException(); + public Task> PostAsync(HttpRequest request) where T : new() => throw new NotImplementedException(); + } + } +} diff --git a/src/NzbDrone.Core/MediaFiles/MediaFileDeletionService.cs b/src/NzbDrone.Core/MediaFiles/MediaFileDeletionService.cs index 35adedf4..f87ba438 100644 --- a/src/NzbDrone.Core/MediaFiles/MediaFileDeletionService.cs +++ b/src/NzbDrone.Core/MediaFiles/MediaFileDeletionService.cs @@ -305,6 +305,10 @@ public void HandleAsync(BookDeletedEvent message) { CleanupEmptyFolders(author, folder); } + + // Notification providers queue work on OnBookDelete and drain it when this event + // signals the files are actually off the disk; author deletes already publish it. + _eventAggregator.PublishEvent(new DeleteCompletedEvent()); } private static void CollectFolder(List folders, string folder) diff --git a/src/NzbDrone.Core/Notifications/Grimmory/Grimmory.cs b/src/NzbDrone.Core/Notifications/Grimmory/Grimmory.cs new file mode 100644 index 00000000..d79eb81f --- /dev/null +++ b/src/NzbDrone.Core/Notifications/Grimmory/Grimmory.cs @@ -0,0 +1,255 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using FluentValidation.Results; +using NLog; +using NzbDrone.Common.Cache; +using NzbDrone.Common.Extensions; +using NzbDrone.Core.Books; +using NzbDrone.Core.MediaFiles; + +namespace NzbDrone.Core.Notifications.Grimmory +{ + public class Grimmory : NotificationBase + { + private readonly IGrimmoryProxy _proxy; + private readonly Logger _logger; + private readonly ICached _pendingLibrariesCache; + + public Grimmory(IGrimmoryProxy proxy, ICacheManager cacheManager, Logger logger) + { + _proxy = proxy; + _logger = logger; + _pendingLibrariesCache = cacheManager.GetRollingCache(GetType(), "pendingLibraries", TimeSpan.FromDays(1)); + } + + public override string Name => "Grimmory"; + public override string Link => "https://github.com/grimmory-tools/grimmory"; + + private class GrimmoryUpdateQueue + { + public HashSet PendingLibraries { get; } = new HashSet(); + public bool Refreshing { get; set; } + } + + public override bool HasPendingQueue + { + get + { + var queue = _pendingLibrariesCache.Find(QueueKey); + + if (queue == null) + { + return false; + } + + lock (queue) + { + return !queue.Refreshing && queue.PendingLibraries.Any(); + } + } + } + + public override void OnReleaseImport(BookDownloadMessage message) + { + if (message.BookFiles == null || message.BookFiles.Empty()) + { + return; + } + + QueueRefresh(GetLibraryId(message.Book, message.BookFiles.FirstOrDefault()), "import"); + } + + public override void OnRename(Author author, List renamedFiles) + { + foreach (var renamedFile in renamedFiles ?? new List()) + { + if (renamedFile?.BookFile != null) + { + QueueRefresh(GetLibraryId(null, renamedFile.BookFile), "rename"); + } + } + } + + public override void OnAuthorDelete(AuthorDeleteMessage message) + { + if (message.DeletedFiles) + { + QueueRefresh(Settings.EbookLibraryId, "author delete"); + QueueRefresh(Settings.AudiobookLibraryId, "author delete"); + } + } + + public override void OnBookDelete(BookDeleteMessage message) + { + if (message.DeletedFiles) + { + QueueRefresh(GetLibraryId(message.Book, null), "book delete"); + } + } + + public override void OnBookFileDelete(BookFileDeleteMessage message) + { + QueueRefresh(GetLibraryId(message.Book, message.BookFile), "file delete"); + } + + public override void OnBookRetag(BookRetagMessage message) + { + QueueRefresh(GetLibraryId(message.Book, message.BookFile), "retag"); + } + + public override void ProcessQueue() + { + var queue = _pendingLibrariesCache.Find(QueueKey); + + if (queue == null) + { + return; + } + + lock (queue) + { + if (queue.Refreshing) + { + return; + } + + queue.Refreshing = true; + } + + try + { + while (true) + { + List libraryIds; + + lock (queue) + { + if (queue.PendingLibraries.Empty()) + { + queue.Refreshing = false; + return; + } + + libraryIds = queue.PendingLibraries.ToList(); + queue.PendingLibraries.Clear(); + } + + var failed = new List(); + + foreach (var libraryId in libraryIds) + { + try + { + _proxy.RefreshLibrary(Settings, libraryId); + } + catch (Exception ex) + { + _logger.Warn(ex, "Failed to trigger Grimmory refresh for library {0}", libraryId); + failed.Add(libraryId); + } + } + + if (failed.Any()) + { + throw new InvalidOperationException($"Failed to trigger Grimmory refresh for libraries: {string.Join(", ", failed)}"); + } + } + } + catch + { + lock (queue) + { + queue.Refreshing = false; + } + + throw; + } + } + + public override ValidationResult Test() + { + var failures = new List(); + + failures.AddIfNotNull(_proxy.Test(Settings)); + + return new ValidationResult(failures); + } + + public override object RequestAction(string action, IDictionary query) + { + if (action == "getLibraries") + { + if (Settings.Url.IsNullOrWhiteSpace() || Settings.Username.IsNullOrWhiteSpace() || Settings.Password.IsNullOrWhiteSpace()) + { + return new { options = new List() }; + } + + try + { + var libraries = _proxy.GetLibraries(Settings); + + return new + { + options = libraries + .OrderBy(l => l.Name, StringComparer.InvariantCultureIgnoreCase) + .Select(l => new + { + Value = l.Id, + Name = l.Name, + Hint = l.AllowedFormats?.Any() == true ? string.Join(", ", l.AllowedFormats) : "All formats" + }) + }; + } + catch (Exception ex) + { + _logger.Error(ex, "Failed to retrieve libraries from Grimmory"); + return new { options = new List() }; + } + } + + return new { }; + } + + private string QueueKey => $"{Settings.Url}:{Settings.Username}"; + + private void QueueRefresh(long libraryId, string reason) + { + if (libraryId <= 0) + { + return; + } + + _logger.Debug("Grimmory: queueing refresh of library {0} after {1}", libraryId, reason); + + var queue = _pendingLibrariesCache.Get(QueueKey, () => new GrimmoryUpdateQueue()); + + lock (queue) + { + queue.PendingLibraries.Add(libraryId); + } + } + + private long GetLibraryId(Book book, BookFile bookFile) + { + if (book != null) + { + return book.MediaType == BookMediaType.Ebook ? Settings.EbookLibraryId : Settings.AudiobookLibraryId; + } + + var mediaType = bookFile?.MediaType; + + if (mediaType.IsNullOrWhiteSpace() && bookFile?.Quality != null) + { + mediaType = BookFile.DetermineMediaType(bookFile.Quality); + } + + return mediaType switch + { + "ebook" => Settings.EbookLibraryId, + "audiobook" => Settings.AudiobookLibraryId, + _ => 0 + }; + } + } +} diff --git a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryProxy.cs b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryProxy.cs new file mode 100644 index 00000000..0eb00e37 --- /dev/null +++ b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryProxy.cs @@ -0,0 +1,196 @@ +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using FluentValidation.Results; +using Newtonsoft.Json; +using NLog; +using NzbDrone.Common.Cache; +using NzbDrone.Common.Extensions; +using NzbDrone.Common.Http; +using NzbDrone.Common.Serializer; + +namespace NzbDrone.Core.Notifications.Grimmory +{ + public interface IGrimmoryProxy + { + List GetLibraries(GrimmorySettings settings); + void RefreshLibrary(GrimmorySettings settings, long libraryId); + ValidationFailure Test(GrimmorySettings settings); + } + + public class GrimmoryProxy : IGrimmoryProxy + { + private static readonly TimeSpan TokenCacheDuration = TimeSpan.FromMinutes(30); + + private readonly IHttpClient _httpClient; + private readonly ICached _tokenCache; + private readonly Logger _logger; + + public GrimmoryProxy(IHttpClient httpClient, ICacheManager cacheManager, Logger logger) + { + _httpClient = httpClient; + _tokenCache = cacheManager.GetCache(GetType(), "tokens"); + _logger = logger; + } + + public List GetLibraries(GrimmorySettings settings) + { + var response = ExecuteWithAuth(settings, token => + { + var request = BuildRequest(settings, "api/v1/libraries", token).Build(); + return _httpClient.Get(request); + }); + + return Json.Deserialize>(response.Content) ?? new List(); + } + + public void RefreshLibrary(GrimmorySettings settings, long libraryId) + { + ExecuteWithAuth(settings, token => + { + var request = BuildRequest(settings, $"api/v1/libraries/{libraryId}/refresh", token).Build(); + request.Method = HttpMethod.Put; + return _httpClient.Execute(request); + }); + + _logger.Debug("Triggered Grimmory refresh for library {0}", libraryId); + } + + public ValidationFailure Test(GrimmorySettings settings) + { + try + { + var libraries = GetLibraries(settings); + + if (settings.EbookLibraryId > 0 && !libraries.Exists(l => l.Id == settings.EbookLibraryId)) + { + return new ValidationFailure(nameof(GrimmorySettings.EbookLibraryId), "The selected ebook library was not found in Grimmory"); + } + + if (settings.AudiobookLibraryId > 0 && !libraries.Exists(l => l.Id == settings.AudiobookLibraryId)) + { + return new ValidationFailure(nameof(GrimmorySettings.AudiobookLibraryId), "The selected audiobook library was not found in Grimmory"); + } + } + catch (GrimmoryAuthenticationException) + { + return new ValidationFailure(nameof(GrimmorySettings.Username), "Authentication failed, check the username and password"); + } + catch (Exception ex) + { + _logger.Error(ex, "Unable to connect to Grimmory"); + return new ValidationFailure(nameof(GrimmorySettings.Url), "Unable to connect: " + ex.Message); + } + + return null; + } + + private HttpResponse ExecuteWithAuth(GrimmorySettings settings, Func action) + { + var token = GetAccessToken(settings, false); + var response = action(token); + + if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden) + { + token = GetAccessToken(settings, true); + response = action(token); + } + + if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden) + { + throw new GrimmoryAuthenticationException("Grimmory rejected the configured credentials"); + } + + if ((int)response.StatusCode >= 400) + { + throw new HttpException(response); + } + + return response; + } + + private string GetAccessToken(GrimmorySettings settings, bool forceRefresh) + { + var cacheKey = $"{settings.Url}:{settings.Username}"; + + if (forceRefresh) + { + _tokenCache.Remove(cacheKey); + } + + return _tokenCache.Get(cacheKey, () => Login(settings), TokenCacheDuration); + } + + private string Login(GrimmorySettings settings) + { + var request = new HttpRequestBuilder(HttpUri.CombinePath(settings.Url, "api/v1/auth/login")) + .Accept(HttpAccept.Json) + .Build(); + + request.Method = HttpMethod.Post; + request.Headers.ContentType = "application/json"; + request.SuppressHttpError = true; + request.SetContent(new { username = settings.Username, password = settings.Password }.ToJson()); + + var response = _httpClient.Execute(request); + + if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden) + { + throw new GrimmoryAuthenticationException("Grimmory rejected the configured credentials"); + } + + if ((int)response.StatusCode >= 400) + { + throw new HttpException(response); + } + + var tokenResponse = Json.Deserialize(response.Content); + + if (tokenResponse?.AccessToken.IsNullOrWhiteSpace() != false) + { + throw new GrimmoryAuthenticationException("Grimmory did not return an access token"); + } + + return tokenResponse.AccessToken; + } + + private static HttpRequestBuilder BuildRequest(GrimmorySettings settings, string relativePath, string token) + { + // Status codes are handled in ExecuteWithAuth so a 401/403 can trigger a re-login + // instead of surfacing as an HttpException from the client. + return new HttpRequestBuilder(HttpUri.CombinePath(settings.Url, relativePath)) + { + SuppressHttpError = true + } + .Accept(HttpAccept.Json) + .SetHeader("Authorization", $"Bearer {token}"); + } + + private class GrimmoryTokenResponse + { + [JsonProperty("accessToken")] + public string AccessToken { get; set; } + } + } + + public class GrimmoryLibrary + { + [JsonProperty("id")] + public long Id { get; set; } + + [JsonProperty("name")] + public string Name { get; set; } + + [JsonProperty("allowedFormats")] + public List AllowedFormats { get; set; } + } + + public class GrimmoryAuthenticationException : Exception + { + public GrimmoryAuthenticationException(string message) + : base(message) + { + } + } +} diff --git a/src/NzbDrone.Core/Notifications/Grimmory/GrimmorySettings.cs b/src/NzbDrone.Core/Notifications/Grimmory/GrimmorySettings.cs new file mode 100644 index 00000000..a865a5b0 --- /dev/null +++ b/src/NzbDrone.Core/Notifications/Grimmory/GrimmorySettings.cs @@ -0,0 +1,48 @@ +using FluentValidation; +using NzbDrone.Common.Extensions; +using NzbDrone.Core.Annotations; +using NzbDrone.Core.ThingiProvider; +using NzbDrone.Core.Validation; + +namespace NzbDrone.Core.Notifications.Grimmory +{ + public class GrimmorySettingsValidator : AbstractValidator + { + public GrimmorySettingsValidator() + { + RuleFor(c => c.Url).NotEmpty().WithMessage("URL cannot be empty"); + RuleFor(c => c.Url).IsValidUrl().When(c => c.Url.IsNotNullOrWhiteSpace()); + RuleFor(c => c.Username).NotEmpty().WithMessage("Username is required"); + RuleFor(c => c.Password).NotEmpty().WithMessage("Password is required"); + RuleFor(c => c.EbookLibraryId) + .GreaterThan(0) + .When(c => c.AudiobookLibraryId <= 0) + .WithMessage("At least one library is required"); + } + } + + public class GrimmorySettings : IProviderConfig + { + private static readonly GrimmorySettingsValidator Validator = new GrimmorySettingsValidator(); + + [FieldDefinition(0, Label = "URL", HelpText = "Grimmory URL, including http(s):// and port, e.g. http://grimmory:6060. Grimmory must see the same files as Chaptarr (shared or identically mounted storage)")] + public string Url { get; set; } + + [FieldDefinition(1, Label = "Username", Privacy = PrivacyLevel.UserName, HelpText = "Grimmory user with permission to manage libraries")] + public string Username { get; set; } + + [FieldDefinition(2, Label = "Password", Type = FieldType.Password, Privacy = PrivacyLevel.Password)] + public string Password { get; set; } + + [FieldDefinition(3, Label = "Ebook Library", Type = FieldType.Select, SelectOptionsProviderAction = "getLibraries", HelpText = "Grimmory library to refresh when Chaptarr imports, renames or deletes ebook files. Leave unset to ignore ebooks")] + public long EbookLibraryId { get; set; } + + [FieldDefinition(4, Label = "Audiobook Library", Type = FieldType.Select, SelectOptionsProviderAction = "getLibraries", HelpText = "Grimmory library to refresh when Chaptarr imports, renames or deletes audiobook files. Leave unset to ignore audiobooks")] + public long AudiobookLibraryId { get; set; } + + public NzbDroneValidationResult Validate() + { + return new NzbDroneValidationResult(Validator.Validate(this)); + } + } +} From 28b5d2fd39dc4b6f1164f8f3b961d05aa9778f0c Mon Sep 17 00:00:00 2001 From: Benjamin Tobalt Date: Wed, 9 Sep 2026 00:27:45 -0400 Subject: [PATCH 02/18] Suppress unfixed Microsoft.Build.Tasks.Git audit advisory so restores build again --- src/Directory.Build.props | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Directory.Build.props b/src/Directory.Build.props index d10f02a7..e5b6ad65 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -108,6 +108,13 @@ + + + + + From 4f8cc83f54b7cf9c07b7435024cd1894e2d62155 Mon Sep 17 00:00:00 2001 From: Benjamin Tobalt Date: Wed, 9 Sep 2026 02:05:32 -0400 Subject: [PATCH 03/18] Add opt-in Grimmory metadata and cover push with manual push dialog Two connector toggles push Chaptarr's metadata (locking pushed fields) and cover to the matched Grimmory book on import, retag, and cover updates. A Grimmory Push toolbar button on book details and a bulk button in the book editor open a field-picker dialog driving the PushGrimmoryMetadata command; both disable without files on disk. A poller forwards edits made in Grimmory (audit log for metadata, cover stamps for covers, own-user echoes excluded) to any connection implementing the new IExternalLibraryEditTarget seam, keeping this branch mergeable without the ABS/CCS branches. --- frontend/src/Book/Details/BookDetails.js | 50 ++- .../src/Book/Details/BookDetailsConnector.js | 56 ++- frontend/src/Book/Editor/BookEditorFooter.js | 72 +++- frontend/src/Commands/commandNames.js | 1 + frontend/src/Grimmory/GrimmoryPushModal.js | 34 ++ .../src/Grimmory/GrimmoryPushModalContent.css | 29 ++ .../GrimmoryPushModalContent.css.d.ts | 11 + .../src/Grimmory/GrimmoryPushModalContent.js | 129 ++++++ .../Notifications/Grimmory/GrimmoryFixture.cs | 19 + .../GrimmoryLibraryChangeForwarderFixture.cs | 286 +++++++++++++ .../Grimmory/GrimmoryProxyFixture.cs | 68 ++++ .../Grimmory/GrimmoryPushServiceFixture.cs | 329 +++++++++++++++ src/NzbDrone.Core/Localization/Core/en.json | 4 + .../Notifications/ExternalLibraryEdits.cs | 35 ++ .../Notifications/Grimmory/Grimmory.cs | 31 +- .../GrimmoryLibraryChangeForwarder.cs | 340 ++++++++++++++++ .../Notifications/Grimmory/GrimmoryProxy.cs | 294 ++++++++++++++ .../Grimmory/GrimmoryPushService.cs | 383 ++++++++++++++++++ .../Grimmory/GrimmorySettings.cs | 9 + .../Grimmory/PushGrimmoryMetadataCommand.cs | 18 + 20 files changed, 2192 insertions(+), 6 deletions(-) create mode 100644 frontend/src/Grimmory/GrimmoryPushModal.js create mode 100644 frontend/src/Grimmory/GrimmoryPushModalContent.css create mode 100644 frontend/src/Grimmory/GrimmoryPushModalContent.css.d.ts create mode 100644 frontend/src/Grimmory/GrimmoryPushModalContent.js create mode 100644 src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs create mode 100644 src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs create mode 100644 src/NzbDrone.Core/Notifications/ExternalLibraryEdits.cs create mode 100644 src/NzbDrone.Core/Notifications/Grimmory/GrimmoryLibraryChangeForwarder.cs create mode 100644 src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs create mode 100644 src/NzbDrone.Core/Notifications/Grimmory/PushGrimmoryMetadataCommand.cs diff --git a/frontend/src/Book/Details/BookDetails.js b/frontend/src/Book/Details/BookDetails.js index b9521369..da4e4af1 100644 --- a/frontend/src/Book/Details/BookDetails.js +++ b/frontend/src/Book/Details/BookDetails.js @@ -1,10 +1,11 @@ import PropTypes from 'prop-types'; -import React, { Component } from 'react'; +import React, { Component, Fragment } from 'react'; import { Tab, TabList, TabPanel, Tabs } from 'react-tabs'; import AuthorHistoryTable from 'Author/History/AuthorHistoryTable'; import DeleteBookModal from 'Book/Delete/DeleteBookModal'; import EditBookModalConnector from 'Book/Edit/EditBookModalConnector'; import BookFileEditorTable from 'BookFile/Editor/BookFileEditorTable'; +import GrimmoryPushModal from 'Grimmory/GrimmoryPushModal'; import IconButton from 'Components/Link/IconButton'; import LoadingIndicator from 'Components/Loading/LoadingIndicator'; import PageContent from 'Components/Page/PageContent'; @@ -36,6 +37,7 @@ class BookDetails extends Component { this.state = { isOrganizeModalOpen: false, isRetagModalOpen: false, + isGrimmoryPushModalOpen: false, isEditBookModalOpen: false, isDeleteBookModalOpen: false, selectedTabIndex: 0 @@ -74,6 +76,19 @@ class BookDetails extends Component { this.setState({ isRetagModalOpen: false }); }; + onGrimmoryPushPress = () => { + this.setState({ isGrimmoryPushModalOpen: true }); + }; + + onGrimmoryPushModalClose = () => { + this.setState({ isGrimmoryPushModalOpen: false }); + }; + + onGrimmoryPushConfirmed = (fields) => { + this.setState({ isGrimmoryPushModalOpen: false }); + this.props.onPushToGrimmoryPress(fields); + }; + onEditBookPress = () => { this.setState({ isEditBookModalOpen: true }); }; @@ -117,6 +132,9 @@ class BookDetails extends Component { nextBook, hasBookNavigation, isSearching, + showPushToGrimmory, + isPushingToGrimmory, + grimmoryPreview, onRefreshPress, onSearchPress, statistics = {} @@ -129,6 +147,7 @@ class BookDetails extends Component { const { isOrganizeModalOpen, isRetagModalOpen, + isGrimmoryPushModalOpen, isEditBookModalOpen, isDeleteBookModalOpen, selectedTabIndex @@ -172,6 +191,23 @@ class BookDetails extends Component { + { + showPushToGrimmory ? + + + + + : + null + } + + + ); @@ -411,6 +455,10 @@ BookDetails.propTypes = { nextBook: PropTypes.object, hasBookNavigation: PropTypes.bool, isSmallScreen: PropTypes.bool.isRequired, + showPushToGrimmory: PropTypes.bool, + isPushingToGrimmory: PropTypes.bool, + grimmoryPreview: PropTypes.object, + onPushToGrimmoryPress: PropTypes.func, onMonitorTogglePress: PropTypes.func.isRequired, onRefreshPress: PropTypes.func, onSearchPress: PropTypes.func.isRequired diff --git a/frontend/src/Book/Details/BookDetailsConnector.js b/frontend/src/Book/Details/BookDetailsConnector.js index fd020ac6..f33b51b2 100644 --- a/frontend/src/Book/Details/BookDetailsConnector.js +++ b/frontend/src/Book/Details/BookDetailsConnector.js @@ -11,6 +11,7 @@ import { executeCommand } from 'Store/Actions/commandActions'; import { clearEditions, fetchEditions } from 'Store/Actions/editionActions'; import { clearQueueDetails, fetchQueueDetails } from 'Store/Actions/queueActions'; import { cancelFetchReleases, clearReleases } from 'Store/Actions/releaseActions'; +import { fetchNotifications } from 'Store/Actions/settingsActions'; import createAllAuthorSelector from 'Store/Selectors/createAllAuthorsSelector'; import createCommandsSelector from 'Store/Selectors/createCommandsSelector'; import createDimensionsSelector from 'Store/Selectors/createDimensionsSelector'; @@ -19,6 +20,34 @@ import { findCommand, isCommandExecuting } from 'Utilities/Command'; import { registerPagePopulator, unregisterPagePopulator } from 'Utilities/pagePopulator'; import BookDetails from './BookDetails'; +function buildGrimmoryPreview(book, author, edition) { + const identifiers = []; + + if (edition?.isbn13) { + identifiers.push(`isbn: ${edition.isbn13}`); + } + + if (edition?.asin) { + identifiers.push(`asin: ${edition.asin}`); + } + + if (edition?.foreignEditionId) { + identifiers.push(`goodreads: ${edition.foreignEditionId}`); + } + + return { + title: edition?.title || book.title, + authors: author.authorName, + series: book.seriesTitle, + description: edition?.overview || book.overview, + publisher: edition?.publisher, + publisheddate: book.releaseDate, + language: edition?.language, + tags: (book.genres || []).join(', '), + identifiers: identifiers.join(', ') + }; +} + const selectBookFiles = createSelector( (state) => state.bookFiles, (bookFiles) => { @@ -51,7 +80,8 @@ function createMapStateToProps() { createCommandsSelector(), createUISettingsSelector(), createDimensionsSelector(), - (bookId, bookFiles, books, editions, authors, commands, uiSettings, dimensions) => { + (state) => state.settings.notifications.items, + (bookId, bookFiles, books, editions, authors, commands, uiSettings, dimensions, notifications) => { try { const book = books.items.find((b) => b.id === bookId); @@ -116,6 +146,15 @@ function createMapStateToProps() { isRenamingAuthorCommand.body.authorIds.indexOf(author.id) > -1 ); + const grimmoryPushCommand = findCommand(commands, { name: commandNames.PUSH_GRIMMORY_METADATA }); + const isPushingToGrimmory = !!( + grimmoryPushCommand && + isCommandExecuting(grimmoryPushCommand) && + grimmoryPushCommand.body && + (grimmoryPushCommand.body.bookIds || []).includes(book.id) + ); + const showPushToGrimmory = notifications.some((n) => n.implementation === 'Grimmory'); + const isFetching = isBookFilesFetching || editions.isFetching; const isPopulated = isBookFilesPopulated && editions.isPopulated; const selectedEdition = editions.items @@ -140,6 +179,9 @@ function createMapStateToProps() { author, isRefreshing, isSearching, + showPushToGrimmory, + isPushingToGrimmory, + grimmoryPreview: buildGrimmoryPreview(book, author, selectedEdition), isRenamingFiles, isRenamingAuthor, isFetching, @@ -163,6 +205,7 @@ function createMapStateToProps() { const mapDispatchToProps = { executeCommand, + fetchNotifications, fetchBookFiles, clearBookFiles, fetchEditions, @@ -228,6 +271,7 @@ class BookDetailsConnector extends Component { this.props.fetchBookFiles({ bookId }); this.props.fetchEditions({ bookId }); this.props.fetchQueueDetails({ bookIds: [bookId] }); + this.props.fetchNotifications(); }; unpopulate = () => { @@ -262,6 +306,14 @@ class BookDetailsConnector extends Component { }); }; + onPushToGrimmoryPress = (fields) => { + this.props.executeCommand({ + name: commandNames.PUSH_GRIMMORY_METADATA, + bookIds: [this.props.id], + fields + }); + }; + // // Render @@ -272,6 +324,7 @@ class BookDetailsConnector extends Component { onMonitorTogglePress={this.onMonitorTogglePress} onRefreshPress={this.onRefreshPress} onSearchPress={this.onSearchPress} + onPushToGrimmoryPress={this.onPushToGrimmoryPress} /> ); } @@ -286,6 +339,7 @@ BookDetailsConnector.propTypes = { isBookFetching: PropTypes.bool, isBookPopulated: PropTypes.bool, bookId: PropTypes.number.isRequired, + fetchNotifications: PropTypes.func.isRequired, fetchBookFiles: PropTypes.func.isRequired, clearBookFiles: PropTypes.func.isRequired, fetchEditions: PropTypes.func.isRequired, diff --git a/frontend/src/Book/Editor/BookEditorFooter.js b/frontend/src/Book/Editor/BookEditorFooter.js index b16d5cc4..6175ed65 100644 --- a/frontend/src/Book/Editor/BookEditorFooter.js +++ b/frontend/src/Book/Editor/BookEditorFooter.js @@ -1,9 +1,15 @@ import PropTypes from 'prop-types'; import React, { Component } from 'react'; +import { connect } from 'react-redux'; +import * as commandNames from 'Commands/commandNames'; import SelectInput from 'Components/Form/SelectInput'; import SpinnerButton from 'Components/Link/SpinnerButton'; import PageContentFooter from 'Components/Page/PageContentFooter'; +import GrimmoryPushModal from 'Grimmory/GrimmoryPushModal'; import { kinds } from 'Helpers/Props'; +import { executeCommand } from 'Store/Actions/commandActions'; +import { fetchNotifications } from 'Store/Actions/settingsActions'; +import createCommandExecutingSelector from 'Store/Selectors/createCommandExecutingSelector'; import translate from 'Utilities/String/translate'; import BookEditorFooterLabel from './BookEditorFooterLabel'; import DeleteBookModal from './Delete/DeleteBookModal'; @@ -24,12 +30,17 @@ class BookEditorFooter extends Component { rootFolderPath: NO_CHANGE, savingTags: false, isDeleteBookModalOpen: false, + isGrimmoryPushModalOpen: false, isTagsModalOpen: false, isConfirmMoveModalOpen: false, destinationRootFolder: null }; } + componentDidMount() { + this.props.fetchNotifications(); + } + componentDidUpdate(prevProps) { const { isSaving, @@ -72,6 +83,24 @@ class BookEditorFooter extends Component { this.setState({ isDeleteBookModalOpen: false }); }; + onPushToGrimmoryPress = () => { + this.setState({ isGrimmoryPushModalOpen: true }); + }; + + onGrimmoryPushModalClose = () => { + this.setState({ isGrimmoryPushModalOpen: false }); + }; + + onGrimmoryPushConfirmed = (fields) => { + this.setState({ isGrimmoryPushModalOpen: false }); + + this.props.executeCommand({ + name: commandNames.PUSH_GRIMMORY_METADATA, + bookIds: this.props.bookIds, + fields + }); + }; + // // Render @@ -80,12 +109,15 @@ class BookEditorFooter extends Component { bookIds, selectedCount, isSaving, - isDeleting + isDeleting, + isPushingToGrimmory, + showPushToGrimmory } = this.props; const { monitored, - isDeleteBookModalOpen + isDeleteBookModalOpen, + isGrimmoryPushModalOpen } = this.state; const monitoredOptions = [ @@ -119,6 +151,20 @@ class BookEditorFooter extends Component { />
+ { + showPushToGrimmory ? + + {translate('PushChaptarrMetadataToGrimmory')} + : + null + } + + + ); } @@ -150,7 +203,20 @@ BookEditorFooter.propTypes = { saveError: PropTypes.object, isDeleting: PropTypes.bool.isRequired, deleteError: PropTypes.object, + isPushingToGrimmory: PropTypes.bool.isRequired, + showPushToGrimmory: PropTypes.bool.isRequired, + fetchNotifications: PropTypes.func.isRequired, + executeCommand: PropTypes.func.isRequired, onSaveSelected: PropTypes.func.isRequired }; -export default BookEditorFooter; +const selectIsPushingToGrimmory = createCommandExecutingSelector(commandNames.PUSH_GRIMMORY_METADATA); + +function mapStateToProps(state) { + return { + isPushingToGrimmory: selectIsPushingToGrimmory(state), + showPushToGrimmory: state.settings.notifications.items.some((n) => n.implementation === 'Grimmory') + }; +} + +export default connect(mapStateToProps, { executeCommand, fetchNotifications })(BookEditorFooter); diff --git a/frontend/src/Commands/commandNames.js b/frontend/src/Commands/commandNames.js index db9a4a0e..f0e7bfa9 100644 --- a/frontend/src/Commands/commandNames.js +++ b/frontend/src/Commands/commandNames.js @@ -13,6 +13,7 @@ export const BOOK_SEARCH = 'BookSearch'; export const INTERACTIVE_IMPORT = 'ManualImport'; export const MISSING_BOOK_SEARCH = 'MissingBookSearch'; export const MOVE_AUTHOR = 'MoveAuthor'; +export const PUSH_GRIMMORY_METADATA = 'PushGrimmoryMetadata'; export const REFRESH_AUTHOR = 'RefreshAuthor'; export const BULK_REFRESH_AUTHOR = 'BulkRefreshAuthor'; export const REFRESH_BOOK = 'RefreshBook'; diff --git a/frontend/src/Grimmory/GrimmoryPushModal.js b/frontend/src/Grimmory/GrimmoryPushModal.js new file mode 100644 index 00000000..a21d4e23 --- /dev/null +++ b/frontend/src/Grimmory/GrimmoryPushModal.js @@ -0,0 +1,34 @@ +import PropTypes from 'prop-types'; +import React from 'react'; +import Modal from 'Components/Modal/Modal'; +import GrimmoryPushModalContent from './GrimmoryPushModalContent'; + +function GrimmoryPushModal(props) { + const { + isOpen, + onModalClose, + ...otherProps + } = props; + + return ( + + { + isOpen && + + } + + ); +} + +GrimmoryPushModal.propTypes = { + isOpen: PropTypes.bool.isRequired, + onModalClose: PropTypes.func.isRequired +}; + +export default GrimmoryPushModal; diff --git a/frontend/src/Grimmory/GrimmoryPushModalContent.css b/frontend/src/Grimmory/GrimmoryPushModalContent.css new file mode 100644 index 00000000..242fed40 --- /dev/null +++ b/frontend/src/Grimmory/GrimmoryPushModalContent.css @@ -0,0 +1,29 @@ +.description { + margin-bottom: 20px; +} + +.field { + display: flex; + align-items: center; + padding: 6px 0; + border-bottom: 1px solid var(--borderColor); +} + +.field:last-child { + border-bottom: none; +} + +.check { + flex: 0 0 30px; +} + +.label { + flex: 0 0 130px; + font-weight: bold; +} + +.value { + flex: 1 1 auto; + color: var(--helpTextColor); + word-break: break-word; +} diff --git a/frontend/src/Grimmory/GrimmoryPushModalContent.css.d.ts b/frontend/src/Grimmory/GrimmoryPushModalContent.css.d.ts new file mode 100644 index 00000000..f8a86fb7 --- /dev/null +++ b/frontend/src/Grimmory/GrimmoryPushModalContent.css.d.ts @@ -0,0 +1,11 @@ +// This file is automatically generated. +// Please do not change this file! +interface CssExports { + 'check': string; + 'description': string; + 'field': string; + 'label': string; + 'value': string; +} +export const cssExports: CssExports; +export default cssExports; diff --git a/frontend/src/Grimmory/GrimmoryPushModalContent.js b/frontend/src/Grimmory/GrimmoryPushModalContent.js new file mode 100644 index 00000000..cb11786e --- /dev/null +++ b/frontend/src/Grimmory/GrimmoryPushModalContent.js @@ -0,0 +1,129 @@ +import PropTypes from 'prop-types'; +import React, { Component } from 'react'; +import CheckInput from 'Components/Form/CheckInput'; +import Button from 'Components/Link/Button'; +import ModalBody from 'Components/Modal/ModalBody'; +import ModalContent from 'Components/Modal/ModalContent'; +import ModalFooter from 'Components/Modal/ModalFooter'; +import ModalHeader from 'Components/Modal/ModalHeader'; +import { kinds } from 'Helpers/Props'; +import translate from 'Utilities/String/translate'; +import styles from './GrimmoryPushModalContent.css'; + +const grimmoryFields = [ + { name: 'cover', label: 'Cover' }, + { name: 'title', label: 'Title' }, + { name: 'authors', label: 'Author' }, + { name: 'series', label: 'Series' }, + { name: 'description', label: 'Description' }, + { name: 'publisher', label: 'Publisher' }, + { name: 'publisheddate', label: 'Publish Date' }, + { name: 'language', label: 'Language' }, + { name: 'tags', label: 'Tags' }, + { name: 'identifiers', label: 'Identifiers' } +]; + +class GrimmoryPushModalContent extends Component { + + constructor(props, context) { + super(props, context); + + const selected = {}; + grimmoryFields.forEach((field) => { + selected[field.name] = true; + }); + + this.state = { selected }; + } + + // + // Listeners + + onFieldChange = ({ name, value }) => { + this.setState((state) => { + return { selected: { ...state.selected, [name]: value } }; + }); + }; + + onPushPress = () => { + const fields = grimmoryFields + .map((field) => field.name) + .filter((name) => this.state.selected[name]); + + this.props.onPushPress(fields); + }; + + // + // Render + + render() { + const { + bookCount, + previewValues, + onModalClose + } = this.props; + + const { + selected + } = this.state; + + const anySelected = grimmoryFields.some((field) => selected[field.name]); + + return ( + + + {translate('PushChaptarrMetadataToGrimmory')} + + + +
+ {translate('GrimmoryPushDescriptionInterp', [bookCount])} +
+ + { + grimmoryFields.map((field) => { + const preview = previewValues ? previewValues[field.name] : null; + + return ( +
+
+ +
+
{field.label}
+
{preview}
+
+ ); + }) + } + + + + + + + + + ); + } +} + +GrimmoryPushModalContent.propTypes = { + bookCount: PropTypes.number.isRequired, + previewValues: PropTypes.object, + onPushPress: PropTypes.func.isRequired, + onModalClose: PropTypes.func.isRequired +}; + +export default GrimmoryPushModalContent; diff --git a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs index 4ffc04d3..f8e858e7 100644 --- a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs +++ b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs @@ -1,11 +1,13 @@ using System; using System.Collections.Generic; +using System.Reflection; using FluentValidation.Results; using NLog; using NUnit.Framework; using NzbDrone.Common.Cache; using NzbDrone.Core.Books; using NzbDrone.Core.MediaFiles; +using NzbDrone.Core.Messaging.Commands; using NzbDrone.Core.Notifications; using NzbDrone.Core.Notifications.Grimmory; using NzbDrone.Core.Qualities; @@ -201,6 +203,7 @@ private static NzbDrone.Core.Notifications.Grimmory.Grimmory CreateSubject(FakeG return new NzbDrone.Core.Notifications.Grimmory.Grimmory( proxy, + DispatchProxy.Create(), new CacheManager(), LogManager.GetLogger("GrimmoryFixture")) { @@ -208,6 +211,14 @@ private static NzbDrone.Core.Notifications.Grimmory.Grimmory CreateSubject(FakeG }; } + public class InertCommandQueueProxy : DispatchProxy + { + protected override object Invoke(MethodInfo targetMethod, object[] args) + { + return null; + } + } + private class FakeGrimmoryProxy : IGrimmoryProxy { public List Libraries { get; set; } = new List(); @@ -233,6 +244,14 @@ public ValidationFailure Test(GrimmorySettings settings) { return null; } + + public List GetLibraryBooks(GrimmorySettings settings, long libraryId, bool bypassCache = false) => new List(); + public GrimmoryBook FindBookByPath(GrimmorySettings settings, long libraryId, string relativePath, bool bypassCache = false) => null; + public void UpdateBookMetadata(GrimmorySettings settings, long bookId, Dictionary metadata) { } + public void UploadBookCover(GrimmorySettings settings, long bookId, byte[] image, string fileName) { } + public byte[] GetBookCover(GrimmorySettings settings, long bookId) => null; + public string BuildCoverUrl(GrimmorySettings settings, long bookId) => string.Empty; + public List GetMetadataAuditEntries(GrimmorySettings settings, DateTime fromUtc) => new List(); } } } diff --git a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs new file mode 100644 index 00000000..791b1eaa --- /dev/null +++ b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs @@ -0,0 +1,286 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using FluentValidation.Results; +using NLog; +using NUnit.Framework; +using NzbDrone.Common.Cache; +using NzbDrone.Core.Books; +using NzbDrone.Core.Lifecycle; +using NzbDrone.Core.MediaFiles; +using NzbDrone.Core.Messaging.Commands; +using NzbDrone.Core.Notifications; +using NzbDrone.Core.Notifications.Grimmory; +using NzbDrone.Core.RootFolders; + +namespace Chaptarr.Core.Test.Notifications.Grimmory +{ + [TestFixture] + public class GrimmoryLibraryChangeForwarderFixture + { + private const long EbookLibraryId = 3; + private const string RelativePath = "Robin Hobb/Assassin's Apprentice/Assassin's Apprentice.epub"; + + public class StubProxy : DispatchProxy + { + public Dictionary> Handlers { get; } = new Dictionary>(); + + protected override object Invoke(MethodInfo targetMethod, object[] args) + { + if (Handlers.TryGetValue(targetMethod.Name, out var handler)) + { + return handler(args); + } + + throw new NotImplementedException($"Stub does not handle {targetMethod.Name}"); + } + } + + private static T Stub(out StubProxy stub) + { + var proxy = DispatchProxy.Create(); + stub = (StubProxy)(object)proxy; + return proxy; + } + + private class ScriptedGrimmoryProxy : IGrimmoryProxy + { + public Dictionary> BooksByLibrary { get; } = new Dictionary>(); + public List AuditEntries { get; } = new List(); + + public List GetLibraries(GrimmorySettings settings) => new List(); + public void RefreshLibrary(GrimmorySettings settings, long libraryId) { } + + public List GetLibraryBooks(GrimmorySettings settings, long libraryId, bool bypassCache = false) + { + return BooksByLibrary.TryGetValue(libraryId, out var books) ? books : new List(); + } + + public GrimmoryBook FindBookByPath(GrimmorySettings settings, long libraryId, string relativePath, bool bypassCache = false) => null; + public void UpdateBookMetadata(GrimmorySettings settings, long bookId, Dictionary metadata) { } + public void UploadBookCover(GrimmorySettings settings, long bookId, byte[] image, string fileName) { } + public byte[] GetBookCover(GrimmorySettings settings, long bookId) => new byte[] { 9 }; + public string BuildCoverUrl(GrimmorySettings settings, long bookId) => $"http://grimmory/cover/{bookId}"; + public List GetMetadataAuditEntries(GrimmorySettings settings, DateTime fromUtc) => AuditEntries.ToList(); + public ValidationFailure Test(GrimmorySettings settings) => null; + } + + private class TestEditTarget : NotificationBase, IExternalLibraryEditTarget + { + public List<(Book Book, ExternalLibraryEditPayload Payload)> Pushes { get; } = new List<(Book, ExternalLibraryEditPayload)>(); + + public override string Name => "TestTarget"; + public override string Link => string.Empty; + public bool AcceptsExternalLibraryEdits => true; + + public void PushExternalLibraryEdit(Book book, List files, ExternalLibraryEditPayload payload) + { + Pushes.Add((book, payload)); + } + + public override ValidationResult Test() + { + return new ValidationResult(); + } + } + + private class Context + { + public ScriptedGrimmoryProxy Proxy; + public GrimmoryLibraryChangeForwarder Forwarder; + public TestEditTarget Target; + public GrimmorySettings Settings; + } + + private static GrimmoryBook BuildGrimmoryBook(DateTime? coverUpdatedOn = null) + { + var slash = RelativePath.LastIndexOf('/'); + + return new GrimmoryBook + { + Id = 100, + LibraryId = EbookLibraryId, + PrimaryFile = new GrimmoryBookFile + { + FileSubPath = RelativePath.Substring(0, slash), + FileName = RelativePath.Substring(slash + 1) + }, + Metadata = new GrimmoryBookMetadata + { + Title = "Assassin's Apprentice", + Description = "Edited in Grimmory.", + Publisher = "Voyager", + SeriesName = "Farseer", + SeriesNumber = 1, + Language = "eng", + Isbn13 = "9780007562252", + Categories = new List { "fantasy" }, + CoverUpdatedOn = coverUpdatedOn + } + }; + } + + private static Context CreateContext(int targetDefinitionId = 2) + { + var context = new Context(); + var proxy = new ScriptedGrimmoryProxy(); + context.Proxy = proxy; + + var settings = new GrimmorySettings + { + Url = "http://grimmory:6060", + Username = "chaptarr", + Password = "secret", + EbookLibraryId = EbookLibraryId, + ForwardEdits = true + }; + context.Settings = settings; + + var commandQueue = Stub(out var commandStub); + commandStub.Handlers["Push"] = _ => null; + + var source = new NzbDrone.Core.Notifications.Grimmory.Grimmory(proxy, commandQueue, new CacheManager(), LogManager.GetLogger("test")) + { + Definition = new NotificationDefinition { Id = 1, Name = "Grimmory", Settings = settings } + }; + + var target = new TestEditTarget + { + Definition = new NotificationDefinition { Id = targetDefinitionId, Name = "TestTarget", Settings = new GrimmorySettings() } + }; + context.Target = target; + + var factory = Stub(out var factoryStub); + factoryStub.Handlers["GetAvailableProviders"] = _ => new List { source, target }; + + var rootFolderService = Stub(out var rootStub); + rootStub.Handlers["All"] = _ => new List { new RootFolder { Id = 1, Path = @"C:\books".AsOsAgnostic() } }; + + var expectedPath = Path.Combine(@"C:\books".AsOsAgnostic(), RelativePath.Replace('/', Path.DirectorySeparatorChar)); + var bookFile = new BookFile { Id = 40, EditionId = 30, Path = expectedPath, MediaType = "ebook" }; + + var mediaFileService = Stub(out var mediaFileStub); + mediaFileStub.Handlers["GetFileWithPath"] = args => string.Equals((string)args[0], expectedPath, StringComparison.OrdinalIgnoreCase) ? bookFile : null; + mediaFileStub.Handlers["GetFilesByBook"] = _ => new List { bookFile }; + + var editionService = Stub(out var editionStub); + editionStub.Handlers["GetEdition"] = args => (int)args[0] == 30 ? new Edition { Id = 30, BookId = 10 } : null; + + var bookService = Stub(out var bookStub); + bookStub.Handlers["GetBook"] = args => (int)args[0] == 10 ? new Book { Id = 10, Title = "Assassin's Apprentice" } : null; + + context.Forwarder = new GrimmoryLibraryChangeForwarder( + factory, + proxy, + rootFolderService, + mediaFileService, + editionService, + bookService, + LogManager.GetLogger("GrimmoryLibraryChangeForwarderFixture")); + + return context; + } + + private static GrimmoryAuditEntry MetadataEditBy(string username) + { + return new GrimmoryAuditEntry + { + Id = 1, + Username = username, + EntityType = "Book", + EntityId = 100, + CreatedAt = DateTime.UtcNow + }; + } + + [Test] + public void should_not_forward_anything_on_first_poll() + { + var context = CreateContext(); + context.Proxy.BooksByLibrary[EbookLibraryId] = new List { BuildGrimmoryBook() }; + context.Proxy.AuditEntries.Add(MetadataEditBy("editor")); + + context.Forwarder.Handle(new ApplicationStartedEvent()); + + Assert.That(context.Target.Pushes, Is.Empty); + + context.Forwarder.Dispose(); + } + + [Test] + public void should_forward_metadata_edit_by_another_user() + { + var context = CreateContext(); + context.Proxy.BooksByLibrary[EbookLibraryId] = new List { BuildGrimmoryBook() }; + + context.Forwarder.Handle(new ApplicationStartedEvent()); + context.Proxy.AuditEntries.Add(MetadataEditBy("editor")); + context.Forwarder.Handle(new ApplicationStartedEvent()); + + Assert.That(context.Target.Pushes, Has.Count.EqualTo(1)); + + var (book, payload) = context.Target.Pushes[0]; + + Assert.Multiple(() => + { + Assert.That(book.Id, Is.EqualTo(10)); + Assert.That(payload.Title, Is.EqualTo("Assassin's Apprentice")); + Assert.That(payload.Description, Is.EqualTo("Edited in Grimmory.")); + Assert.That(payload.SeriesName, Is.EqualTo("Farseer")); + Assert.That(payload.Identifiers["isbn"], Is.EqualTo("9780007562252")); + Assert.That(payload.CoverBytes, Is.Not.Null); + }); + + context.Forwarder.Dispose(); + } + + [Test] + public void should_ignore_edits_made_by_the_connections_own_user() + { + var context = CreateContext(); + context.Proxy.BooksByLibrary[EbookLibraryId] = new List { BuildGrimmoryBook() }; + + context.Forwarder.Handle(new ApplicationStartedEvent()); + context.Proxy.AuditEntries.Add(MetadataEditBy("chaptarr")); + context.Forwarder.Handle(new ApplicationStartedEvent()); + + Assert.That(context.Target.Pushes, Is.Empty); + + context.Forwarder.Dispose(); + } + + [Test] + public void should_forward_cover_change_detected_from_timestamps() + { + var context = CreateContext(); + var initial = DateTime.UtcNow.AddHours(-1); + context.Proxy.BooksByLibrary[EbookLibraryId] = new List { BuildGrimmoryBook(initial) }; + + context.Forwarder.Handle(new ApplicationStartedEvent()); + + context.Proxy.BooksByLibrary[EbookLibraryId] = new List { BuildGrimmoryBook(DateTime.UtcNow) }; + context.Forwarder.Handle(new ApplicationStartedEvent()); + + Assert.That(context.Target.Pushes, Has.Count.EqualTo(1)); + + context.Forwarder.Dispose(); + } + + [Test] + public void should_skip_target_sharing_the_sources_definition() + { + var context = CreateContext(targetDefinitionId: 1); + context.Proxy.BooksByLibrary[EbookLibraryId] = new List { BuildGrimmoryBook() }; + + context.Forwarder.Handle(new ApplicationStartedEvent()); + context.Proxy.AuditEntries.Add(MetadataEditBy("editor")); + context.Forwarder.Handle(new ApplicationStartedEvent()); + + Assert.That(context.Target.Pushes, Is.Empty); + + context.Forwarder.Dispose(); + } + } +} diff --git a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryProxyFixture.cs b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryProxyFixture.cs index ed6ab938..7e2d03a3 100644 --- a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryProxyFixture.cs +++ b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryProxyFixture.cs @@ -16,6 +16,7 @@ namespace Chaptarr.Core.Test.Notifications.Grimmory public class GrimmoryProxyFixture { private const string LibrariesJson = "[{\"id\":10,\"name\":\"Ebooks\",\"allowedFormats\":[\"EPUB\",\"PDF\"]},{\"id\":20,\"name\":\"Audiobooks\",\"allowedFormats\":[\"AUDIOBOOK\"]}]"; + private const string LibraryBooksJson = "[{\"id\":100,\"libraryId\":10,\"primaryFile\":{\"fileName\":\"Book One.epub\",\"fileSubPath\":\"Author Name/Book One\"},\"metadata\":{\"title\":\"Book One\"}}]"; [Test] public void should_login_and_fetch_libraries_with_bearer_token() @@ -117,6 +118,58 @@ public void test_should_pass_when_configured_libraries_exist() Assert.That(proxy.Test(BuildSettings()), Is.Null); } + [Test] + public void should_find_book_by_path_ignoring_slash_direction_and_case() + { + var httpClient = new ScriptedHttpClient { ValidTokens = { "token1" } }; + var proxy = CreateProxy(httpClient); + + var book = proxy.FindBookByPath(BuildSettings(), 10, "Author Name\\book one\\Book One.epub"); + + Assert.That(book, Is.Not.Null); + Assert.That(book.Id, Is.EqualTo(100)); + } + + [Test] + public void should_send_metadata_update_with_replace_when_provided_mode() + { + var httpClient = new ScriptedHttpClient { ValidTokens = { "token1" } }; + var proxy = CreateProxy(httpClient); + + proxy.UpdateBookMetadata(BuildSettings(), 100, new Dictionary { { "title", "New Title" }, { "titleLocked", true } }); + + var request = httpClient.Requests.Last(); + + Assert.Multiple(() => + { + Assert.That(request.Method, Is.EqualTo(HttpMethod.Put)); + Assert.That(request.Url.ToString(), Does.Contain("/api/v1/books/100/metadata")); + Assert.That(request.Url.ToString(), Does.Contain("replaceMode=REPLACE_WHEN_PROVIDED")); + + var body = System.Text.Encoding.UTF8.GetString(request.ContentData); + Assert.That(body, Does.Contain("\"metadata\"")); + Assert.That(body, Does.Contain("\"titleLocked\": true").Or.Contain("\"titleLocked\":true")); + }); + } + + [Test] + public void should_upload_cover_as_multipart_file() + { + var httpClient = new ScriptedHttpClient { ValidTokens = { "token1" } }; + var proxy = CreateProxy(httpClient); + + proxy.UploadBookCover(BuildSettings(), 100, new byte[] { 1, 2, 3 }, "cover.jpg"); + + var request = httpClient.Requests.Last(); + + Assert.Multiple(() => + { + Assert.That(request.Method, Is.EqualTo(HttpMethod.Post)); + Assert.That(request.Url.ToString(), Does.EndWith("/api/v1/books/100/metadata/cover/upload")); + Assert.That(request.Headers.ContentType, Does.Contain("multipart/form-data")); + }); + } + private static GrimmoryProxy CreateProxy(ScriptedHttpClient httpClient) { return new GrimmoryProxy(httpClient, new CacheManager(), LogManager.GetLogger("GrimmoryProxyFixture")); @@ -179,6 +232,21 @@ public HttpResponse Execute(HttpRequest request) return new HttpResponse(request, headers, string.Empty, HttpStatusCode.NoContent); } + if (url.Contains("/api/v1/libraries/") && url.EndsWith("/book")) + { + return new HttpResponse(request, headers, LibraryBooksJson); + } + + if (url.Contains("/metadata/cover/upload")) + { + return new HttpResponse(request, headers, string.Empty); + } + + if (url.Contains("/api/v1/books/") && url.Contains("/metadata")) + { + return new HttpResponse(request, headers, "{}"); + } + return new HttpResponse(request, headers, string.Empty, HttpStatusCode.NotFound); } diff --git a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs new file mode 100644 index 00000000..73505978 --- /dev/null +++ b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs @@ -0,0 +1,329 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using FluentValidation.Results; +using NLog; +using NUnit.Framework; +using NzbDrone.Common.Cache; +using NzbDrone.Core.Books; +using NzbDrone.Core.MediaCover; +using NzbDrone.Core.MediaFiles; +using NzbDrone.Core.Messaging.Commands; +using NzbDrone.Core.Notifications; +using NzbDrone.Core.Notifications.Grimmory; +using NzbDrone.Core.RootFolders; + +namespace Chaptarr.Core.Test.Notifications.Grimmory +{ + [TestFixture] + public class GrimmoryPushServiceFixture + { + private const long EbookLibraryId = 3; + private const long AudiobookLibraryId = 4; + + public class StubProxy : DispatchProxy + { + public Dictionary> Handlers { get; } = new Dictionary>(); + + protected override object Invoke(MethodInfo targetMethod, object[] args) + { + if (Handlers.TryGetValue(targetMethod.Name, out var handler)) + { + return handler(args); + } + + throw new NotImplementedException($"Stub does not handle {targetMethod.Name}"); + } + } + + private static T Stub(out StubProxy stub) + { + var proxy = DispatchProxy.Create(); + stub = (StubProxy)(object)proxy; + return proxy; + } + + private class FakeGrimmoryProxy : IGrimmoryProxy + { + public Dictionary BooksByPath { get; } = new Dictionary(StringComparer.OrdinalIgnoreCase); + public List<(long BookId, Dictionary Metadata)> MetadataUpdates { get; } = new List<(long, Dictionary)>(); + public List<(long BookId, string FileName)> CoverUploads { get; } = new List<(long, string)>(); + + public List GetLibraries(GrimmorySettings settings) => new List(); + public void RefreshLibrary(GrimmorySettings settings, long libraryId) { } + public List GetLibraryBooks(GrimmorySettings settings, long libraryId, bool bypassCache = false) => BooksByPath.Values.ToList(); + + public GrimmoryBook FindBookByPath(GrimmorySettings settings, long libraryId, string relativePath, bool bypassCache = false) + { + return BooksByPath.TryGetValue(relativePath.Replace('\\', '/'), out var book) ? book : null; + } + + public void UpdateBookMetadata(GrimmorySettings settings, long bookId, Dictionary metadata) => MetadataUpdates.Add((bookId, metadata)); + public void UploadBookCover(GrimmorySettings settings, long bookId, byte[] image, string fileName) => CoverUploads.Add((bookId, fileName)); + public byte[] GetBookCover(GrimmorySettings settings, long bookId) => null; + public string BuildCoverUrl(GrimmorySettings settings, long bookId) => $"http://grimmory/cover/{bookId}"; + public List GetMetadataAuditEntries(GrimmorySettings settings, DateTime fromUtc) => new List(); + public ValidationFailure Test(GrimmorySettings settings) => null; + } + + private class Context + { + public FakeGrimmoryProxy Proxy; + public GrimmoryPushService Service; + public List PushedCommands = new List(); + public GrimmorySettings Settings; + } + + private static Context CreateContext(bool pushMetadata = true, bool pushCovers = true, string coverPath = null) + { + var context = new Context(); + var proxy = new FakeGrimmoryProxy(); + context.Proxy = proxy; + + var settings = new GrimmorySettings + { + Url = "http://grimmory:6060", + Username = "chaptarr", + Password = "secret", + EbookLibraryId = EbookLibraryId, + AudiobookLibraryId = AudiobookLibraryId, + PushMetadata = pushMetadata, + PushCovers = pushCovers + }; + context.Settings = settings; + + var commandQueue = Stub(out var commandStub); + commandStub.Handlers["Push"] = args => + { + context.PushedCommands.Add((Command)args[0]); + return null; + }; + + var provider = new NzbDrone.Core.Notifications.Grimmory.Grimmory(proxy, commandQueue, new CacheManager(), LogManager.GetLogger("test")) + { + Definition = new NotificationDefinition { Id = 1, Name = "Grimmory", Settings = settings } + }; + + var factory = Stub(out var factoryStub); + factoryStub.Handlers["GetAvailableProviders"] = _ => new List { provider }; + + var book = new Book + { + Id = 10, + AuthorId = 20, + Title = "Assassin's Apprentice", + Overview = "A royal bastard trains as an assassin.", + MediaType = BookMediaType.Ebook, + Genres = new List { "fantasy" } + }; + + var bookService = Stub(out var bookStub); + bookStub.Handlers["GetBook"] = args => (int)args[0] == 10 ? book : null; + + var authorService = Stub(out var authorStub); + authorStub.Handlers["GetAuthor"] = _ => new Author { Id = 20, Name = "Robin Hobb" }; + + var edition = new Edition + { + Id = 30, + BookId = 10, + Title = "Assassin's Apprentice", + Overview = "Edition overview.", + Publisher = "Voyager", + Language = "eng", + Isbn13 = "9780007562252", + Monitored = true, + Images = new List { new NzbDrone.Core.MediaCover.MediaCover(MediaCoverTypes.Cover, "http://x/cover.jpg") } + }; + + var editionService = Stub(out var editionStub); + editionStub.Handlers["GetEditionsByBook"] = _ => new List { edition }; + + var mediaFileService = Stub(out var mediaFileStub); + mediaFileStub.Handlers["GetFilesByBook"] = _ => new List + { + new BookFile { Id = 40, EditionId = 30, Path = @"C:\books\Robin Hobb\Assassin's Apprentice\Assassin's Apprentice.epub".AsOsAgnostic(), MediaType = "ebook" } + }; + + var rootFolderService = Stub(out var rootStub); + rootStub.Handlers["GetBestRootFolder"] = _ => new RootFolder { Id = 1, Path = @"C:\books".AsOsAgnostic() }; + + var coverMapper = Stub(out var coverStub); + coverStub.Handlers["GetCoverPath"] = _ => coverPath ?? @"C:\nonexistent\cover.jpg".AsOsAgnostic(); + + context.Service = new GrimmoryPushService( + factory, + proxy, + bookService, + authorService, + editionService, + mediaFileService, + rootFolderService, + coverMapper, + commandQueue, + new CacheManager(), + LogManager.GetLogger("GrimmoryPushServiceFixture")); + + return context; + } + + private static GrimmoryBook GrimmoryBookAt(string relativePath, long id = 100) + { + var slash = relativePath.LastIndexOf('/'); + + return new GrimmoryBook + { + Id = id, + LibraryId = EbookLibraryId, + PrimaryFile = new GrimmoryBookFile + { + FileSubPath = slash > 0 ? relativePath.Substring(0, slash) : string.Empty, + FileName = relativePath.Substring(slash + 1) + } + }; + } + + [Test] + public void should_push_metadata_with_locks_to_matched_book() + { + var context = CreateContext(); + context.Proxy.BooksByPath["Robin Hobb/Assassin's Apprentice/Assassin's Apprentice.epub"] = GrimmoryBookAt("Robin Hobb/Assassin's Apprentice/Assassin's Apprentice.epub"); + + context.Service.Execute(new PushGrimmoryMetadataCommand + { + BookIds = new List { 10 }, + Fields = new List { "title", "description", "authors", "identifiers" } + }); + + Assert.That(context.Proxy.MetadataUpdates, Has.Count.EqualTo(1)); + + var (bookId, metadata) = context.Proxy.MetadataUpdates[0]; + + Assert.Multiple(() => + { + Assert.That(bookId, Is.EqualTo(100)); + Assert.That(metadata["title"], Is.EqualTo("Assassin's Apprentice")); + Assert.That(metadata["titleLocked"], Is.True); + Assert.That(metadata["description"], Is.EqualTo("Edition overview.")); + Assert.That(metadata["authors"], Is.EqualTo(new List { "Robin Hobb" })); + Assert.That(metadata["isbn13"], Is.EqualTo("9780007562252")); + Assert.That(metadata["isbn13Locked"], Is.True); + Assert.That(metadata.ContainsKey("publisher"), Is.False); + }); + } + + [Test] + public void should_skip_when_book_not_found_in_grimmory() + { + var context = CreateContext(); + + context.Service.Execute(new PushGrimmoryMetadataCommand + { + BookIds = new List { 10 }, + Fields = new List { "title" } + }); + + Assert.That(context.Proxy.MetadataUpdates, Is.Empty); + } + + [Test] + public void should_upload_cover_when_cover_field_selected_and_file_exists() + { + var coverFile = Path.GetTempFileName(); + File.WriteAllBytes(coverFile, new byte[] { 1, 2, 3 }); + + try + { + var context = CreateContext(coverPath: coverFile); + context.Proxy.BooksByPath["Robin Hobb/Assassin's Apprentice/Assassin's Apprentice.epub"] = GrimmoryBookAt("Robin Hobb/Assassin's Apprentice/Assassin's Apprentice.epub"); + + context.Service.Execute(new PushGrimmoryMetadataCommand + { + BookIds = new List { 10 }, + Fields = new List { "cover" } + }); + + Assert.That(context.Proxy.CoverUploads, Has.Count.EqualTo(1)); + Assert.That(context.Proxy.MetadataUpdates, Is.Empty); + } + finally + { + File.Delete(coverFile); + } + } + + [Test] + public void should_skip_cover_upload_when_cover_file_missing() + { + var context = CreateContext(); + context.Proxy.BooksByPath["Robin Hobb/Assassin's Apprentice/Assassin's Apprentice.epub"] = GrimmoryBookAt("Robin Hobb/Assassin's Apprentice/Assassin's Apprentice.epub"); + + context.Service.Execute(new PushGrimmoryMetadataCommand + { + BookIds = new List { 10 }, + Fields = new List { "cover" } + }); + + Assert.That(context.Proxy.CoverUploads, Is.Empty); + } + + [Test] + public void should_queue_push_command_for_book_scoped_cover_event() + { + var context = CreateContext(); + + context.Service.Handle(new MediaCoversUpdatedEvent(new Book { Id = 10 })); + + Assert.That(context.PushedCommands.OfType().Count(), Is.EqualTo(1)); + + var command = context.PushedCommands.OfType().Single(); + Assert.That(command.BookIds, Is.EqualTo(new List { 10 })); + Assert.That(command.Fields, Does.Contain("cover")); + Assert.That(command.Fields, Does.Contain("title")); + } + + [Test] + public void should_not_queue_push_for_author_scoped_cover_event() + { + var context = CreateContext(); + + context.Service.Handle(new MediaCoversUpdatedEvent(new Author { Id = 20 })); + + Assert.That(context.PushedCommands, Is.Empty); + } + + [Test] + public void should_not_queue_push_when_toggles_disabled() + { + var context = CreateContext(pushMetadata: false, pushCovers: false); + + context.Service.Handle(new MediaCoversUpdatedEvent(new Book { Id = 10 })); + + Assert.That(context.PushedCommands, Is.Empty); + } + + [Test] + public void should_dedupe_repeated_cover_events_for_same_book() + { + var context = CreateContext(); + + context.Service.Handle(new MediaCoversUpdatedEvent(new Book { Id = 10 })); + context.Service.Handle(new MediaCoversUpdatedEvent(new Book { Id = 10 })); + + Assert.That(context.PushedCommands, Has.Count.EqualTo(1)); + } + + [Test] + public void toggle_fields_should_reflect_settings() + { + Assert.Multiple(() => + { + Assert.That(GrimmoryPushService.ToggleFields(new GrimmorySettings { PushCovers = true }), Is.EqualTo(new List { "cover" })); + Assert.That(GrimmoryPushService.ToggleFields(new GrimmorySettings { PushMetadata = true }), Does.Not.Contain("cover")); + Assert.That(GrimmoryPushService.ToggleFields(new GrimmorySettings()), Is.Empty); + }); + } + } +} diff --git a/src/NzbDrone.Core/Localization/Core/en.json b/src/NzbDrone.Core/Localization/Core/en.json index 12c5600a..e869f89c 100644 --- a/src/NzbDrone.Core/Localization/Core/en.json +++ b/src/NzbDrone.Core/Localization/Core/en.json @@ -818,6 +818,8 @@ "GlobalProxy": "Default Proxy", "GlobalProxyHelpText": "Default proxy used when proxy routing applies. In Indexers Only mode it is used for indexers without a specific override. In Proxy Everything mode it is also used for metadata, covers, notifications, updates, and other app HTTP requests.", "GoToAuthorListing": "Go to author listing", + "GrimmoryPush": "Grimmory Push", + "GrimmoryPushDescriptionInterp": "Choose which fields to push to Grimmory for {0} book(s). Pushed fields are locked in Grimmory so its own metadata refreshes do not overwrite them; anything left unticked is untouched.", "GoToInteractiveSearch": "Go to Interactive Search", "GoToInterp": "Go to {0}", "GoodreadsImportListBookshelfRequired": "Select at least one bookshelf.", @@ -1503,6 +1505,8 @@ "Publisher": "Publisher", "PurgeAndReaddAuthor": "Purge & Re-add Author", "PurgeAndReaddAuthorHelpText": "Remove all database records for this author and immediately re-add and refresh from scratch. No files on disk will be touched.", + "PushChaptarrMetadataToGrimmory": "Push Chaptarr metadata to Grimmory", + "PushToGrimmory": "Push to Grimmory", "Qualities": "Qualities", "Quality": "Quality", "QualityCriteriaAllowUpgrades": "Allow Upgrades", diff --git a/src/NzbDrone.Core/Notifications/ExternalLibraryEdits.cs b/src/NzbDrone.Core/Notifications/ExternalLibraryEdits.cs new file mode 100644 index 00000000..9cf9ef2e --- /dev/null +++ b/src/NzbDrone.Core/Notifications/ExternalLibraryEdits.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Generic; +using NzbDrone.Core.Books; +using NzbDrone.Core.MediaFiles; +using NzbDrone.Core.ThingiProvider; + +namespace NzbDrone.Core.Notifications +{ + public class ExternalLibraryEditPayload + { + public string Title { get; set; } + public string Subtitle { get; set; } + public string Description { get; set; } + public string Publisher { get; set; } + public DateTime? PublishedDate { get; set; } + public string SeriesName { get; set; } + public double? SeriesPosition { get; set; } + public List Languages { get; set; } + public List Genres { get; set; } + public Dictionary Identifiers { get; set; } + public string CoverUrl { get; set; } + public byte[] CoverBytes { get; set; } + } + + // Seam between library-edit sources (e.g. the Grimmory forwarder) and connections able to + // mirror those edits outward. Sources discover targets via the notification factory, so a + // provider opts in simply by implementing this on top of NotificationBase; nothing here + // references a concrete provider, keeping each side independently mergeable. + public interface IExternalLibraryEditTarget + { + ProviderDefinition Definition { get; } + bool AcceptsExternalLibraryEdits { get; } + void PushExternalLibraryEdit(Book book, List files, ExternalLibraryEditPayload payload); + } +} diff --git a/src/NzbDrone.Core/Notifications/Grimmory/Grimmory.cs b/src/NzbDrone.Core/Notifications/Grimmory/Grimmory.cs index d79eb81f..f46b097b 100644 --- a/src/NzbDrone.Core/Notifications/Grimmory/Grimmory.cs +++ b/src/NzbDrone.Core/Notifications/Grimmory/Grimmory.cs @@ -7,18 +7,21 @@ using NzbDrone.Common.Extensions; using NzbDrone.Core.Books; using NzbDrone.Core.MediaFiles; +using NzbDrone.Core.Messaging.Commands; namespace NzbDrone.Core.Notifications.Grimmory { public class Grimmory : NotificationBase { private readonly IGrimmoryProxy _proxy; + private readonly IManageCommandQueue _commandQueueManager; private readonly Logger _logger; private readonly ICached _pendingLibrariesCache; - public Grimmory(IGrimmoryProxy proxy, ICacheManager cacheManager, Logger logger) + public Grimmory(IGrimmoryProxy proxy, IManageCommandQueue commandQueueManager, ICacheManager cacheManager, Logger logger) { _proxy = proxy; + _commandQueueManager = commandQueueManager; _logger = logger; _pendingLibrariesCache = cacheManager.GetRollingCache(GetType(), "pendingLibraries", TimeSpan.FromDays(1)); } @@ -58,6 +61,9 @@ public override void OnReleaseImport(BookDownloadMessage message) } QueueRefresh(GetLibraryId(message.Book, message.BookFiles.FirstOrDefault()), "import"); + + // The push waits for Grimmory's (async) refresh to ingest the new files first. + QueueAutoPush(message.Book, waitForBook: true); } public override void OnRename(Author author, List renamedFiles) @@ -96,6 +102,29 @@ public override void OnBookFileDelete(BookFileDeleteMessage message) public override void OnBookRetag(BookRetagMessage message) { QueueRefresh(GetLibraryId(message.Book, message.BookFile), "retag"); + QueueAutoPush(message.Book, waitForBook: false); + } + + private void QueueAutoPush(Book book, bool waitForBook) + { + if (book == null || (!Settings.PushMetadata && !Settings.PushCovers)) + { + return; + } + + var fields = GrimmoryPushService.ToggleFields(Settings); + + if (fields.Empty()) + { + return; + } + + _commandQueueManager.Push(new PushGrimmoryMetadataCommand + { + BookIds = new List { book.Id }, + Fields = fields, + WaitForBook = waitForBook + }); } public override void ProcessQueue() diff --git a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryLibraryChangeForwarder.cs b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryLibraryChangeForwarder.cs new file mode 100644 index 00000000..c99a8932 --- /dev/null +++ b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryLibraryChangeForwarder.cs @@ -0,0 +1,340 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using NLog; +using NzbDrone.Common.Extensions; +using NzbDrone.Core.Books; +using NzbDrone.Core.Lifecycle; +using NzbDrone.Core.MediaFiles; +using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.RootFolders; + +namespace NzbDrone.Core.Notifications.Grimmory +{ + // Watches Grimmory for metadata and cover edits and forwards them to connections that + // implement IExternalLibraryEditTarget. Grimmory is remote, so unlike the calibre + // forwarder there is no filesystem signal to hook - metadata edits are detected through + // Grimmory's audit log (filtered by the connection's own username, so Chaptarr's pushes + // do not echo back) and cover edits through each book's coverUpdatedOn stamps. + public class GrimmoryLibraryChangeForwarder : IHandle, IDisposable + { + private static readonly TimeSpan PollInterval = TimeSpan.FromMinutes(2); + private static readonly TimeSpan AuditOverlap = TimeSpan.FromMinutes(5); + + private readonly INotificationFactory _notificationFactory; + private readonly IGrimmoryProxy _proxy; + private readonly IRootFolderService _rootFolderService; + private readonly IMediaFileService _mediaFileService; + private readonly IEditionService _editionService; + private readonly IBookService _bookService; + private readonly Logger _logger; + + private readonly System.Timers.Timer _timer; + private readonly object _pollLock = new object(); + private readonly ConcurrentDictionary _states = new ConcurrentDictionary(); + + private class SourceState + { + public DateTime LastAuditPoll { get; set; } + public Dictionary CoverStamps { get; } = new Dictionary(); + public bool Primed { get; set; } + } + + public GrimmoryLibraryChangeForwarder(INotificationFactory notificationFactory, + IGrimmoryProxy proxy, + IRootFolderService rootFolderService, + IMediaFileService mediaFileService, + IEditionService editionService, + IBookService bookService, + Logger logger) + { + _notificationFactory = notificationFactory; + _proxy = proxy; + _rootFolderService = rootFolderService; + _mediaFileService = mediaFileService; + _editionService = editionService; + _bookService = bookService; + _logger = logger; + + _timer = new System.Timers.Timer(PollInterval.TotalMilliseconds) { AutoReset = true }; + _timer.Elapsed += (s, e) => Poll(); + } + + public void Handle(ApplicationStartedEvent message) + { + Poll(); + _timer.Start(); + } + + public void Dispose() + { + _timer.Dispose(); + } + + private void Poll() + { + if (!System.Threading.Monitor.TryEnter(_pollLock)) + { + return; + } + + try + { + foreach (var source in ForwardingSources()) + { + try + { + PollSource(source); + } + catch (Exception ex) + { + _logger.Warn(ex, "Failed to poll Grimmory '{0}' for library edits", source.Definition.Name); + } + } + } + finally + { + System.Threading.Monitor.Exit(_pollLock); + } + } + + private List ForwardingSources() + { + return _notificationFactory.GetAvailableProviders() + .OfType() + .Where(g => (g.Definition?.Settings as GrimmorySettings)?.ForwardEdits == true) + .ToList(); + } + + private void PollSource(Grimmory source) + { + var settings = (GrimmorySettings)source.Definition.Settings; + var state = _states.GetOrAdd(source.Definition.Id, _ => new SourceState { LastAuditPoll = DateTime.UtcNow }); + var pollStarted = DateTime.UtcNow; + + var libraryIds = new[] { settings.EbookLibraryId, settings.AudiobookLibraryId }.Where(id => id > 0).Distinct().ToList(); + var books = libraryIds + .SelectMany(id => FetchLibraryBooks(settings, id)) + .GroupBy(b => b.Id) + .Select(g => g.First()) + .ToDictionary(b => b.Id); + + var changedIds = new HashSet(DetectCoverChanges(state, books)); + + if (state.Primed) + { + foreach (var entry in _proxy.GetMetadataAuditEntries(settings, state.LastAuditPoll - AuditOverlap)) + { + if (entry.EntityId == null || entry.EntityType != "Book") + { + continue; + } + + if (entry.CreatedAt == null || entry.CreatedAt <= state.LastAuditPoll - AuditOverlap) + { + continue; + } + + if (entry.Username.IsNotNullOrWhiteSpace() && entry.Username.Equals(settings.Username, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + changedIds.Add(entry.EntityId.Value); + } + } + + state.LastAuditPoll = pollStarted; + + if (!state.Primed) + { + state.Primed = true; + return; + } + + if (changedIds.Empty()) + { + return; + } + + var targets = _notificationFactory.GetAvailableProviders() + .OfType() + .Where(t => t.AcceptsExternalLibraryEdits && t.Definition?.Id != source.Definition.Id) + .ToList(); + + if (targets.Empty()) + { + _logger.Debug("Grimmory '{0}' has {1} changed book(s) but no connections accept library edits", source.Definition.Name, changedIds.Count); + return; + } + + foreach (var grimmoryId in changedIds) + { + if (!books.TryGetValue(grimmoryId, out var grimmoryBook)) + { + continue; + } + + try + { + ForwardBook(settings, grimmoryBook, targets); + } + catch (Exception ex) + { + _logger.Warn(ex, "Failed to forward Grimmory edit for book {0}", grimmoryId); + } + } + } + + private List FetchLibraryBooks(GrimmorySettings settings, long libraryId) + { + try + { + return _proxy.GetLibraryBooks(settings, libraryId, bypassCache: true); + } + catch (Exception ex) + { + _logger.Warn(ex, "Failed to list Grimmory library {0}", libraryId); + return new List(); + } + } + + private List DetectCoverChanges(SourceState state, Dictionary books) + { + var changed = new List(); + + foreach (var book in books.Values) + { + var stamps = (book.Metadata?.CoverUpdatedOn, book.Metadata?.AudiobookCoverUpdatedOn); + + if (state.CoverStamps.TryGetValue(book.Id, out var previous) && state.Primed) + { + if ((stamps.Item1 != null && stamps.Item1 > (previous.Cover ?? DateTime.MinValue)) || + (stamps.Item2 != null && stamps.Item2 > (previous.AudiobookCover ?? DateTime.MinValue))) + { + changed.Add(book.Id); + } + } + + state.CoverStamps[book.Id] = stamps; + } + + return changed; + } + + private void ForwardBook(GrimmorySettings settings, GrimmoryBook grimmoryBook, List targets) + { + var bookFile = ResolveBookFile(grimmoryBook); + + if (bookFile == null) + { + _logger.Debug("No Chaptarr file matches Grimmory book {0}; skipping forward", grimmoryBook.Id); + return; + } + + var edition = _editionService.GetEdition(bookFile.EditionId); + var book = edition == null ? null : _bookService.GetBook(edition.BookId); + + if (book == null) + { + return; + } + + var files = _mediaFileService.GetFilesByBook(book.Id); + var payload = BuildPayload(settings, grimmoryBook); + + foreach (var target in targets) + { + try + { + target.PushExternalLibraryEdit(book, files, payload); + _logger.Debug("Forwarded Grimmory edit of '{0}' to {1}", book.Title, target.Definition?.Name); + } + catch (Exception ex) + { + _logger.Warn(ex, "Failed to forward Grimmory edit of '{0}' to {1}", book.Title, target.Definition?.Name); + } + } + } + + private BookFile ResolveBookFile(GrimmoryBook grimmoryBook) + { + foreach (var relativePath in grimmoryBook.AllFiles().Select(f => f?.RelativePath()).Where(p => p.IsNotNullOrWhiteSpace())) + { + var osRelative = relativePath.Replace('/', Path.DirectorySeparatorChar); + + foreach (var rootFolder in _rootFolderService.All()) + { + var candidate = Path.Combine(rootFolder.Path, osRelative); + var file = _mediaFileService.GetFileWithPath(candidate); + + if (file != null) + { + return file; + } + } + } + + return null; + } + + private ExternalLibraryEditPayload BuildPayload(GrimmorySettings settings, GrimmoryBook grimmoryBook) + { + var metadata = grimmoryBook.Metadata; + var payload = new ExternalLibraryEditPayload(); + + if (metadata != null) + { + payload.Title = metadata.Title; + payload.Subtitle = metadata.Subtitle; + payload.Description = metadata.Description; + payload.Publisher = metadata.Publisher; + payload.SeriesName = metadata.SeriesName; + payload.SeriesPosition = metadata.SeriesNumber; + payload.Languages = metadata.Language.IsNotNullOrWhiteSpace() ? new List { metadata.Language } : null; + payload.Genres = metadata.Categories?.Any() == true ? metadata.Categories : null; + + if (DateTime.TryParse(metadata.PublishedDate, out var published)) + { + payload.PublishedDate = published; + } + + var identifiers = new Dictionary(); + + if (metadata.Isbn13.IsNotNullOrWhiteSpace()) + { + identifiers["isbn"] = metadata.Isbn13; + } + + if (metadata.Asin.IsNotNullOrWhiteSpace()) + { + identifiers["asin"] = metadata.Asin; + } + + if (metadata.GoodreadsId.IsNotNullOrWhiteSpace()) + { + identifiers["goodreads"] = metadata.GoodreadsId; + } + + if (identifiers.Any()) + { + payload.Identifiers = identifiers; + } + } + + try + { + payload.CoverBytes = _proxy.GetBookCover(settings, grimmoryBook.Id); + payload.CoverUrl = _proxy.BuildCoverUrl(settings, grimmoryBook.Id); + } + catch (Exception ex) + { + _logger.Debug(ex, "Could not fetch Grimmory cover for book {0}", grimmoryBook.Id); + } + + return payload; + } + } +} diff --git a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryProxy.cs b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryProxy.cs index 0eb00e37..e23b71d8 100644 --- a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryProxy.cs +++ b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryProxy.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Net; using System.Net.Http; using FluentValidation.Results; @@ -16,21 +17,33 @@ public interface IGrimmoryProxy { List GetLibraries(GrimmorySettings settings); void RefreshLibrary(GrimmorySettings settings, long libraryId); + List GetLibraryBooks(GrimmorySettings settings, long libraryId, bool bypassCache = false); + GrimmoryBook FindBookByPath(GrimmorySettings settings, long libraryId, string relativePath, bool bypassCache = false); + void UpdateBookMetadata(GrimmorySettings settings, long bookId, Dictionary metadata); + void UploadBookCover(GrimmorySettings settings, long bookId, byte[] image, string fileName); + byte[] GetBookCover(GrimmorySettings settings, long bookId); + string BuildCoverUrl(GrimmorySettings settings, long bookId); + List GetMetadataAuditEntries(GrimmorySettings settings, DateTime fromUtc); ValidationFailure Test(GrimmorySettings settings); } public class GrimmoryProxy : IGrimmoryProxy { private static readonly TimeSpan TokenCacheDuration = TimeSpan.FromMinutes(30); + private static readonly TimeSpan BookListCacheDuration = TimeSpan.FromMinutes(1); + private const int AuditPageSize = 200; + private const int MaxAuditPages = 5; private readonly IHttpClient _httpClient; private readonly ICached _tokenCache; + private readonly ICached> _bookListCache; private readonly Logger _logger; public GrimmoryProxy(IHttpClient httpClient, ICacheManager cacheManager, Logger logger) { _httpClient = httpClient; _tokenCache = cacheManager.GetCache(GetType(), "tokens"); + _bookListCache = cacheManager.GetCache>(GetType(), "books"); _logger = logger; } @@ -57,6 +70,137 @@ public void RefreshLibrary(GrimmorySettings settings, long libraryId) _logger.Debug("Triggered Grimmory refresh for library {0}", libraryId); } + public List GetLibraryBooks(GrimmorySettings settings, long libraryId, bool bypassCache = false) + { + var cacheKey = $"{settings.Url}:{settings.Username}:{libraryId}"; + + if (bypassCache) + { + _bookListCache.Remove(cacheKey); + } + + return _bookListCache.Get(cacheKey, + () => + { + var response = ExecuteWithAuth(settings, token => + { + var request = BuildRequest(settings, $"api/v1/libraries/{libraryId}/book", token).Build(); + return _httpClient.Get(request); + }); + + return Json.Deserialize>(response.Content) ?? new List(); + }, + BookListCacheDuration); + } + + public GrimmoryBook FindBookByPath(GrimmorySettings settings, long libraryId, string relativePath, bool bypassCache = false) + { + var normalized = NormalizeRelativePath(relativePath); + + if (normalized.IsNullOrWhiteSpace()) + { + return null; + } + + return GetLibraryBooks(settings, libraryId, bypassCache) + .FirstOrDefault(b => b.AllFiles().Any(f => NormalizeRelativePath(f?.RelativePath()) == normalized)); + } + + public void UpdateBookMetadata(GrimmorySettings settings, long bookId, Dictionary metadata) + { + ExecuteWithAuth(settings, token => + { + var request = BuildRequest(settings, $"api/v1/books/{bookId}/metadata", token) + .AddQueryParam("replaceMode", "REPLACE_WHEN_PROVIDED") + .Build(); + + request.Method = HttpMethod.Put; + request.Headers.ContentType = "application/json"; + request.SetContent(new Dictionary { { "metadata", metadata } }.ToJson()); + + return _httpClient.Execute(request); + }); + + _logger.Debug("Updated Grimmory metadata for book {0}", bookId); + } + + public void UploadBookCover(GrimmorySettings settings, long bookId, byte[] image, string fileName) + { + ExecuteWithAuth(settings, token => + { + var request = BuildRequest(settings, $"api/v1/books/{bookId}/metadata/cover/upload", token) + .Post() + .AddFormUpload("file", fileName, image, GetImageContentType(fileName)) + .Build(); + + return _httpClient.Execute(request); + }); + + _logger.Debug("Uploaded Grimmory cover for book {0}", bookId); + } + + public byte[] GetBookCover(GrimmorySettings settings, long bookId) + { + try + { + var response = ExecuteWithAuth(settings, token => + { + var request = BuildRequest(settings, $"api/v1/media/book/{bookId}/cover", token).Build(); + return _httpClient.Get(request); + }); + + return response.ResponseData; + } + catch (HttpException ex) when (ex.Response?.StatusCode == HttpStatusCode.NotFound) + { + return null; + } + } + + public string BuildCoverUrl(GrimmorySettings settings, long bookId) + { + var token = GetAccessToken(settings, false); + + return $"{HttpUri.CombinePath(settings.Url, $"api/v1/media/book/{bookId}/cover")}?token={token}"; + } + + public List GetMetadataAuditEntries(GrimmorySettings settings, DateTime fromUtc) + { + var entries = new List(); + + for (var page = 0; page < MaxAuditPages; page++) + { + var pageNumber = page; + var response = ExecuteWithAuth(settings, token => + { + var request = BuildRequest(settings, "api/v1/audit-logs", token) + .AddQueryParam("action", "METADATA_UPDATED") + .AddQueryParam("size", AuditPageSize) + .AddQueryParam("page", pageNumber) + .AddQueryParam("from", fromUtc.ToString("yyyy-MM-dd'T'HH:mm:ss")) + .Build(); + + return _httpClient.Get(request); + }); + + var result = Json.Deserialize(response.Content); + + if (result?.Content == null) + { + break; + } + + entries.AddRange(result.Content); + + if (result.Last) + { + break; + } + } + + return entries; + } + public ValidationFailure Test(GrimmorySettings settings) { try @@ -72,6 +216,18 @@ public ValidationFailure Test(GrimmorySettings settings) { return new ValidationFailure(nameof(GrimmorySettings.AudiobookLibraryId), "The selected audiobook library was not found in Grimmory"); } + + if (settings.ForwardEdits) + { + try + { + GetMetadataAuditEntries(settings, DateTime.UtcNow.AddMinutes(-1)); + } + catch (HttpException ex) when (ex.Response?.StatusCode == HttpStatusCode.Forbidden || ex.Response?.StatusCode == HttpStatusCode.Unauthorized) + { + return new ValidationFailure(nameof(GrimmorySettings.ForwardEdits), "Forwarding Grimmory edits requires an admin user, as change detection reads the audit log"); + } + } } catch (GrimmoryAuthenticationException) { @@ -86,6 +242,24 @@ public ValidationFailure Test(GrimmorySettings settings) return null; } + private static string NormalizeRelativePath(string path) + { + return path?.Replace('\\', '/').Trim('/').ToLowerInvariant() ?? string.Empty; + } + + private static string GetImageContentType(string fileName) + { + var extension = System.IO.Path.GetExtension(fileName)?.ToLowerInvariant(); + + return extension switch + { + ".png" => "image/png", + ".gif" => "image/gif", + ".webp" => "image/webp", + _ => "image/jpeg" + }; + } + private HttpResponse ExecuteWithAuth(GrimmorySettings settings, Func action) { var token = GetAccessToken(settings, false); @@ -186,6 +360,126 @@ public class GrimmoryLibrary public List AllowedFormats { get; set; } } + public class GrimmoryBook + { + [JsonProperty("id")] + public long Id { get; set; } + + [JsonProperty("libraryId")] + public long LibraryId { get; set; } + + [JsonProperty("primaryFile")] + public GrimmoryBookFile PrimaryFile { get; set; } + + [JsonProperty("alternativeFormats")] + public List AlternativeFormats { get; set; } + + [JsonProperty("metadata")] + public GrimmoryBookMetadata Metadata { get; set; } + + public IEnumerable AllFiles() + { + if (PrimaryFile != null) + { + yield return PrimaryFile; + } + + foreach (var file in AlternativeFormats ?? Enumerable.Empty()) + { + yield return file; + } + } + } + + public class GrimmoryBookFile + { + [JsonProperty("fileName")] + public string FileName { get; set; } + + [JsonProperty("fileSubPath")] + public string FileSubPath { get; set; } + + public string RelativePath() + { + return FileSubPath.IsNotNullOrWhiteSpace() ? $"{FileSubPath}/{FileName}" : FileName; + } + } + + public class GrimmoryBookMetadata + { + [JsonProperty("title")] + public string Title { get; set; } + + [JsonProperty("subtitle")] + public string Subtitle { get; set; } + + [JsonProperty("description")] + public string Description { get; set; } + + [JsonProperty("publisher")] + public string Publisher { get; set; } + + [JsonProperty("publishedDate")] + public string PublishedDate { get; set; } + + [JsonProperty("seriesName")] + public string SeriesName { get; set; } + + [JsonProperty("seriesNumber")] + public double? SeriesNumber { get; set; } + + [JsonProperty("language")] + public string Language { get; set; } + + [JsonProperty("isbn13")] + public string Isbn13 { get; set; } + + [JsonProperty("asin")] + public string Asin { get; set; } + + [JsonProperty("goodreadsId")] + public string GoodreadsId { get; set; } + + [JsonProperty("authors")] + public List Authors { get; set; } + + [JsonProperty("categories")] + public List Categories { get; set; } + + [JsonProperty("coverUpdatedOn")] + public DateTime? CoverUpdatedOn { get; set; } + + [JsonProperty("audiobookCoverUpdatedOn")] + public DateTime? AudiobookCoverUpdatedOn { get; set; } + } + + public class GrimmoryAuditPage + { + [JsonProperty("content")] + public List Content { get; set; } + + [JsonProperty("last")] + public bool Last { get; set; } + } + + public class GrimmoryAuditEntry + { + [JsonProperty("id")] + public long Id { get; set; } + + [JsonProperty("username")] + public string Username { get; set; } + + [JsonProperty("entityType")] + public string EntityType { get; set; } + + [JsonProperty("entityId")] + public long? EntityId { get; set; } + + [JsonProperty("createdAt")] + public DateTime? CreatedAt { get; set; } + } + public class GrimmoryAuthenticationException : Exception { public GrimmoryAuthenticationException(string message) diff --git a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs new file mode 100644 index 00000000..a458e9df --- /dev/null +++ b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs @@ -0,0 +1,383 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using NLog; +using NzbDrone.Common.Cache; +using NzbDrone.Common.Extensions; +using NzbDrone.Core.Books; +using NzbDrone.Core.MediaCover; +using NzbDrone.Core.MediaFiles; +using NzbDrone.Core.Messaging.Commands; +using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.RootFolders; +using NzbDrone.Core.ThingiProvider; + +namespace NzbDrone.Core.Notifications.Grimmory +{ + public class GrimmoryPushService : IExecute, IHandle + { + public static readonly string[] AllFields = + { + "cover", "title", "subtitle", "authors", "series", "description", + "publisher", "publisheddate", "language", "tags", "identifiers" + }; + + private static readonly TimeSpan WaitForBookTimeout = TimeSpan.FromSeconds(90); + private static readonly TimeSpan WaitForBookInterval = TimeSpan.FromSeconds(10); + private static readonly TimeSpan AutoPushCooldown = TimeSpan.FromMinutes(5); + + private readonly INotificationFactory _notificationFactory; + private readonly IGrimmoryProxy _proxy; + private readonly IBookService _bookService; + private readonly IAuthorService _authorService; + private readonly IEditionService _editionService; + private readonly IMediaFileService _mediaFileService; + private readonly IRootFolderService _rootFolderService; + private readonly IMapCoversToLocal _coverMapper; + private readonly IManageCommandQueue _commandQueueManager; + private readonly ICached _recentAutoPushes; + private readonly Logger _logger; + + public GrimmoryPushService(INotificationFactory notificationFactory, + IGrimmoryProxy proxy, + IBookService bookService, + IAuthorService authorService, + IEditionService editionService, + IMediaFileService mediaFileService, + IRootFolderService rootFolderService, + IMapCoversToLocal coverMapper, + IManageCommandQueue commandQueueManager, + ICacheManager cacheManager, + Logger logger) + { + _notificationFactory = notificationFactory; + _proxy = proxy; + _bookService = bookService; + _authorService = authorService; + _editionService = editionService; + _mediaFileService = mediaFileService; + _rootFolderService = rootFolderService; + _coverMapper = coverMapper; + _commandQueueManager = commandQueueManager; + _recentAutoPushes = cacheManager.GetCache(GetType(), "recentAutoPushes"); + _logger = logger; + } + + public static List ToggleFields(GrimmorySettings settings) + { + var fields = new List(); + + if (settings.PushCovers) + { + fields.Add("cover"); + } + + if (settings.PushMetadata) + { + fields.AddRange(AllFields.Where(f => f != "cover")); + } + + return fields; + } + + // Fires when a book's cover/metadata is updated in Chaptarr (e.g. through the UI). + // Author-scoped cover events are deliberately ignored - they fire during routine + // author refreshes and would fan out into pushes for every book of the author. + public void Handle(MediaCoversUpdatedEvent message) + { + var book = message.Book; + + if (book == null) + { + return; + } + + var fields = _notificationFactory.GetAvailableProviders() + .OfType() + .Select(g => g.Definition?.Settings as GrimmorySettings) + .Where(s => s != null) + .SelectMany(ToggleFields) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); + + if (fields.Empty()) + { + return; + } + + var cacheKey = book.Id.ToString(); + + if (_recentAutoPushes.Find(cacheKey) != default) + { + return; + } + + _recentAutoPushes.Set(cacheKey, DateTime.UtcNow, AutoPushCooldown); + + _commandQueueManager.Push(new PushGrimmoryMetadataCommand + { + BookIds = new List { book.Id }, + Fields = fields + }); + } + + public void Execute(PushGrimmoryMetadataCommand message) + { + var bookIds = message.BookIds?.Where(id => id > 0).Distinct().ToList() ?? new List(); + var fields = message.Fields?.Any() == true ? message.Fields : AllFields.ToList(); + + var connections = _notificationFactory.GetAvailableProviders() + .OfType() + .Where(g => g.Definition?.Settings is GrimmorySettings) + .ToList(); + + if (!connections.Any()) + { + _logger.Debug("No enabled Grimmory connections; nothing to push"); + return; + } + + var pushed = 0; + var failed = 0; + + foreach (var bookId in bookIds) + { + try + { + if (PushBook(bookId, fields, connections, message.WaitForBook)) + { + pushed++; + } + } + catch (Exception ex) + { + failed++; + _logger.Warn(ex, "Failed to push book {0} to Grimmory", bookId); + } + } + + _logger.Info("Pushed {0} of {1} book(s) to Grimmory", pushed, bookIds.Count); + + if (failed > 0 && pushed == 0) + { + throw new InvalidOperationException($"Failed to push {failed} book(s) to Grimmory"); + } + } + + private bool PushBook(int bookId, List fields, List connections, bool waitForBook) + { + var book = _bookService.GetBook(bookId); + + if (book == null) + { + return false; + } + + var files = _mediaFileService.GetFilesByBook(bookId) + .Where(f => f?.Path.IsNotNullOrWhiteSpace() == true) + .ToList(); + + if (!files.Any()) + { + _logger.Debug("No files on disk for '{0}'; nothing to push to Grimmory", book.Title); + return false; + } + + var author = _authorService.GetAuthor(book.AuthorId); + var edition = ResolveEdition(book); + var anyPushed = false; + + foreach (var connection in connections) + { + var settings = (GrimmorySettings)connection.Definition.Settings; + var libraryId = book.MediaType == BookMediaType.Ebook ? settings.EbookLibraryId : settings.AudiobookLibraryId; + + if (libraryId <= 0) + { + continue; + } + + var grimmoryBook = FindGrimmoryBook(settings, libraryId, files, waitForBook); + + if (grimmoryBook == null) + { + _logger.Debug("'{0}' not found in Grimmory library {1} on {2}; skipping", book.Title, libraryId, settings.Url); + continue; + } + + var metadata = BuildMetadata(book, author, edition, fields); + + if (metadata.Any()) + { + _proxy.UpdateBookMetadata(settings, grimmoryBook.Id, metadata); + } + + if (fields.Contains("cover", StringComparer.OrdinalIgnoreCase)) + { + PushCover(settings, grimmoryBook.Id, book, edition); + } + + _logger.Debug("Pushed '{0}' to Grimmory book {1} on {2}", book.Title, grimmoryBook.Id, settings.Url); + anyPushed = true; + } + + return anyPushed; + } + + private GrimmoryBook FindGrimmoryBook(GrimmorySettings settings, long libraryId, List files, bool waitForBook) + { + var deadline = waitForBook ? DateTime.UtcNow + WaitForBookTimeout : DateTime.UtcNow; + var bypassCache = false; + + while (true) + { + foreach (var file in files) + { + var relativePath = GetRootRelativePath(file.Path); + + if (relativePath.IsNullOrWhiteSpace()) + { + continue; + } + + var grimmoryBook = _proxy.FindBookByPath(settings, libraryId, relativePath, bypassCache); + + if (grimmoryBook != null) + { + return grimmoryBook; + } + } + + if (DateTime.UtcNow >= deadline) + { + return null; + } + + // Freshly imported books only exist in Grimmory once its (async) refresh has + // scanned them, so re-fetch the library list until the book shows up. + Thread.Sleep(WaitForBookInterval); + bypassCache = true; + } + } + + private string GetRootRelativePath(string path) + { + var rootFolder = _rootFolderService.GetBestRootFolder(path); + + if (rootFolder?.Path == null || rootFolder.Path.PathEquals(path)) + { + return null; + } + + return rootFolder.Path.GetRelativePath(path); + } + + private Edition ResolveEdition(Book book) + { + var editions = _editionService.GetEditionsByBook(book.Id); + + return editions.FirstOrDefault(e => e.Monitored) ?? editions.FirstOrDefault(); + } + + private Dictionary BuildMetadata(Book book, Author author, Edition edition, List fields) + { + var metadata = new Dictionary(); + var wanted = new HashSet(fields, StringComparer.OrdinalIgnoreCase); + + void Add(string field, string grimmoryField, object value) + { + if (!wanted.Contains(field) || value == null || (value is string s && s.IsNullOrWhiteSpace())) + { + return; + } + + metadata[grimmoryField] = value; + metadata[$"{grimmoryField}Locked"] = true; + } + + Add("title", "title", edition?.Title ?? book.Title); + Add("description", "description", edition?.Overview ?? book.Overview); + Add("publisher", "publisher", edition?.Publisher); + Add("language", "language", edition?.Language); + + if (wanted.Contains("publisheddate") && book.ReleaseDate.HasValue && book.ReleaseDate.Value > DateTime.MinValue) + { + metadata["publishedDate"] = book.ReleaseDate.Value.ToString("yyyy-MM-dd"); + metadata["publishedDateLocked"] = true; + } + + if (wanted.Contains("authors") && author?.Name.IsNotNullOrWhiteSpace() == true) + { + metadata["authors"] = new List { author.Name }; + } + + if (wanted.Contains("series")) + { + var seriesLink = book.SeriesLinks?.FirstOrDefault(l => l?.Series?.Value?.Title.IsNotNullOrWhiteSpace() == true); + + if (seriesLink != null) + { + metadata["seriesName"] = seriesLink.Series.Value.Title; + metadata["seriesNameLocked"] = true; + + if (double.TryParse(seriesLink.Position, out var position)) + { + metadata["seriesNumber"] = position; + metadata["seriesNumberLocked"] = true; + } + } + } + + if (wanted.Contains("tags") && book.Genres?.Any() == true) + { + metadata["categories"] = book.Genres; + } + + if (wanted.Contains("identifiers")) + { + if (edition?.Isbn13.IsNotNullOrWhiteSpace() == true) + { + metadata["isbn13"] = edition.Isbn13; + metadata["isbn13Locked"] = true; + } + + if (edition?.Asin.IsNotNullOrWhiteSpace() == true) + { + metadata["asin"] = edition.Asin; + metadata["asinLocked"] = true; + } + + if (edition?.ForeignEditionId.IsNotNullOrWhiteSpace() == true) + { + metadata["goodreadsId"] = edition.ForeignEditionId; + metadata["goodreadsIdLocked"] = true; + } + } + + return metadata; + } + + private void PushCover(GrimmorySettings settings, long grimmoryBookId, Book book, Edition edition) + { + var cover = (edition?.Images ?? book.Images)?.FirstOrDefault(i => i.CoverType == MediaCoverTypes.Cover); + + if (cover == null) + { + _logger.Debug("No cover known for '{0}'; skipping cover push", book.Title); + return; + } + + var coverPath = _coverMapper.GetCoverPath(book.Id, MediaCoverEntity.Book, cover.CoverType, cover.Extension); + + if (coverPath.IsNullOrWhiteSpace() || !File.Exists(coverPath)) + { + _logger.Debug("Cover file for '{0}' not present at {1}; skipping cover push", book.Title, coverPath); + return; + } + + _proxy.UploadBookCover(settings, grimmoryBookId, File.ReadAllBytes(coverPath), Path.GetFileName(coverPath)); + } + } +} diff --git a/src/NzbDrone.Core/Notifications/Grimmory/GrimmorySettings.cs b/src/NzbDrone.Core/Notifications/Grimmory/GrimmorySettings.cs index a865a5b0..4e97ec6f 100644 --- a/src/NzbDrone.Core/Notifications/Grimmory/GrimmorySettings.cs +++ b/src/NzbDrone.Core/Notifications/Grimmory/GrimmorySettings.cs @@ -40,6 +40,15 @@ public class GrimmorySettings : IProviderConfig [FieldDefinition(4, Label = "Audiobook Library", Type = FieldType.Select, SelectOptionsProviderAction = "getLibraries", HelpText = "Grimmory library to refresh when Chaptarr imports, renames or deletes audiobook files. Leave unset to ignore audiobooks")] public long AudiobookLibraryId { get; set; } + [FieldDefinition(5, Label = "Push Metadata", Type = FieldType.Checkbox, HelpText = "Push Chaptarr's metadata for a book to Grimmory, locking the pushed fields there, whenever the book is imported, retagged, or its metadata changes in Chaptarr")] + public bool PushMetadata { get; set; } + + [FieldDefinition(6, Label = "Push Covers", Type = FieldType.Checkbox, HelpText = "Push Chaptarr's cover image for a book to Grimmory whenever the book is imported, retagged, or its cover changes in Chaptarr")] + public bool PushCovers { get; set; } + + [FieldDefinition(7, Label = "Forward Grimmory Edits", Type = FieldType.Checkbox, HelpText = "Watch Grimmory for metadata and cover edits and forward them to other connections that accept library edits. The Grimmory user must be an admin, as change detection reads the audit log")] + public bool ForwardEdits { get; set; } + public NzbDroneValidationResult Validate() { return new NzbDroneValidationResult(Validator.Validate(this)); diff --git a/src/NzbDrone.Core/Notifications/Grimmory/PushGrimmoryMetadataCommand.cs b/src/NzbDrone.Core/Notifications/Grimmory/PushGrimmoryMetadataCommand.cs new file mode 100644 index 00000000..91d36cd9 --- /dev/null +++ b/src/NzbDrone.Core/Notifications/Grimmory/PushGrimmoryMetadataCommand.cs @@ -0,0 +1,18 @@ +using System.Collections.Generic; +using NzbDrone.Core.Messaging.Commands; + +namespace NzbDrone.Core.Notifications.Grimmory +{ + public class PushGrimmoryMetadataCommand : Command + { + public List BookIds { get; set; } = new List(); + + public List Fields { get; set; } = new List(); + + // Set for pushes queued right after an import, when Grimmory may not have scanned the + // new files yet - the executor then waits for the book to appear before giving up. + public bool WaitForBook { get; set; } + + public override bool SendUpdatesToClient => true; + } +} From 3a7a340e4f0e06ac0ebcd5461119f79fa243d8d7 Mon Sep 17 00:00:00 2001 From: Benjamin Tobalt Date: Wed, 9 Sep 2026 02:34:05 -0400 Subject: [PATCH 04/18] Detect Grimmory edits by watching sidecar files instead of polling Grimmory rewrites .metadata.json (and .cover.jpg) next to the book on every edit when sidecar write-on-update is enabled, so the forwarder now watches the root folders for those writes - the same mechanism the calibre forwarder uses on metadata.db - replacing the 2-minute audit-log poll and its admin requirement. Chaptarr's own pushes are filtered through a recent-push registry rather than by username, so edits made in Grimmory under the connection's account are forwarded too. --- .../Notifications/Grimmory/GrimmoryFixture.cs | 1 - .../GrimmoryLibraryChangeForwarderFixture.cs | 139 +++---- .../Grimmory/GrimmoryPushServiceFixture.cs | 23 +- .../GrimmoryLibraryChangeForwarder.cs | 353 ++++++++++-------- .../Notifications/Grimmory/GrimmoryProxy.cs | 78 ---- .../Grimmory/GrimmoryPushRegistry.cs | 38 ++ .../Grimmory/GrimmoryPushService.cs | 1 + .../Grimmory/GrimmorySettings.cs | 2 +- 8 files changed, 339 insertions(+), 296 deletions(-) create mode 100644 src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushRegistry.cs diff --git a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs index f8e858e7..be7103f9 100644 --- a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs +++ b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs @@ -251,7 +251,6 @@ public void UpdateBookMetadata(GrimmorySettings settings, long bookId, Dictionar public void UploadBookCover(GrimmorySettings settings, long bookId, byte[] image, string fileName) { } public byte[] GetBookCover(GrimmorySettings settings, long bookId) => null; public string BuildCoverUrl(GrimmorySettings settings, long bookId) => string.Empty; - public List GetMetadataAuditEntries(GrimmorySettings settings, DateTime fromUtc) => new List(); } } } diff --git a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs index 791b1eaa..e2b2e855 100644 --- a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs +++ b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs @@ -8,7 +8,6 @@ using NUnit.Framework; using NzbDrone.Common.Cache; using NzbDrone.Core.Books; -using NzbDrone.Core.Lifecycle; using NzbDrone.Core.MediaFiles; using NzbDrone.Core.Messaging.Commands; using NzbDrone.Core.Notifications; @@ -23,6 +22,12 @@ public class GrimmoryLibraryChangeForwarderFixture private const long EbookLibraryId = 3; private const string RelativePath = "Robin Hobb/Assassin's Apprentice/Assassin's Apprentice.epub"; + [SetUp] + public void Setup() + { + GrimmoryPushRegistry.Clear(); + } + public class StubProxy : DispatchProxy { public Dictionary> Handlers { get; } = new Dictionary>(); @@ -47,23 +52,21 @@ private static T Stub(out StubProxy stub) private class ScriptedGrimmoryProxy : IGrimmoryProxy { - public Dictionary> BooksByLibrary { get; } = new Dictionary>(); - public List AuditEntries { get; } = new List(); + public Dictionary BooksByPath { get; } = new Dictionary(StringComparer.OrdinalIgnoreCase); public List GetLibraries(GrimmorySettings settings) => new List(); public void RefreshLibrary(GrimmorySettings settings, long libraryId) { } + public List GetLibraryBooks(GrimmorySettings settings, long libraryId, bool bypassCache = false) => BooksByPath.Values.ToList(); - public List GetLibraryBooks(GrimmorySettings settings, long libraryId, bool bypassCache = false) + public GrimmoryBook FindBookByPath(GrimmorySettings settings, long libraryId, string relativePath, bool bypassCache = false) { - return BooksByLibrary.TryGetValue(libraryId, out var books) ? books : new List(); + return BooksByPath.TryGetValue(relativePath.Replace('\\', '/'), out var book) ? book : null; } - public GrimmoryBook FindBookByPath(GrimmorySettings settings, long libraryId, string relativePath, bool bypassCache = false) => null; public void UpdateBookMetadata(GrimmorySettings settings, long bookId, Dictionary metadata) { } public void UploadBookCover(GrimmorySettings settings, long bookId, byte[] image, string fileName) { } public byte[] GetBookCover(GrimmorySettings settings, long bookId) => new byte[] { 9 }; public string BuildCoverUrl(GrimmorySettings settings, long bookId) => $"http://grimmory/cover/{bookId}"; - public List GetMetadataAuditEntries(GrimmorySettings settings, DateTime fromUtc) => AuditEntries.ToList(); public ValidationFailure Test(GrimmorySettings settings) => null; } @@ -91,10 +94,11 @@ private class Context public ScriptedGrimmoryProxy Proxy; public GrimmoryLibraryChangeForwarder Forwarder; public TestEditTarget Target; - public GrimmorySettings Settings; + public string SidecarPath; + public string CoverSidecarPath; } - private static GrimmoryBook BuildGrimmoryBook(DateTime? coverUpdatedOn = null) + private static GrimmoryBook BuildGrimmoryBook() { var slash = RelativePath.LastIndexOf('/'); @@ -116,8 +120,7 @@ private static GrimmoryBook BuildGrimmoryBook(DateTime? coverUpdatedOn = null) SeriesNumber = 1, Language = "eng", Isbn13 = "9780007562252", - Categories = new List { "fantasy" }, - CoverUpdatedOn = coverUpdatedOn + Categories = new List { "fantasy" } } }; } @@ -136,7 +139,6 @@ private static Context CreateContext(int targetDefinitionId = 2) EbookLibraryId = EbookLibraryId, ForwardEdits = true }; - context.Settings = settings; var commandQueue = Stub(out var commandStub); commandStub.Handlers["Push"] = _ => null; @@ -155,21 +157,29 @@ private static Context CreateContext(int targetDefinitionId = 2) var factory = Stub(out var factoryStub); factoryStub.Handlers["GetAvailableProviders"] = _ => new List { source, target }; - var rootFolderService = Stub(out var rootStub); - rootStub.Handlers["All"] = _ => new List { new RootFolder { Id = 1, Path = @"C:\books".AsOsAgnostic() } }; + var rootPath = @"C:\books".AsOsAgnostic(); + var bookDir = Path.Combine(rootPath, "Robin Hobb", "Assassin's Apprentice"); + var bookFilePath = Path.Combine(bookDir, "Assassin's Apprentice.epub"); + context.SidecarPath = Path.Combine(bookDir, "Assassin's Apprentice.metadata.json"); + context.CoverSidecarPath = Path.Combine(bookDir, "Assassin's Apprentice.cover.jpg"); + + var bookFile = new BookFile { Id = 40, EditionId = 30, Path = bookFilePath, MediaType = "ebook" }; - var expectedPath = Path.Combine(@"C:\books".AsOsAgnostic(), RelativePath.Replace('/', Path.DirectorySeparatorChar)); - var bookFile = new BookFile { Id = 40, EditionId = 30, Path = expectedPath, MediaType = "ebook" }; + var rootFolderService = Stub(out var rootStub); + rootStub.Handlers["All"] = _ => new List { new RootFolder { Id = 1, Path = rootPath } }; + rootStub.Handlers["GetBestRootFolder"] = _ => new RootFolder { Id = 1, Path = rootPath }; var mediaFileService = Stub(out var mediaFileStub); - mediaFileStub.Handlers["GetFileWithPath"] = args => string.Equals((string)args[0], expectedPath, StringComparison.OrdinalIgnoreCase) ? bookFile : null; + mediaFileStub.Handlers["GetFilesWithBasePath"] = args => string.Equals((string)args[0], bookDir, StringComparison.OrdinalIgnoreCase) + ? new List { bookFile } + : new List(); mediaFileStub.Handlers["GetFilesByBook"] = _ => new List { bookFile }; var editionService = Stub(out var editionStub); editionStub.Handlers["GetEdition"] = args => (int)args[0] == 30 ? new Edition { Id = 30, BookId = 10 } : null; var bookService = Stub(out var bookStub); - bookStub.Handlers["GetBook"] = args => (int)args[0] == 10 ? new Book { Id = 10, Title = "Assassin's Apprentice" } : null; + bookStub.Handlers["GetBook"] = args => (int)args[0] == 10 ? new Book { Id = 10, Title = "Assassin's Apprentice", MediaType = BookMediaType.Ebook } : null; context.Forwarder = new GrimmoryLibraryChangeForwarder( factory, @@ -183,68 +193,71 @@ private static Context CreateContext(int targetDefinitionId = 2) return context; } - private static GrimmoryAuditEntry MetadataEditBy(string username) + [Test] + public void should_forward_edit_signalled_by_metadata_sidecar() { - return new GrimmoryAuditEntry + var context = CreateContext(); + context.Proxy.BooksByPath[RelativePath] = BuildGrimmoryBook(); + + context.Forwarder.QueueSidecar(context.SidecarPath); + context.Forwarder.ForwardPending(); + + Assert.That(context.Target.Pushes, Has.Count.EqualTo(1)); + + var (book, payload) = context.Target.Pushes[0]; + + Assert.Multiple(() => { - Id = 1, - Username = username, - EntityType = "Book", - EntityId = 100, - CreatedAt = DateTime.UtcNow - }; + Assert.That(book.Id, Is.EqualTo(10)); + Assert.That(payload.Title, Is.EqualTo("Assassin's Apprentice")); + Assert.That(payload.Description, Is.EqualTo("Edited in Grimmory.")); + Assert.That(payload.SeriesName, Is.EqualTo("Farseer")); + Assert.That(payload.Identifiers["isbn"], Is.EqualTo("9780007562252")); + Assert.That(payload.CoverBytes, Is.Not.Null); + }); + + context.Forwarder.Dispose(); } [Test] - public void should_not_forward_anything_on_first_poll() + public void should_forward_edit_signalled_by_cover_sidecar() { var context = CreateContext(); - context.Proxy.BooksByLibrary[EbookLibraryId] = new List { BuildGrimmoryBook() }; - context.Proxy.AuditEntries.Add(MetadataEditBy("editor")); + context.Proxy.BooksByPath[RelativePath] = BuildGrimmoryBook(); - context.Forwarder.Handle(new ApplicationStartedEvent()); + context.Forwarder.QueueSidecar(context.CoverSidecarPath); + context.Forwarder.ForwardPending(); - Assert.That(context.Target.Pushes, Is.Empty); + Assert.That(context.Target.Pushes, Has.Count.EqualTo(1)); context.Forwarder.Dispose(); } [Test] - public void should_forward_metadata_edit_by_another_user() + public void should_dedupe_metadata_and_cover_sidecars_for_same_book() { var context = CreateContext(); - context.Proxy.BooksByLibrary[EbookLibraryId] = new List { BuildGrimmoryBook() }; + context.Proxy.BooksByPath[RelativePath] = BuildGrimmoryBook(); - context.Forwarder.Handle(new ApplicationStartedEvent()); - context.Proxy.AuditEntries.Add(MetadataEditBy("editor")); - context.Forwarder.Handle(new ApplicationStartedEvent()); + context.Forwarder.QueueSidecar(context.SidecarPath); + context.Forwarder.QueueSidecar(context.CoverSidecarPath); + context.Forwarder.ForwardPending(); Assert.That(context.Target.Pushes, Has.Count.EqualTo(1)); - var (book, payload) = context.Target.Pushes[0]; - - Assert.Multiple(() => - { - Assert.That(book.Id, Is.EqualTo(10)); - Assert.That(payload.Title, Is.EqualTo("Assassin's Apprentice")); - Assert.That(payload.Description, Is.EqualTo("Edited in Grimmory.")); - Assert.That(payload.SeriesName, Is.EqualTo("Farseer")); - Assert.That(payload.Identifiers["isbn"], Is.EqualTo("9780007562252")); - Assert.That(payload.CoverBytes, Is.Not.Null); - }); - context.Forwarder.Dispose(); } [Test] - public void should_ignore_edits_made_by_the_connections_own_user() + public void should_not_forward_sidecar_written_after_chaptarrs_own_push() { var context = CreateContext(); - context.Proxy.BooksByLibrary[EbookLibraryId] = new List { BuildGrimmoryBook() }; + context.Proxy.BooksByPath[RelativePath] = BuildGrimmoryBook(); + + GrimmoryPushRegistry.RecordPush(10); - context.Forwarder.Handle(new ApplicationStartedEvent()); - context.Proxy.AuditEntries.Add(MetadataEditBy("chaptarr")); - context.Forwarder.Handle(new ApplicationStartedEvent()); + context.Forwarder.QueueSidecar(context.SidecarPath); + context.Forwarder.ForwardPending(); Assert.That(context.Target.Pushes, Is.Empty); @@ -252,18 +265,15 @@ public void should_ignore_edits_made_by_the_connections_own_user() } [Test] - public void should_forward_cover_change_detected_from_timestamps() + public void should_ignore_sidecar_without_matching_book_file() { var context = CreateContext(); - var initial = DateTime.UtcNow.AddHours(-1); - context.Proxy.BooksByLibrary[EbookLibraryId] = new List { BuildGrimmoryBook(initial) }; - - context.Forwarder.Handle(new ApplicationStartedEvent()); + context.Proxy.BooksByPath[RelativePath] = BuildGrimmoryBook(); - context.Proxy.BooksByLibrary[EbookLibraryId] = new List { BuildGrimmoryBook(DateTime.UtcNow) }; - context.Forwarder.Handle(new ApplicationStartedEvent()); + context.Forwarder.QueueSidecar(Path.Combine(@"C:\books".AsOsAgnostic(), "Unknown", "Unknown.metadata.json")); + context.Forwarder.ForwardPending(); - Assert.That(context.Target.Pushes, Has.Count.EqualTo(1)); + Assert.That(context.Target.Pushes, Is.Empty); context.Forwarder.Dispose(); } @@ -272,11 +282,10 @@ public void should_forward_cover_change_detected_from_timestamps() public void should_skip_target_sharing_the_sources_definition() { var context = CreateContext(targetDefinitionId: 1); - context.Proxy.BooksByLibrary[EbookLibraryId] = new List { BuildGrimmoryBook() }; + context.Proxy.BooksByPath[RelativePath] = BuildGrimmoryBook(); - context.Forwarder.Handle(new ApplicationStartedEvent()); - context.Proxy.AuditEntries.Add(MetadataEditBy("editor")); - context.Forwarder.Handle(new ApplicationStartedEvent()); + context.Forwarder.QueueSidecar(context.SidecarPath); + context.Forwarder.ForwardPending(); Assert.That(context.Target.Pushes, Is.Empty); diff --git a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs index 73505978..a05c907e 100644 --- a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs +++ b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs @@ -23,6 +23,12 @@ public class GrimmoryPushServiceFixture private const long EbookLibraryId = 3; private const long AudiobookLibraryId = 4; + [SetUp] + public void Setup() + { + GrimmoryPushRegistry.Clear(); + } + public class StubProxy : DispatchProxy { public Dictionary> Handlers { get; } = new Dictionary>(); @@ -64,7 +70,6 @@ public GrimmoryBook FindBookByPath(GrimmorySettings settings, long libraryId, st public void UploadBookCover(GrimmorySettings settings, long bookId, byte[] image, string fileName) => CoverUploads.Add((bookId, fileName)); public byte[] GetBookCover(GrimmorySettings settings, long bookId) => null; public string BuildCoverUrl(GrimmorySettings settings, long bookId) => $"http://grimmory/cover/{bookId}"; - public List GetMetadataAuditEntries(GrimmorySettings settings, DateTime fromUtc) => new List(); public ValidationFailure Test(GrimmorySettings settings) => null; } @@ -214,6 +219,22 @@ public void should_push_metadata_with_locks_to_matched_book() }); } + [Test] + public void should_record_push_in_registry_for_echo_suppression() + { + var context = CreateContext(); + context.Proxy.BooksByPath["Robin Hobb/Assassin's Apprentice/Assassin's Apprentice.epub"] = GrimmoryBookAt("Robin Hobb/Assassin's Apprentice/Assassin's Apprentice.epub"); + + context.Service.Execute(new PushGrimmoryMetadataCommand + { + BookIds = new List { 10 }, + Fields = new List { "title" } + }); + + Assert.That(GrimmoryPushRegistry.WasRecentlyPushed(10), Is.True); + Assert.That(GrimmoryPushRegistry.WasRecentlyPushed(11), Is.False); + } + [Test] public void should_skip_when_book_not_found_in_grimmory() { diff --git a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryLibraryChangeForwarder.cs b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryLibraryChangeForwarder.cs index c99a8932..514047b0 100644 --- a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryLibraryChangeForwarder.cs +++ b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryLibraryChangeForwarder.cs @@ -6,22 +6,31 @@ using NLog; using NzbDrone.Common.Extensions; using NzbDrone.Core.Books; +using NzbDrone.Core.Datastore.Events; using NzbDrone.Core.Lifecycle; using NzbDrone.Core.MediaFiles; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.RootFolders; +using NzbDrone.Core.ThingiProvider.Events; namespace NzbDrone.Core.Notifications.Grimmory { - // Watches Grimmory for metadata and cover edits and forwards them to connections that - // implement IExternalLibraryEditTarget. Grimmory is remote, so unlike the calibre - // forwarder there is no filesystem signal to hook - metadata edits are detected through - // Grimmory's audit log (filtered by the connection's own username, so Chaptarr's pushes - // do not echo back) and cover edits through each book's coverUpdatedOn stamps. - public class GrimmoryLibraryChangeForwarder : IHandle, IDisposable + // Forwards metadata and cover edits made in Grimmory to connections that implement + // IExternalLibraryEditTarget. Grimmory's database is remote, but with sidecar + // write-on-update enabled it rewrites ".metadata.json" (and, when configured, + // ".cover.jpg") next to the book after every edit - so, like the calibre forwarder + // watching metadata.db, watching the root folders for sidecar writes is the change + // signal. No polling. Chaptarr's own pushes also rewrite the sidecar; those are filtered + // through GrimmoryPushRegistry rather than by author, so edits a person makes in Grimmory + // are forwarded even when they use the connection's own account. + public class GrimmoryLibraryChangeForwarder : + IHandle, + IHandle>, + IHandle>, + IDisposable { - private static readonly TimeSpan PollInterval = TimeSpan.FromMinutes(2); - private static readonly TimeSpan AuditOverlap = TimeSpan.FromMinutes(5); + private static readonly TimeSpan DebounceDelay = TimeSpan.FromSeconds(30); + private static readonly string[] SidecarSuffixes = { ".metadata.json", ".cover.jpg" }; private readonly INotificationFactory _notificationFactory; private readonly IGrimmoryProxy _proxy; @@ -31,16 +40,10 @@ public class GrimmoryLibraryChangeForwarder : IHandle, private readonly IBookService _bookService; private readonly Logger _logger; - private readonly System.Timers.Timer _timer; - private readonly object _pollLock = new object(); - private readonly ConcurrentDictionary _states = new ConcurrentDictionary(); - - private class SourceState - { - public DateTime LastAuditPoll { get; set; } - public Dictionary CoverStamps { get; } = new Dictionary(); - public bool Primed { get; set; } - } + private readonly ConcurrentDictionary _watchers = new ConcurrentDictionary(); + private readonly ConcurrentDictionary _pendingSidecars = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); + private readonly System.Timers.Timer _debounce; + private readonly object _forwardLock = new object(); public GrimmoryLibraryChangeForwarder(INotificationFactory notificationFactory, IGrimmoryProxy proxy, @@ -58,226 +61,276 @@ public GrimmoryLibraryChangeForwarder(INotificationFactory notificationFactory, _bookService = bookService; _logger = logger; - _timer = new System.Timers.Timer(PollInterval.TotalMilliseconds) { AutoReset = true }; - _timer.Elapsed += (s, e) => Poll(); + _debounce = new System.Timers.Timer(DebounceDelay.TotalMilliseconds) { AutoReset = false }; + _debounce.Elapsed += (s, e) => ForwardPending(); } public void Handle(ApplicationStartedEvent message) { - Poll(); - _timer.Start(); + SyncWatchers(); + } + + public void Handle(ModelEvent message) + { + SyncWatchers(); + } + + public void Handle(ProviderUpdatedEvent message) + { + SyncWatchers(); } public void Dispose() { - _timer.Dispose(); + _debounce.Dispose(); + + foreach (var watcher in _watchers.Values) + { + watcher.Dispose(); + } + + _watchers.Clear(); } - private void Poll() + private void SyncWatchers() { - if (!System.Threading.Monitor.TryEnter(_pollLock)) + List wanted; + + try + { + wanted = ForwardingSources().Any() ? _rootFolderService.All() : new List(); + } + catch (Exception ex) { + _logger.Debug(ex, "Unable to evaluate Grimmory forwarding sources"); return; } - try + var wantedIds = wanted.Select(r => r.Id).ToHashSet(); + + foreach (var stale in _watchers.Keys.Where(id => !wantedIds.Contains(id)).ToList()) { - foreach (var source in ForwardingSources()) + if (_watchers.TryRemove(stale, out var watcher)) { - try + watcher.Dispose(); + } + } + + foreach (var rootFolder in wanted) + { + if (_watchers.ContainsKey(rootFolder.Id) || rootFolder.Path.IsNullOrWhiteSpace()) + { + continue; + } + + try + { + var watcher = new FileSystemWatcher(rootFolder.Path) + { + IncludeSubdirectories = true, + NotifyFilter = NotifyFilters.FileName | NotifyFilters.LastWrite, + InternalBufferSize = 65536 + }; + + watcher.Filters.Add("*.metadata.json"); + watcher.Filters.Add("*.cover.jpg"); + + watcher.Changed += (s, e) => QueueSidecar(e.FullPath); + watcher.Created += (s, e) => QueueSidecar(e.FullPath); + watcher.Renamed += (s, e) => QueueSidecar(e.FullPath); + watcher.Error += (s, e) => + { + _logger.Debug(e.GetException(), "Grimmory sidecar watcher error for {0}; recreating", rootFolder.Path); + + if (_watchers.TryRemove(rootFolder.Id, out var broken)) + { + broken.Dispose(); + } + + SyncWatchers(); + }; + + watcher.EnableRaisingEvents = true; + + if (!_watchers.TryAdd(rootFolder.Id, watcher)) { - PollSource(source); + watcher.Dispose(); } - catch (Exception ex) + else { - _logger.Warn(ex, "Failed to poll Grimmory '{0}' for library edits", source.Definition.Name); + _logger.Debug("Watching {0} for Grimmory sidecar changes", rootFolder.Path); } } - } - finally - { - System.Threading.Monitor.Exit(_pollLock); + catch (Exception ex) + { + _logger.Warn(ex, "Unable to watch {0} for Grimmory sidecar changes", rootFolder.Path); + } } } - private List ForwardingSources() + public void QueueSidecar(string path) { - return _notificationFactory.GetAvailableProviders() - .OfType() - .Where(g => (g.Definition?.Settings as GrimmorySettings)?.ForwardEdits == true) - .ToList(); + if (path.IsNullOrWhiteSpace() || !SidecarSuffixes.Any(s => path.EndsWith(s, StringComparison.OrdinalIgnoreCase))) + { + return; + } + + _pendingSidecars[path] = 1; + _debounce.Stop(); + _debounce.Start(); } - private void PollSource(Grimmory source) + public void ForwardPending() { - var settings = (GrimmorySettings)source.Definition.Settings; - var state = _states.GetOrAdd(source.Definition.Id, _ => new SourceState { LastAuditPoll = DateTime.UtcNow }); - var pollStarted = DateTime.UtcNow; + lock (_forwardLock) + { + var pending = _pendingSidecars.Keys.ToList(); + _pendingSidecars.Clear(); - var libraryIds = new[] { settings.EbookLibraryId, settings.AudiobookLibraryId }.Where(id => id > 0).Distinct().ToList(); - var books = libraryIds - .SelectMany(id => FetchLibraryBooks(settings, id)) - .GroupBy(b => b.Id) - .Select(g => g.First()) - .ToDictionary(b => b.Id); + if (pending.Empty()) + { + return; + } - var changedIds = new HashSet(DetectCoverChanges(state, books)); + var sources = ForwardingSources(); - if (state.Primed) - { - foreach (var entry in _proxy.GetMetadataAuditEntries(settings, state.LastAuditPoll - AuditOverlap)) + if (sources.Empty()) { - if (entry.EntityId == null || entry.EntityType != "Book") - { - continue; - } + return; + } + + var forwardedBooks = new HashSet(); - if (entry.CreatedAt == null || entry.CreatedAt <= state.LastAuditPoll - AuditOverlap) + foreach (var sidecarPath in pending) + { + try { - continue; + ForwardSidecarChange(sidecarPath, sources, forwardedBooks); } - - if (entry.Username.IsNotNullOrWhiteSpace() && entry.Username.Equals(settings.Username, StringComparison.OrdinalIgnoreCase)) + catch (Exception ex) { - continue; + _logger.Warn(ex, "Failed to forward Grimmory edit signalled by {0}", sidecarPath); } - - changedIds.Add(entry.EntityId.Value); } } + } - state.LastAuditPoll = pollStarted; + private void ForwardSidecarChange(string sidecarPath, List sources, HashSet forwardedBooks) + { + var bookFile = ResolveSidecarBookFile(sidecarPath); - if (!state.Primed) + if (bookFile == null) { - state.Primed = true; + _logger.Debug("No Chaptarr file matches sidecar {0}; skipping forward", sidecarPath); return; } - if (changedIds.Empty()) + var edition = _editionService.GetEdition(bookFile.EditionId); + var book = edition == null ? null : _bookService.GetBook(edition.BookId); + + if (book == null || !forwardedBooks.Add(book.Id)) + { + return; + } + + if (GrimmoryPushRegistry.WasRecentlyPushed(book.Id)) { + _logger.Debug("Sidecar change for '{0}' follows Chaptarr's own push; not forwarding back out", book.Title); return; } var targets = _notificationFactory.GetAvailableProviders() .OfType() - .Where(t => t.AcceptsExternalLibraryEdits && t.Definition?.Id != source.Definition.Id) + .Where(t => t.AcceptsExternalLibraryEdits) .ToList(); - if (targets.Empty()) - { - _logger.Debug("Grimmory '{0}' has {1} changed book(s) but no connections accept library edits", source.Definition.Name, changedIds.Count); - return; - } + var files = _mediaFileService.GetFilesByBook(book.Id); - foreach (var grimmoryId in changedIds) + foreach (var source in sources) { - if (!books.TryGetValue(grimmoryId, out var grimmoryBook)) + var settings = (GrimmorySettings)source.Definition.Settings; + var libraryId = book.MediaType == BookMediaType.Ebook ? settings.EbookLibraryId : settings.AudiobookLibraryId; + + if (libraryId <= 0) { continue; } - try + var relativePath = GetRootRelativePath(bookFile.Path); + + if (relativePath.IsNullOrWhiteSpace()) { - ForwardBook(settings, grimmoryBook, targets); + continue; } - catch (Exception ex) + + var grimmoryBook = _proxy.FindBookByPath(settings, libraryId, relativePath, bypassCache: true); + + if (grimmoryBook == null) { - _logger.Warn(ex, "Failed to forward Grimmory edit for book {0}", grimmoryId); + continue; } - } - } - private List FetchLibraryBooks(GrimmorySettings settings, long libraryId) - { - try - { - return _proxy.GetLibraryBooks(settings, libraryId, bypassCache: true); - } - catch (Exception ex) - { - _logger.Warn(ex, "Failed to list Grimmory library {0}", libraryId); - return new List(); - } - } + var sourceTargets = targets.Where(t => t.Definition?.Id != source.Definition.Id).ToList(); - private List DetectCoverChanges(SourceState state, Dictionary books) - { - var changed = new List(); + if (sourceTargets.Empty()) + { + _logger.Debug("Grimmory edit of '{0}' detected but no connections accept library edits", book.Title); + continue; + } - foreach (var book in books.Values) - { - var stamps = (book.Metadata?.CoverUpdatedOn, book.Metadata?.AudiobookCoverUpdatedOn); + var payload = BuildPayload(settings, grimmoryBook); - if (state.CoverStamps.TryGetValue(book.Id, out var previous) && state.Primed) + foreach (var target in sourceTargets) { - if ((stamps.Item1 != null && stamps.Item1 > (previous.Cover ?? DateTime.MinValue)) || - (stamps.Item2 != null && stamps.Item2 > (previous.AudiobookCover ?? DateTime.MinValue))) + try { - changed.Add(book.Id); + target.PushExternalLibraryEdit(book, files, payload); + _logger.Debug("Forwarded Grimmory edit of '{0}' to {1}", book.Title, target.Definition?.Name); + } + catch (Exception ex) + { + _logger.Warn(ex, "Failed to forward Grimmory edit of '{0}' to {1}", book.Title, target.Definition?.Name); } } - - state.CoverStamps[book.Id] = stamps; } - - return changed; } - private void ForwardBook(GrimmorySettings settings, GrimmoryBook grimmoryBook, List targets) + private List ForwardingSources() { - var bookFile = ResolveBookFile(grimmoryBook); - - if (bookFile == null) - { - _logger.Debug("No Chaptarr file matches Grimmory book {0}; skipping forward", grimmoryBook.Id); - return; - } + return _notificationFactory.GetAvailableProviders() + .OfType() + .Where(g => (g.Definition?.Settings as GrimmorySettings)?.ForwardEdits == true) + .ToList(); + } - var edition = _editionService.GetEdition(bookFile.EditionId); - var book = edition == null ? null : _bookService.GetBook(edition.BookId); + private BookFile ResolveSidecarBookFile(string sidecarPath) + { + var fileName = Path.GetFileName(sidecarPath); + var suffix = SidecarSuffixes.FirstOrDefault(s => fileName.EndsWith(s, StringComparison.OrdinalIgnoreCase)); + var directory = Path.GetDirectoryName(sidecarPath); - if (book == null) + if (suffix == null || directory.IsNullOrWhiteSpace()) { - return; + return null; } - var files = _mediaFileService.GetFilesByBook(book.Id); - var payload = BuildPayload(settings, grimmoryBook); + var baseName = fileName.Substring(0, fileName.Length - suffix.Length); - foreach (var target in targets) - { - try - { - target.PushExternalLibraryEdit(book, files, payload); - _logger.Debug("Forwarded Grimmory edit of '{0}' to {1}", book.Title, target.Definition?.Name); - } - catch (Exception ex) - { - _logger.Warn(ex, "Failed to forward Grimmory edit of '{0}' to {1}", book.Title, target.Definition?.Name); - } - } + return _mediaFileService.GetFilesWithBasePath(directory) + .FirstOrDefault(f => f?.Path.IsNotNullOrWhiteSpace() == true && + Path.GetFileNameWithoutExtension(f.Path).Equals(baseName, StringComparison.OrdinalIgnoreCase)); } - private BookFile ResolveBookFile(GrimmoryBook grimmoryBook) + private string GetRootRelativePath(string path) { - foreach (var relativePath in grimmoryBook.AllFiles().Select(f => f?.RelativePath()).Where(p => p.IsNotNullOrWhiteSpace())) - { - var osRelative = relativePath.Replace('/', Path.DirectorySeparatorChar); - - foreach (var rootFolder in _rootFolderService.All()) - { - var candidate = Path.Combine(rootFolder.Path, osRelative); - var file = _mediaFileService.GetFileWithPath(candidate); + var rootFolder = _rootFolderService.GetBestRootFolder(path); - if (file != null) - { - return file; - } - } + if (rootFolder?.Path == null || rootFolder.Path.PathEquals(path)) + { + return null; } - return null; + return rootFolder.Path.GetRelativePath(path); } private ExternalLibraryEditPayload BuildPayload(GrimmorySettings settings, GrimmoryBook grimmoryBook) diff --git a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryProxy.cs b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryProxy.cs index e23b71d8..ec43029f 100644 --- a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryProxy.cs +++ b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryProxy.cs @@ -23,7 +23,6 @@ public interface IGrimmoryProxy void UploadBookCover(GrimmorySettings settings, long bookId, byte[] image, string fileName); byte[] GetBookCover(GrimmorySettings settings, long bookId); string BuildCoverUrl(GrimmorySettings settings, long bookId); - List GetMetadataAuditEntries(GrimmorySettings settings, DateTime fromUtc); ValidationFailure Test(GrimmorySettings settings); } @@ -31,8 +30,6 @@ public class GrimmoryProxy : IGrimmoryProxy { private static readonly TimeSpan TokenCacheDuration = TimeSpan.FromMinutes(30); private static readonly TimeSpan BookListCacheDuration = TimeSpan.FromMinutes(1); - private const int AuditPageSize = 200; - private const int MaxAuditPages = 5; private readonly IHttpClient _httpClient; private readonly ICached _tokenCache; @@ -164,43 +161,6 @@ public string BuildCoverUrl(GrimmorySettings settings, long bookId) return $"{HttpUri.CombinePath(settings.Url, $"api/v1/media/book/{bookId}/cover")}?token={token}"; } - public List GetMetadataAuditEntries(GrimmorySettings settings, DateTime fromUtc) - { - var entries = new List(); - - for (var page = 0; page < MaxAuditPages; page++) - { - var pageNumber = page; - var response = ExecuteWithAuth(settings, token => - { - var request = BuildRequest(settings, "api/v1/audit-logs", token) - .AddQueryParam("action", "METADATA_UPDATED") - .AddQueryParam("size", AuditPageSize) - .AddQueryParam("page", pageNumber) - .AddQueryParam("from", fromUtc.ToString("yyyy-MM-dd'T'HH:mm:ss")) - .Build(); - - return _httpClient.Get(request); - }); - - var result = Json.Deserialize(response.Content); - - if (result?.Content == null) - { - break; - } - - entries.AddRange(result.Content); - - if (result.Last) - { - break; - } - } - - return entries; - } - public ValidationFailure Test(GrimmorySettings settings) { try @@ -217,17 +177,6 @@ public ValidationFailure Test(GrimmorySettings settings) return new ValidationFailure(nameof(GrimmorySettings.AudiobookLibraryId), "The selected audiobook library was not found in Grimmory"); } - if (settings.ForwardEdits) - { - try - { - GetMetadataAuditEntries(settings, DateTime.UtcNow.AddMinutes(-1)); - } - catch (HttpException ex) when (ex.Response?.StatusCode == HttpStatusCode.Forbidden || ex.Response?.StatusCode == HttpStatusCode.Unauthorized) - { - return new ValidationFailure(nameof(GrimmorySettings.ForwardEdits), "Forwarding Grimmory edits requires an admin user, as change detection reads the audit log"); - } - } } catch (GrimmoryAuthenticationException) { @@ -453,33 +402,6 @@ public class GrimmoryBookMetadata public DateTime? AudiobookCoverUpdatedOn { get; set; } } - public class GrimmoryAuditPage - { - [JsonProperty("content")] - public List Content { get; set; } - - [JsonProperty("last")] - public bool Last { get; set; } - } - - public class GrimmoryAuditEntry - { - [JsonProperty("id")] - public long Id { get; set; } - - [JsonProperty("username")] - public string Username { get; set; } - - [JsonProperty("entityType")] - public string EntityType { get; set; } - - [JsonProperty("entityId")] - public long? EntityId { get; set; } - - [JsonProperty("createdAt")] - public DateTime? CreatedAt { get; set; } - } - public class GrimmoryAuthenticationException : Exception { public GrimmoryAuthenticationException(string message) diff --git a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushRegistry.cs b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushRegistry.cs new file mode 100644 index 00000000..2aa90c3c --- /dev/null +++ b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushRegistry.cs @@ -0,0 +1,38 @@ +using System; +using System.Collections.Concurrent; +using System.Linq; + +namespace NzbDrone.Core.Notifications.Grimmory +{ + // Remembers books Chaptarr itself just pushed to Grimmory so the sidecar watcher can tell + // Chaptarr's own writes (Grimmory rewrites the sidecar after every metadata update, ours + // included) apart from edits a person made in Grimmory. Static because both the push + // service and the forwarder are singletons and notification instances are transient. + public static class GrimmoryPushRegistry + { + private static readonly ConcurrentDictionary RecentPushes = new ConcurrentDictionary(); + private static readonly TimeSpan Window = TimeSpan.FromMinutes(10); + + public static void RecordPush(int bookId) + { + RecentPushes[bookId] = DateTime.UtcNow; + } + + public static bool WasRecentlyPushed(int bookId) + { + var now = DateTime.UtcNow; + + foreach (var stale in RecentPushes.Where(p => now - p.Value > Window).Select(p => p.Key).ToList()) + { + RecentPushes.TryRemove(stale, out _); + } + + return RecentPushes.TryGetValue(bookId, out var pushed) && now - pushed <= Window; + } + + public static void Clear() + { + RecentPushes.Clear(); + } + } +} diff --git a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs index a458e9df..59c3c77a 100644 --- a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs +++ b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs @@ -221,6 +221,7 @@ private bool PushBook(int bookId, List fields, List connection _logger.Debug("Pushed '{0}' to Grimmory book {1} on {2}", book.Title, grimmoryBook.Id, settings.Url); anyPushed = true; + GrimmoryPushRegistry.RecordPush(book.Id); } return anyPushed; diff --git a/src/NzbDrone.Core/Notifications/Grimmory/GrimmorySettings.cs b/src/NzbDrone.Core/Notifications/Grimmory/GrimmorySettings.cs index 4e97ec6f..8ba34dc8 100644 --- a/src/NzbDrone.Core/Notifications/Grimmory/GrimmorySettings.cs +++ b/src/NzbDrone.Core/Notifications/Grimmory/GrimmorySettings.cs @@ -46,7 +46,7 @@ public class GrimmorySettings : IProviderConfig [FieldDefinition(6, Label = "Push Covers", Type = FieldType.Checkbox, HelpText = "Push Chaptarr's cover image for a book to Grimmory whenever the book is imported, retagged, or its cover changes in Chaptarr")] public bool PushCovers { get; set; } - [FieldDefinition(7, Label = "Forward Grimmory Edits", Type = FieldType.Checkbox, HelpText = "Watch Grimmory for metadata and cover edits and forward them to other connections that accept library edits. The Grimmory user must be an admin, as change detection reads the audit log")] + [FieldDefinition(7, Label = "Forward Grimmory Edits", Type = FieldType.Checkbox, HelpText = "Forward metadata and cover edits made in Grimmory to other connections that accept library edits. Requires Grimmory's sidecar 'write on update' setting so edits appear as sidecar files Chaptarr can watch for")] public bool ForwardEdits { get; set; } public NzbDroneValidationResult Validate() From 91f946ba34fe7d3ecc3666ebdf93705cf9ea7578 Mon Sep 17 00:00:00 2001 From: Benjamin Tobalt Date: Wed, 9 Sep 2026 02:53:26 -0400 Subject: [PATCH 05/18] Give targets time to settle before forwarding Grimmory edits --- .../Grimmory/GrimmoryLibraryChangeForwarder.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryLibraryChangeForwarder.cs b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryLibraryChangeForwarder.cs index 514047b0..e7cebfc2 100644 --- a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryLibraryChangeForwarder.cs +++ b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryLibraryChangeForwarder.cs @@ -29,7 +29,11 @@ public class GrimmoryLibraryChangeForwarder : IHandle>, IDisposable { - private static readonly TimeSpan DebounceDelay = TimeSpan.FromSeconds(30); + // Long enough that a target's own reaction to the same edit settles first: with + // save-to-original-file enabled Grimmory rewrites the book alongside the sidecar, and + // AudioBookShelf rescans the rewritten file ~30s later, rebuilding item metadata - a + // push that lands before that rescan is silently overwritten by it. + private static readonly TimeSpan DebounceDelay = TimeSpan.FromSeconds(90); private static readonly string[] SidecarSuffixes = { ".metadata.json", ".cover.jpg" }; private readonly INotificationFactory _notificationFactory; From 8fd10548631473175f14a963c16c2a23babd0097 Mon Sep 17 00:00:00 2001 From: Benjamin Tobalt Date: Wed, 9 Sep 2026 10:58:18 -0400 Subject: [PATCH 06/18] Consume a push's echo once so edits right after it still forward A push makes Grimmory rewrite the sidecar exactly once, so the suppression entry is spent on the first matching sidecar event; a person's edit made minutes later - previously discarded for the whole 10-minute window - forwards again. --- .../GrimmoryLibraryChangeForwarderFixture.cs | 21 +++++++++++++++++++ .../GrimmoryLibraryChangeForwarder.cs | 2 +- .../Grimmory/GrimmoryPushRegistry.cs | 19 +++++++++++++++-- 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs index e2b2e855..712b4281 100644 --- a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs +++ b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs @@ -264,6 +264,27 @@ public void should_not_forward_sidecar_written_after_chaptarrs_own_push() context.Forwarder.Dispose(); } + [Test] + public void should_forward_edit_made_after_push_echo_was_consumed() + { + var context = CreateContext(); + context.Proxy.BooksByPath[RelativePath] = BuildGrimmoryBook(); + + GrimmoryPushRegistry.RecordPush(10); + + // Grimmory rewriting the sidecar in response to Chaptarr's own push - suppressed. + context.Forwarder.QueueSidecar(context.SidecarPath); + context.Forwarder.ForwardPending(); + Assert.That(context.Target.Pushes, Is.Empty); + + // A person's edit right after - the push entry is spent, so this forwards. + context.Forwarder.QueueSidecar(context.SidecarPath); + context.Forwarder.ForwardPending(); + Assert.That(context.Target.Pushes, Has.Count.EqualTo(1)); + + context.Forwarder.Dispose(); + } + [Test] public void should_ignore_sidecar_without_matching_book_file() { diff --git a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryLibraryChangeForwarder.cs b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryLibraryChangeForwarder.cs index e7cebfc2..3f7f4d7d 100644 --- a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryLibraryChangeForwarder.cs +++ b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryLibraryChangeForwarder.cs @@ -237,7 +237,7 @@ private void ForwardSidecarChange(string sidecarPath, List sources, Ha return; } - if (GrimmoryPushRegistry.WasRecentlyPushed(book.Id)) + if (GrimmoryPushRegistry.TryConsumeRecentPush(book.Id)) { _logger.Debug("Sidecar change for '{0}' follows Chaptarr's own push; not forwarding back out", book.Title); return; diff --git a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushRegistry.cs b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushRegistry.cs index 2aa90c3c..3182b448 100644 --- a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushRegistry.cs +++ b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushRegistry.cs @@ -19,6 +19,23 @@ public static void RecordPush(int bookId) } public static bool WasRecentlyPushed(int bookId) + { + Sweep(); + + return RecentPushes.TryGetValue(bookId, out var pushed) && DateTime.UtcNow - pushed <= Window; + } + + // One-shot: a push causes exactly one sidecar rewrite in Grimmory, so the first + // matching sidecar event consumes the entry. A person's edit made shortly after a + // push is then still forwarded instead of being discarded as an echo. + public static bool TryConsumeRecentPush(int bookId) + { + Sweep(); + + return RecentPushes.TryRemove(bookId, out var pushed) && DateTime.UtcNow - pushed <= Window; + } + + private static void Sweep() { var now = DateTime.UtcNow; @@ -26,8 +43,6 @@ public static bool WasRecentlyPushed(int bookId) { RecentPushes.TryRemove(stale, out _); } - - return RecentPushes.TryGetValue(bookId, out var pushed) && now - pushed <= Window; } public static void Clear() From 059e36141a12984450240a019c714f74b33f921d Mon Sep 17 00:00:00 2001 From: Benjamin Tobalt Date: Wed, 9 Sep 2026 11:01:41 -0400 Subject: [PATCH 07/18] Record only metadata pushes for echo suppression Cover-only pushes leave no sidecar echo to consume, so recording them could swallow the next real edit made in Grimmory. --- .../Grimmory/GrimmoryPushServiceFixture.cs | 15 +++++++++++++++ .../Notifications/Grimmory/GrimmoryPushService.cs | 6 +++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs index a05c907e..14f09fda 100644 --- a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs +++ b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs @@ -235,6 +235,21 @@ public void should_record_push_in_registry_for_echo_suppression() Assert.That(GrimmoryPushRegistry.WasRecentlyPushed(11), Is.False); } + [Test] + public void should_not_record_cover_only_push_in_registry() + { + var context = CreateContext(); + context.Proxy.BooksByPath["Robin Hobb/Assassin's Apprentice/Assassin's Apprentice.epub"] = GrimmoryBookAt("Robin Hobb/Assassin's Apprentice/Assassin's Apprentice.epub"); + + context.Service.Execute(new PushGrimmoryMetadataCommand + { + BookIds = new List { 10 }, + Fields = new List { "cover" } + }); + + Assert.That(GrimmoryPushRegistry.WasRecentlyPushed(10), Is.False); + } + [Test] public void should_skip_when_book_not_found_in_grimmory() { diff --git a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs index 59c3c77a..20ca73f4 100644 --- a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs +++ b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs @@ -212,6 +212,11 @@ private bool PushBook(int bookId, List fields, List connection if (metadata.Any()) { _proxy.UpdateBookMetadata(settings, grimmoryBook.Id, metadata); + + // Only a metadata update makes Grimmory rewrite the sidecar, so only then + // is there an echo for the forwarder to consume. A cover-only push leaves + // no registry entry that could swallow the person's next edit. + GrimmoryPushRegistry.RecordPush(book.Id); } if (fields.Contains("cover", StringComparer.OrdinalIgnoreCase)) @@ -221,7 +226,6 @@ private bool PushBook(int bookId, List fields, List connection _logger.Debug("Pushed '{0}' to Grimmory book {1} on {2}", book.Title, grimmoryBook.Id, settings.Url); anyPushed = true; - GrimmoryPushRegistry.RecordPush(book.Id); } return anyPushed; From eaca6f12483c6c38741fbeda76e0a93e3003d1f2 Mon Sep 17 00:00:00 2001 From: Benjamin Tobalt Date: Wed, 9 Sep 2026 11:19:19 -0400 Subject: [PATCH 08/18] Decide push-echo suppression per sidecar event, not per batch A push's echo and a person's edit made inside the same debounce window coalesced into one batch entry and were discarded together. Each filesystem event is now checked on arrival: the first event after a push consumes the entry, duplicates inside a short shadow are absorbed, and anything later is queued and forwarded. --- .../GrimmoryLibraryChangeForwarderFixture.cs | 26 ++++++++++++++ .../GrimmoryLibraryChangeForwarder.cs | 23 ++++++++---- .../Grimmory/GrimmoryPushRegistry.cs | 36 ++++++++++++++++--- 3 files changed, 74 insertions(+), 11 deletions(-) diff --git a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs index 712b4281..51a80498 100644 --- a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs +++ b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs @@ -26,6 +26,13 @@ public class GrimmoryLibraryChangeForwarderFixture public void Setup() { GrimmoryPushRegistry.Clear(); + GrimmoryPushRegistry.EchoShadow = TimeSpan.Zero; + } + + [TearDown] + public void TearDown() + { + GrimmoryPushRegistry.EchoShadow = TimeSpan.FromSeconds(15); } public class StubProxy : DispatchProxy @@ -285,6 +292,25 @@ public void should_forward_edit_made_after_push_echo_was_consumed() context.Forwarder.Dispose(); } + [Test] + public void should_forward_edit_coalesced_into_the_same_batch_as_a_push_echo() + { + var context = CreateContext(); + context.Proxy.BooksByPath[RelativePath] = BuildGrimmoryBook(); + + GrimmoryPushRegistry.RecordPush(10); + + // The push's echo and a person's edit land inside one debounce window: the echo + // is dropped at arrival, so the edit still comes out of the shared batch. + context.Forwarder.QueueSidecar(context.SidecarPath); + context.Forwarder.QueueSidecar(context.SidecarPath); + context.Forwarder.ForwardPending(); + + Assert.That(context.Target.Pushes, Has.Count.EqualTo(1)); + + context.Forwarder.Dispose(); + } + [Test] public void should_ignore_sidecar_without_matching_book_file() { diff --git a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryLibraryChangeForwarder.cs b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryLibraryChangeForwarder.cs index 3f7f4d7d..cd0a5858 100644 --- a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryLibraryChangeForwarder.cs +++ b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryLibraryChangeForwarder.cs @@ -179,6 +179,23 @@ public void QueueSidecar(string path) return; } + try + { + var bookFile = ResolveSidecarBookFile(path); + var edition = bookFile == null ? null : _editionService.GetEdition(bookFile.EditionId); + var book = edition == null ? null : _bookService.GetBook(edition.BookId); + + if (book != null && GrimmoryPushRegistry.ShouldSuppressSidecarEvent(book.Id)) + { + _logger.Debug("Sidecar change for '{0}' follows Chaptarr's own push; not forwarding back out", book.Title); + return; + } + } + catch (Exception ex) + { + _logger.Debug(ex, "Unable to check sidecar {0} for push echo; queueing it", path); + } + _pendingSidecars[path] = 1; _debounce.Stop(); _debounce.Start(); @@ -237,12 +254,6 @@ private void ForwardSidecarChange(string sidecarPath, List sources, Ha return; } - if (GrimmoryPushRegistry.TryConsumeRecentPush(book.Id)) - { - _logger.Debug("Sidecar change for '{0}' follows Chaptarr's own push; not forwarding back out", book.Title); - return; - } - var targets = _notificationFactory.GetAvailableProviders() .OfType() .Where(t => t.AcceptsExternalLibraryEdits) diff --git a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushRegistry.cs b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushRegistry.cs index 3182b448..134a7fe8 100644 --- a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushRegistry.cs +++ b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushRegistry.cs @@ -11,8 +11,13 @@ namespace NzbDrone.Core.Notifications.Grimmory public static class GrimmoryPushRegistry { private static readonly ConcurrentDictionary RecentPushes = new ConcurrentDictionary(); + private static readonly ConcurrentDictionary ConsumedEchoes = new ConcurrentDictionary(); private static readonly TimeSpan Window = TimeSpan.FromMinutes(10); + // The filesystem raises several events for one sidecar write, so the echo of a push is + // absorbed for this long after it is first consumed. Settable so tests need not wait. + public static TimeSpan EchoShadow { get; set; } = TimeSpan.FromSeconds(15); + public static void RecordPush(int bookId) { RecentPushes[bookId] = DateTime.UtcNow; @@ -25,14 +30,29 @@ public static bool WasRecentlyPushed(int bookId) return RecentPushes.TryGetValue(bookId, out var pushed) && DateTime.UtcNow - pushed <= Window; } - // One-shot: a push causes exactly one sidecar rewrite in Grimmory, so the first - // matching sidecar event consumes the entry. A person's edit made shortly after a - // push is then still forwarded instead of being discarded as an echo. - public static bool TryConsumeRecentPush(int bookId) + // Decided per filesystem event, at arrival: the first sidecar event after a push is + // its echo (Grimmory rewrites the sidecar in response to the push) and consumes the + // entry; further events inside the shadow are duplicate notifications for that same + // write. Anything later is a real edit and must be forwarded - deciding per batch + // instead would let an echo and a genuine edit coalesce and be discarded together. + public static bool ShouldSuppressSidecarEvent(int bookId) { Sweep(); - return RecentPushes.TryRemove(bookId, out var pushed) && DateTime.UtcNow - pushed <= Window; + var now = DateTime.UtcNow; + + if (ConsumedEchoes.TryGetValue(bookId, out var consumedAt) && now - consumedAt <= EchoShadow) + { + return true; + } + + if (RecentPushes.TryRemove(bookId, out var pushed) && now - pushed <= Window) + { + ConsumedEchoes[bookId] = now; + return true; + } + + return false; } private static void Sweep() @@ -43,11 +63,17 @@ private static void Sweep() { RecentPushes.TryRemove(stale, out _); } + + foreach (var stale in ConsumedEchoes.Where(p => now - p.Value > EchoShadow).Select(p => p.Key).ToList()) + { + ConsumedEchoes.TryRemove(stale, out _); + } } public static void Clear() { RecentPushes.Clear(); + ConsumedEchoes.Clear(); } } } From a116094aa7a2594684306414de008564c9e2f364 Mon Sep 17 00:00:00 2001 From: Benjamin Tobalt Date: Wed, 9 Sep 2026 11:52:52 -0400 Subject: [PATCH 09/18] Clean up empty subfolders of a populated root during scans The mount guard skipped cleanup whenever a scan found no media files, so a granular scan of a book folder whose only file was deleted externally (e.g. removed in Grimmory) never pruned the tracked rows and the deletion never reached connections. An empty subfolder scan now cleans up when the root folder itself provably has content; root-level empty scans keep the guard. --- .../MediaFiles/DiskScanServiceFixture.cs | 26 +++++++++++++++++++ .../MediaFiles/DiskScanService.cs | 19 +++++++++++++- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/src/Chaptarr.Core.Test/MediaFiles/DiskScanServiceFixture.cs b/src/Chaptarr.Core.Test/MediaFiles/DiskScanServiceFixture.cs index 86ef1b41..a44641e6 100644 --- a/src/Chaptarr.Core.Test/MediaFiles/DiskScanServiceFixture.cs +++ b/src/Chaptarr.Core.Test/MediaFiles/DiskScanServiceFixture.cs @@ -185,6 +185,7 @@ private class DiskProviderProxy : DispatchProxy public bool FileExistsResult { get; set; } = true; public long FileLength { get; set; } = 100; public DateTime FileLastWriteTime { get; set; } = DateTime.UtcNow; + public string[] GetDirectoriesResult { get; set; } = { "/books/Some Author" }; protected override object Invoke(MethodInfo targetMethod, object[] args) { @@ -193,6 +194,11 @@ protected override object Invoke(MethodInfo targetMethod, object[] args) return FolderExistsResult; } + if (targetMethod?.Name == "GetDirectories") + { + return GetDirectoriesResult; + } + if (targetMethod?.Name == "GetFileInfo") { var fileInfo = DispatchProxy.Create(); @@ -803,6 +809,26 @@ public void scan_should_skip_cleanup_when_safe_scan_finds_no_media_files() Assert.That(cleanupProxy.CleanedPaths, Is.Empty); } + [Test] + public void scan_should_cleanup_empty_subfolder_when_root_is_populated() + { + var sut = CreateScanService( + folderExists: true, + orchestratorResult: new OrchestratorImportResult + { + CleanupSafe = true, + ScannedFilePaths = new List() + }, + out var importOrchestratorProxy, + out var cleanupProxy); + + sut.Scan(new List { "/books/Some Author/Deleted Book" }, authorIds: new List()); + + Assert.That(importOrchestratorProxy.Calls, Is.EqualTo(1)); + Assert.That(cleanupProxy.CleanedPaths, Has.Count.EqualTo(1)); + Assert.That(cleanupProxy.CleanedPaths.Single(), Is.Empty); + } + [Test] public void scan_should_cleanup_with_scanned_paths_only_when_orchestrator_result_is_cleanup_safe() { diff --git a/src/NzbDrone.Core/MediaFiles/DiskScanService.cs b/src/NzbDrone.Core/MediaFiles/DiskScanService.cs index ee5aa18d..9020f824 100644 --- a/src/NzbDrone.Core/MediaFiles/DiskScanService.cs +++ b/src/NzbDrone.Core/MediaFiles/DiskScanService.cs @@ -185,7 +185,24 @@ public void Scan(List folders = null, FilterFilesType filter = FilterFil } else if (!result.ScannedFilePaths.Any()) { - _logger.Warn("Skipping scan cleanup for {0} because the scan found no media files. This avoids wiping tracked files when a mount is visible but empty.", folder); + // An empty result is ambiguous: a mount that dropped out from under the + // scan, or files genuinely deleted (e.g. a book removed in an external + // library app). For a subfolder of a root that provably still has content + // it is the latter, and skipping would leave the pruned-on-disk book + // tracked forever; only a root-level empty scan keeps the mount guard. + var subfolderOfHealthyRoot = !rootFolder.Path.PathEquals(folder) && + _diskProvider.FolderExists(rootFolder.Path) && + _diskProvider.GetDirectories(rootFolder.Path).Any(); + + if (subfolderOfHealthyRoot) + { + _logger.Debug("Scan of {0} found no media files but root folder {1} is populated; cleaning up files tracked under it", folder, rootFolder.Path); + CleanMediaFiles(folder, result.ScannedFilePaths, rootFolder); + } + else + { + _logger.Warn("Skipping scan cleanup for {0} because the scan found no media files. This avoids wiping tracked files when a mount is visible but empty.", folder); + } } else { From 033b5406f977b2fbf4bebbfe39777844e2b5f5fc Mon Sep 17 00:00:00 2001 From: Benjamin Tobalt Date: Wed, 9 Sep 2026 12:08:28 -0400 Subject: [PATCH 10/18] Unlock pushed fields before writing so re-pushes take effect Grimmory never updates a locked field, even for the writer who locked it, and applies a request's values before its lock flags - so the first pushed value froze forever and corrected re-pushes silently no-oped. Each push now clears its target locks via toggle-field-locks first; the update itself re-locks them. Publish date also prefers the monitored edition's release date over the book's. --- .../Notifications/Grimmory/GrimmoryFixture.cs | 1 + .../GrimmoryLibraryChangeForwarderFixture.cs | 1 + .../Grimmory/GrimmoryPushServiceFixture.cs | 12 +++++++- .../Notifications/Grimmory/GrimmoryProxy.cs | 28 +++++++++++++++++++ .../Grimmory/GrimmoryPushService.cs | 7 +++-- 5 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs index be7103f9..09d40db1 100644 --- a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs +++ b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs @@ -248,6 +248,7 @@ public ValidationFailure Test(GrimmorySettings settings) public List GetLibraryBooks(GrimmorySettings settings, long libraryId, bool bypassCache = false) => new List(); public GrimmoryBook FindBookByPath(GrimmorySettings settings, long libraryId, string relativePath, bool bypassCache = false) => null; public void UpdateBookMetadata(GrimmorySettings settings, long bookId, Dictionary metadata) { } + public void UnlockBookFields(GrimmorySettings settings, long bookId, IEnumerable lockFieldNames) { } public void UploadBookCover(GrimmorySettings settings, long bookId, byte[] image, string fileName) { } public byte[] GetBookCover(GrimmorySettings settings, long bookId) => null; public string BuildCoverUrl(GrimmorySettings settings, long bookId) => string.Empty; diff --git a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs index 51a80498..31734227 100644 --- a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs +++ b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs @@ -71,6 +71,7 @@ public GrimmoryBook FindBookByPath(GrimmorySettings settings, long libraryId, st } public void UpdateBookMetadata(GrimmorySettings settings, long bookId, Dictionary metadata) { } + public void UnlockBookFields(GrimmorySettings settings, long bookId, IEnumerable lockFieldNames) { } public void UploadBookCover(GrimmorySettings settings, long bookId, byte[] image, string fileName) { } public byte[] GetBookCover(GrimmorySettings settings, long bookId) => new byte[] { 9 }; public string BuildCoverUrl(GrimmorySettings settings, long bookId) => $"http://grimmory/cover/{bookId}"; diff --git a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs index 14f09fda..400a571f 100644 --- a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs +++ b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs @@ -56,6 +56,7 @@ private class FakeGrimmoryProxy : IGrimmoryProxy public Dictionary BooksByPath { get; } = new Dictionary(StringComparer.OrdinalIgnoreCase); public List<(long BookId, Dictionary Metadata)> MetadataUpdates { get; } = new List<(long, Dictionary)>(); public List<(long BookId, string FileName)> CoverUploads { get; } = new List<(long, string)>(); + public List<(long BookId, List Fields)> Unlocks { get; } = new List<(long, List)>(); public List GetLibraries(GrimmorySettings settings) => new List(); public void RefreshLibrary(GrimmorySettings settings, long libraryId) { } @@ -66,7 +67,13 @@ public GrimmoryBook FindBookByPath(GrimmorySettings settings, long libraryId, st return BooksByPath.TryGetValue(relativePath.Replace('\\', '/'), out var book) ? book : null; } - public void UpdateBookMetadata(GrimmorySettings settings, long bookId, Dictionary metadata) => MetadataUpdates.Add((bookId, metadata)); + public void UpdateBookMetadata(GrimmorySettings settings, long bookId, Dictionary metadata) + { + Assert.That(Unlocks.Any(u => u.BookId == bookId), Is.True, "fields must be unlocked before the metadata update"); + MetadataUpdates.Add((bookId, metadata)); + } + + public void UnlockBookFields(GrimmorySettings settings, long bookId, IEnumerable lockFieldNames) => Unlocks.Add((bookId, lockFieldNames.ToList())); public void UploadBookCover(GrimmorySettings settings, long bookId, byte[] image, string fileName) => CoverUploads.Add((bookId, fileName)); public byte[] GetBookCover(GrimmorySettings settings, long bookId) => null; public string BuildCoverUrl(GrimmorySettings settings, long bookId) => $"http://grimmory/cover/{bookId}"; @@ -217,6 +224,9 @@ public void should_push_metadata_with_locks_to_matched_book() Assert.That(metadata["isbn13Locked"], Is.True); Assert.That(metadata.ContainsKey("publisher"), Is.False); }); + + Assert.That(context.Proxy.Unlocks, Has.Count.EqualTo(1)); + Assert.That(context.Proxy.Unlocks[0].Fields, Is.EquivalentTo(new[] { "titleLocked", "descriptionLocked", "isbn13Locked" })); } [Test] diff --git a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryProxy.cs b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryProxy.cs index ec43029f..af896f90 100644 --- a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryProxy.cs +++ b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryProxy.cs @@ -20,6 +20,7 @@ public interface IGrimmoryProxy List GetLibraryBooks(GrimmorySettings settings, long libraryId, bool bypassCache = false); GrimmoryBook FindBookByPath(GrimmorySettings settings, long libraryId, string relativePath, bool bypassCache = false); void UpdateBookMetadata(GrimmorySettings settings, long bookId, Dictionary metadata); + void UnlockBookFields(GrimmorySettings settings, long bookId, IEnumerable lockFieldNames); void UploadBookCover(GrimmorySettings settings, long bookId, byte[] image, string fileName); byte[] GetBookCover(GrimmorySettings settings, long bookId); string BuildCoverUrl(GrimmorySettings settings, long bookId); @@ -121,6 +122,33 @@ public void UpdateBookMetadata(GrimmorySettings settings, long bookId, Dictionar _logger.Debug("Updated Grimmory metadata for book {0}", bookId); } + // Grimmory never writes a locked field, not even for the writer who locked it, and it + // applies a request's values before its lock flags - so a push that locks its fields + // must explicitly unlock them first or every later push silently keeps the old value. + public void UnlockBookFields(GrimmorySettings settings, long bookId, IEnumerable lockFieldNames) + { + var fieldActions = lockFieldNames.Distinct().ToDictionary(f => f, _ => (object)"UNLOCK"); + + if (fieldActions.Count == 0) + { + return; + } + + ExecuteWithAuth(settings, token => + { + var request = BuildRequest(settings, "api/v1/books/metadata/toggle-field-locks", token).Build(); + request.Method = HttpMethod.Put; + request.Headers.ContentType = "application/json"; + request.SetContent(new Dictionary + { + { "bookIds", new List { bookId } }, + { "fieldActions", fieldActions } + }.ToJson()); + + return _httpClient.Execute(request); + }); + } + public void UploadBookCover(GrimmorySettings settings, long bookId, byte[] image, string fileName) { ExecuteWithAuth(settings, token => diff --git a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs index 20ca73f4..71205d93 100644 --- a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs +++ b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs @@ -211,6 +211,7 @@ private bool PushBook(int bookId, List fields, List connection if (metadata.Any()) { + _proxy.UnlockBookFields(settings, grimmoryBook.Id, metadata.Keys.Where(k => k.EndsWith("Locked", StringComparison.Ordinal))); _proxy.UpdateBookMetadata(settings, grimmoryBook.Id, metadata); // Only a metadata update makes Grimmory rewrite the sidecar, so only then @@ -307,9 +308,11 @@ void Add(string field, string grimmoryField, object value) Add("publisher", "publisher", edition?.Publisher); Add("language", "language", edition?.Language); - if (wanted.Contains("publisheddate") && book.ReleaseDate.HasValue && book.ReleaseDate.Value > DateTime.MinValue) + var releaseDate = edition?.ReleaseDate ?? book.ReleaseDate; + + if (wanted.Contains("publisheddate") && releaseDate.HasValue && releaseDate.Value > DateTime.MinValue) { - metadata["publishedDate"] = book.ReleaseDate.Value.ToString("yyyy-MM-dd"); + metadata["publishedDate"] = releaseDate.Value.ToString("yyyy-MM-dd"); metadata["publishedDateLocked"] = true; } From 70bf545fa4bf6ffbd49729bcd5f90b1fb34ab16f Mon Sep 17 00:00:00 2001 From: Benjamin Tobalt Date: Wed, 9 Sep 2026 12:27:13 -0400 Subject: [PATCH 11/18] Respect Grimmory field locks on pushes Reverts the unlock-before-write pass: a locked field in Grimmory now stays exactly as locked, including against Chaptarr's own re-pushes. Updating a locked value means unlocking it in Grimmory first. The monitored-edition release date preference stays. --- .../Notifications/Grimmory/GrimmoryFixture.cs | 1 - .../GrimmoryLibraryChangeForwarderFixture.cs | 1 - .../Grimmory/GrimmoryPushServiceFixture.cs | 12 +------- .../Notifications/Grimmory/GrimmoryProxy.cs | 28 ------------------- .../Grimmory/GrimmoryPushService.cs | 4 ++- 5 files changed, 4 insertions(+), 42 deletions(-) diff --git a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs index 09d40db1..be7103f9 100644 --- a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs +++ b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs @@ -248,7 +248,6 @@ public ValidationFailure Test(GrimmorySettings settings) public List GetLibraryBooks(GrimmorySettings settings, long libraryId, bool bypassCache = false) => new List(); public GrimmoryBook FindBookByPath(GrimmorySettings settings, long libraryId, string relativePath, bool bypassCache = false) => null; public void UpdateBookMetadata(GrimmorySettings settings, long bookId, Dictionary metadata) { } - public void UnlockBookFields(GrimmorySettings settings, long bookId, IEnumerable lockFieldNames) { } public void UploadBookCover(GrimmorySettings settings, long bookId, byte[] image, string fileName) { } public byte[] GetBookCover(GrimmorySettings settings, long bookId) => null; public string BuildCoverUrl(GrimmorySettings settings, long bookId) => string.Empty; diff --git a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs index 31734227..51a80498 100644 --- a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs +++ b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs @@ -71,7 +71,6 @@ public GrimmoryBook FindBookByPath(GrimmorySettings settings, long libraryId, st } public void UpdateBookMetadata(GrimmorySettings settings, long bookId, Dictionary metadata) { } - public void UnlockBookFields(GrimmorySettings settings, long bookId, IEnumerable lockFieldNames) { } public void UploadBookCover(GrimmorySettings settings, long bookId, byte[] image, string fileName) { } public byte[] GetBookCover(GrimmorySettings settings, long bookId) => new byte[] { 9 }; public string BuildCoverUrl(GrimmorySettings settings, long bookId) => $"http://grimmory/cover/{bookId}"; diff --git a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs index 400a571f..14f09fda 100644 --- a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs +++ b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs @@ -56,7 +56,6 @@ private class FakeGrimmoryProxy : IGrimmoryProxy public Dictionary BooksByPath { get; } = new Dictionary(StringComparer.OrdinalIgnoreCase); public List<(long BookId, Dictionary Metadata)> MetadataUpdates { get; } = new List<(long, Dictionary)>(); public List<(long BookId, string FileName)> CoverUploads { get; } = new List<(long, string)>(); - public List<(long BookId, List Fields)> Unlocks { get; } = new List<(long, List)>(); public List GetLibraries(GrimmorySettings settings) => new List(); public void RefreshLibrary(GrimmorySettings settings, long libraryId) { } @@ -67,13 +66,7 @@ public GrimmoryBook FindBookByPath(GrimmorySettings settings, long libraryId, st return BooksByPath.TryGetValue(relativePath.Replace('\\', '/'), out var book) ? book : null; } - public void UpdateBookMetadata(GrimmorySettings settings, long bookId, Dictionary metadata) - { - Assert.That(Unlocks.Any(u => u.BookId == bookId), Is.True, "fields must be unlocked before the metadata update"); - MetadataUpdates.Add((bookId, metadata)); - } - - public void UnlockBookFields(GrimmorySettings settings, long bookId, IEnumerable lockFieldNames) => Unlocks.Add((bookId, lockFieldNames.ToList())); + public void UpdateBookMetadata(GrimmorySettings settings, long bookId, Dictionary metadata) => MetadataUpdates.Add((bookId, metadata)); public void UploadBookCover(GrimmorySettings settings, long bookId, byte[] image, string fileName) => CoverUploads.Add((bookId, fileName)); public byte[] GetBookCover(GrimmorySettings settings, long bookId) => null; public string BuildCoverUrl(GrimmorySettings settings, long bookId) => $"http://grimmory/cover/{bookId}"; @@ -224,9 +217,6 @@ public void should_push_metadata_with_locks_to_matched_book() Assert.That(metadata["isbn13Locked"], Is.True); Assert.That(metadata.ContainsKey("publisher"), Is.False); }); - - Assert.That(context.Proxy.Unlocks, Has.Count.EqualTo(1)); - Assert.That(context.Proxy.Unlocks[0].Fields, Is.EquivalentTo(new[] { "titleLocked", "descriptionLocked", "isbn13Locked" })); } [Test] diff --git a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryProxy.cs b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryProxy.cs index af896f90..ec43029f 100644 --- a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryProxy.cs +++ b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryProxy.cs @@ -20,7 +20,6 @@ public interface IGrimmoryProxy List GetLibraryBooks(GrimmorySettings settings, long libraryId, bool bypassCache = false); GrimmoryBook FindBookByPath(GrimmorySettings settings, long libraryId, string relativePath, bool bypassCache = false); void UpdateBookMetadata(GrimmorySettings settings, long bookId, Dictionary metadata); - void UnlockBookFields(GrimmorySettings settings, long bookId, IEnumerable lockFieldNames); void UploadBookCover(GrimmorySettings settings, long bookId, byte[] image, string fileName); byte[] GetBookCover(GrimmorySettings settings, long bookId); string BuildCoverUrl(GrimmorySettings settings, long bookId); @@ -122,33 +121,6 @@ public void UpdateBookMetadata(GrimmorySettings settings, long bookId, Dictionar _logger.Debug("Updated Grimmory metadata for book {0}", bookId); } - // Grimmory never writes a locked field, not even for the writer who locked it, and it - // applies a request's values before its lock flags - so a push that locks its fields - // must explicitly unlock them first or every later push silently keeps the old value. - public void UnlockBookFields(GrimmorySettings settings, long bookId, IEnumerable lockFieldNames) - { - var fieldActions = lockFieldNames.Distinct().ToDictionary(f => f, _ => (object)"UNLOCK"); - - if (fieldActions.Count == 0) - { - return; - } - - ExecuteWithAuth(settings, token => - { - var request = BuildRequest(settings, "api/v1/books/metadata/toggle-field-locks", token).Build(); - request.Method = HttpMethod.Put; - request.Headers.ContentType = "application/json"; - request.SetContent(new Dictionary - { - { "bookIds", new List { bookId } }, - { "fieldActions", fieldActions } - }.ToJson()); - - return _httpClient.Execute(request); - }); - } - public void UploadBookCover(GrimmorySettings settings, long bookId, byte[] image, string fileName) { ExecuteWithAuth(settings, token => diff --git a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs index 71205d93..c1a9713c 100644 --- a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs +++ b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs @@ -211,7 +211,9 @@ private bool PushBook(int bookId, List fields, List connection if (metadata.Any()) { - _proxy.UnlockBookFields(settings, grimmoryBook.Id, metadata.Keys.Where(k => k.EndsWith("Locked", StringComparison.Ordinal))); + // Locked fields are deliberately left alone: Grimmory skips them even for + // the writer that locked them, so a re-push only lands on fields someone + // has unlocked in Grimmory (or never locked). Locks stay authoritative. _proxy.UpdateBookMetadata(settings, grimmoryBook.Id, metadata); // Only a metadata update makes Grimmory rewrite the sidecar, so only then From 858c13f23da4fbea81594dd541a92fec543bcc2c Mon Sep 17 00:00:00 2001 From: Benjamin Tobalt Date: Wed, 9 Sep 2026 13:18:57 -0400 Subject: [PATCH 12/18] Add Grimmory card to the quickstart connections Restores the archived quickstart section; the card drives the standard notification modal off the schema, so it picks up the current library dropdowns and push/forward toggles as-is. --- frontend/src/System/Quickstart/Quickstart.js | 18 ++ .../System/Quickstart/QuickstartConnector.js | 9 +- .../Quickstart/QuickstartGrimmorySection.js | 278 ++++++++++++++++++ src/NzbDrone.Core/Localization/Core/en.json | 2 + 4 files changed, 306 insertions(+), 1 deletion(-) create mode 100644 frontend/src/System/Quickstart/QuickstartGrimmorySection.js diff --git a/frontend/src/System/Quickstart/Quickstart.js b/frontend/src/System/Quickstart/Quickstart.js index 4f8dc3fd..7a7a8200 100644 --- a/frontend/src/System/Quickstart/Quickstart.js +++ b/frontend/src/System/Quickstart/Quickstart.js @@ -12,6 +12,7 @@ import translate from 'Utilities/String/translate'; import QuickstartAudioBookShelfSection from './QuickstartAudioBookShelfSection'; import QuickstartCustomFormatsSection from './QuickstartCustomFormatsSection'; import QuickstartDownloadClientsSection from './QuickstartDownloadClientsSection'; +import QuickstartGrimmorySection from './QuickstartGrimmorySection'; import QuickstartHardcoverSection from './QuickstartHardcoverSection'; import QuickstartMAMSection from './QuickstartMAMSection'; import QuickstartMatchingSection from './QuickstartMatchingSection'; @@ -98,6 +99,8 @@ function Quickstart(props) { const { hasActiveAudioBookShelf, audioBookShelfNotification, + hasActiveGrimmory, + grimmoryNotification, mamIndexer, indexersState, notificationsState, @@ -169,6 +172,19 @@ function Quickstart(props) { />
+
+ +
+
+ const audioBookShelfNotification = notifications.find((notification) => notification.implementationName === 'AudioBookShelf' ); + // Find first Grimmory notification + const grimmoryNotification = notifications.find((notification) => + notification.implementationName === 'Grimmory' + ); + // Check if proxy is configured const proxyMode = generalSettings.item?.proxyMode?.value || 'disabled'; const globalProxyId = generalSettings.item?.globalProxyId?.value; @@ -73,8 +78,10 @@ function createMapStateToProps() { return { hasActiveMAMIndexer: !!(mamIndexer && mamIndexer.enable), hasActiveAudioBookShelf: !!(audioBookShelfNotification && audioBookShelfNotification.enable), + hasActiveGrimmory: !!(grimmoryNotification && grimmoryNotification.enable), mamIndexer, audioBookShelfNotification, + grimmoryNotification, indexersState, notificationsState, downloadClientsState, diff --git a/frontend/src/System/Quickstart/QuickstartGrimmorySection.js b/frontend/src/System/Quickstart/QuickstartGrimmorySection.js new file mode 100644 index 00000000..984225f5 --- /dev/null +++ b/frontend/src/System/Quickstart/QuickstartGrimmorySection.js @@ -0,0 +1,278 @@ +import PropTypes from 'prop-types'; +import React, { Component } from 'react'; +import { connect } from 'react-redux'; +import Alert from 'Components/Alert'; +import ConfirmModal from 'Components/Modal/ConfirmModal'; +import { kinds } from 'Helpers/Props'; +import EditNotificationModalConnector from 'Settings/Notifications/Notifications/EditNotificationModalConnector'; +import { deleteNotification } from 'Store/Actions/settingsActions'; +import translate from 'Utilities/String/translate'; +import styles from './Quickstart.css'; + +class QuickstartGrimmorySection extends Component { + // + // Lifecycle + + constructor(props, context) { + super(props, context); + + this.state = { + isEditNotificationModalOpen: false, + isDeleteNotificationModalOpen: false, + pendingOpenGrimmory: false, + schemaSelectionError: false + }; + } + + componentDidMount() { + // Pre-fetch the schema so it's ready when user clicks + if (!this.props.notificationsState.isSchemaPopulated) { + this.props.fetchNotificationSchema(); + } + } + + componentDidUpdate(prevProps) { + const previousSchemaWasUsable = prevProps.notificationsState.isSchemaPopulated && + !prevProps.notificationsState.schemaError; + const schemaIsUsable = this.props.notificationsState.isSchemaPopulated && + !this.props.notificationsState.schemaError; + const schemaJustBecameUsable = !previousSchemaWasUsable && schemaIsUsable; + const schemaFetchJustFailed = prevProps.notificationsState.isSchemaFetching && + !this.props.notificationsState.isSchemaFetching && + this.props.notificationsState.schemaError; + + if (this.state.pendingOpenGrimmory && schemaJustBecameUsable) { + this.openAddGrimmoryNotification(); + } + + if (this.state.pendingOpenGrimmory && schemaFetchJustFailed) { + this.setState({ pendingOpenGrimmory: false }); + } + } + + // + // Listeners + + onButtonPress = () => { + const { + grimmoryNotification, + notificationsState, + fetchNotificationSchema + } = this.props; + + if (grimmoryNotification) { + // Edit existing Grimmory notification + this.setState({ + isEditNotificationModalOpen: true, + schemaSelectionError: false + }); + } else { + const hasUsableSchema = notificationsState.isSchemaPopulated && !notificationsState.schemaError; + + if (!hasUsableSchema) { + if (!notificationsState.isSchemaFetching && fetchNotificationSchema) { + fetchNotificationSchema(); + } + + this.setState({ + pendingOpenGrimmory: true, + schemaSelectionError: false + }); + return; + } + + this.openAddGrimmoryNotification(); + } + }; + + openAddGrimmoryNotification = () => { + const schemaItems = Array.isArray(this.props.notificationsState?.schema) ? this.props.notificationsState.schema : []; + const hasGrimmorySchema = schemaItems.some((schemaItem) => schemaItem.implementation === 'Grimmory'); + + if (!hasGrimmorySchema) { + this.setState({ + pendingOpenGrimmory: false, + schemaSelectionError: true + }); + return; + } + + this.props.selectNotificationSchema({ implementation: 'Grimmory' }); + this.setState({ + isEditNotificationModalOpen: true, + pendingOpenGrimmory: false, + schemaSelectionError: false + }); + }; + + onEditNotificationModalClose = () => { + this.setState({ + isEditNotificationModalOpen: false, + pendingOpenGrimmory: false, + schemaSelectionError: false + }); + + // Refresh notifications to ensure we have the latest state + // This will update the button text and state after deletion + if (this.props.fetchNotifications) { + this.props.fetchNotifications(); + } + }; + + onDeleteNotificationPress = () => { + this.setState({ + isEditNotificationModalOpen: false, + isDeleteNotificationModalOpen: true + }); + }; + + onDeleteNotificationModalClose = () => { + this.setState({ isDeleteNotificationModalOpen: false }); + }; + + onConfirmDeleteNotification = () => { + const { grimmoryNotification } = this.props; + + if (grimmoryNotification) { + this.props.deleteNotification({ id: grimmoryNotification.id }); + } + + this.onDeleteNotificationModalClose(); + }; + + onTestConnectionSuccess = () => { + // Mark this section as interacted when test connection succeeds + const { markSectionInteracted } = this.props; + if (markSectionInteracted) { + markSectionInteracted({ section: 'grimmory' }); + } + }; + + // + // Render + + render() { + const { + hasActiveGrimmory, + grimmoryNotification + } = this.props; + + const { + isEditNotificationModalOpen, + isDeleteNotificationModalOpen, + pendingOpenGrimmory, + schemaSelectionError + } = this.state; + + const buttonText = grimmoryNotification ? + translate('ConfigureName', { name: 'Grimmory' }) : + translate('AddName', { name: 'Grimmory' }); + const isAddSchemaLoading = !grimmoryNotification && + (this.props.notificationsState.isSchemaFetching || pendingOpenGrimmory); + const schemaError = !this.props.notificationsState.isSchemaFetching && + (this.props.notificationsState.schemaError || schemaSelectionError); + + if (this.props.compact) { + return ( + <> +
+ +
+ + { + schemaError && + + {translate('QuickstartUnableToLoadNotificationOptions')} + + } + + + + + + ); + } + + return ( +
+

+ {translate('QuickstartGrimmoryConnectHeader')} +

+ {!hasActiveGrimmory && ( +
+ {translate('QuickstartGrimmoryConnectDescription')} +
+ )} + +
+ +
+ + { + schemaError && + + {translate('QuickstartUnableToLoadNotificationOptions')} + + } + + + + +
+ ); + } +} + +QuickstartGrimmorySection.propTypes = { + hasActiveGrimmory: PropTypes.bool, + grimmoryNotification: PropTypes.object, + compact: PropTypes.bool, + notificationsState: PropTypes.object.isRequired, + fetchNotificationSchema: PropTypes.func.isRequired, + selectNotificationSchema: PropTypes.func.isRequired, + deleteNotification: PropTypes.func.isRequired, + markSectionInteracted: PropTypes.func, + fetchNotifications: PropTypes.func +}; + +export default connect(null, { deleteNotification })(QuickstartGrimmorySection); diff --git a/src/NzbDrone.Core/Localization/Core/en.json b/src/NzbDrone.Core/Localization/Core/en.json index e869f89c..af8f3368 100644 --- a/src/NzbDrone.Core/Localization/Core/en.json +++ b/src/NzbDrone.Core/Localization/Core/en.json @@ -1535,6 +1535,8 @@ "QuickstartCustomFormatsTitle": "3. Custom Formats (Optional)", "QuickstartDownloadClientsDescription": "Configure download clients to handle your audiobook downloads. Download clients manage torrent and usenet downloads.", "QuickstartDownloadClientsTitle": "2. Download Clients", + "QuickstartGrimmoryConnectDescription": "Connect to Grimmory so Chaptarr can refresh its libraries after imports, renames and deletes, push metadata and covers, and forward edits made in Grimmory to other connections.", + "QuickstartGrimmoryConnectHeader": "Connect Grimmory", "QuickstartHardcoverConnectDescription": "Connect to Hardcover to enable direct metadata searching and library-based features.", "QuickstartMamAddMyAnonaMouse": "Add MyAnonaMouse", "QuickstartMamSectionDescription": "Add indexers to search for audiobooks.", From 40082c798779becf66b48daf0f6335727ed6327a Mon Sep 17 00:00:00 2001 From: Benjamin Tobalt Date: Wed, 9 Sep 2026 17:30:47 -0400 Subject: [PATCH 13/18] Let sibling Grimmory connections receive forwarded edits Grimmory now implements IExternalLibraryEditTarget, so an edit made in one instance reaches every other configured instance alongside the ABS and content server targets; the forwarder already excludes the source. Applies descriptive fields and covers per each connection's push toggles, resolves the book by root-relative path in that connection's own libraries, respects its locks, and records the push so the target's own sidecar rewrite is absorbed instead of ping- ponging - Grimmory's no-change detection ends the chain once values converge. --- .../Notifications/Grimmory/GrimmoryFixture.cs | 1 + .../GrimmoryLibraryChangeForwarderFixture.cs | 82 +++++++++++--- .../Grimmory/GrimmoryPushServiceFixture.cs | 2 +- .../Notifications/Grimmory/Grimmory.cs | 102 +++++++++++++++++- 4 files changed, 170 insertions(+), 17 deletions(-) diff --git a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs index be7103f9..c79f0fe9 100644 --- a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs +++ b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs @@ -204,6 +204,7 @@ private static NzbDrone.Core.Notifications.Grimmory.Grimmory CreateSubject(FakeG return new NzbDrone.Core.Notifications.Grimmory.Grimmory( proxy, DispatchProxy.Create(), + null, new CacheManager(), LogManager.GetLogger("GrimmoryFixture")) { diff --git a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs index 51a80498..8e36b1b7 100644 --- a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs +++ b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs @@ -60,6 +60,7 @@ private static T Stub(out StubProxy stub) private class ScriptedGrimmoryProxy : IGrimmoryProxy { public Dictionary BooksByPath { get; } = new Dictionary(StringComparer.OrdinalIgnoreCase); + public List<(long BookId, Dictionary Metadata)> MetadataUpdates { get; } = new List<(long, Dictionary)>(); public List GetLibraries(GrimmorySettings settings) => new List(); public void RefreshLibrary(GrimmorySettings settings, long libraryId) { } @@ -70,7 +71,7 @@ public GrimmoryBook FindBookByPath(GrimmorySettings settings, long libraryId, st return BooksByPath.TryGetValue(relativePath.Replace('\\', '/'), out var book) ? book : null; } - public void UpdateBookMetadata(GrimmorySettings settings, long bookId, Dictionary metadata) { } + public void UpdateBookMetadata(GrimmorySettings settings, long bookId, Dictionary metadata) => MetadataUpdates.Add((bookId, metadata)); public void UploadBookCover(GrimmorySettings settings, long bookId, byte[] image, string fileName) { } public byte[] GetBookCover(GrimmorySettings settings, long bookId) => new byte[] { 9 }; public string BuildCoverUrl(GrimmorySettings settings, long bookId) => $"http://grimmory/cover/{bookId}"; @@ -99,6 +100,7 @@ public override ValidationResult Test() private class Context { public ScriptedGrimmoryProxy Proxy; + public ScriptedGrimmoryProxy SiblingProxy; public GrimmoryLibraryChangeForwarder Forwarder; public TestEditTarget Target; public string SidecarPath; @@ -132,7 +134,7 @@ private static GrimmoryBook BuildGrimmoryBook() }; } - private static Context CreateContext(int targetDefinitionId = 2) + private static Context CreateContext(int targetDefinitionId = 2, bool withSibling = false) { var context = new Context(); var proxy = new ScriptedGrimmoryProxy(); @@ -150,7 +152,19 @@ private static Context CreateContext(int targetDefinitionId = 2) var commandQueue = Stub(out var commandStub); commandStub.Handlers["Push"] = _ => null; - var source = new NzbDrone.Core.Notifications.Grimmory.Grimmory(proxy, commandQueue, new CacheManager(), LogManager.GetLogger("test")) + var rootPath = @"C:\books".AsOsAgnostic(); + var bookDir = Path.Combine(rootPath, "Robin Hobb", "Assassin's Apprentice"); + var bookFilePath = Path.Combine(bookDir, "Assassin's Apprentice.epub"); + context.SidecarPath = Path.Combine(bookDir, "Assassin's Apprentice.metadata.json"); + context.CoverSidecarPath = Path.Combine(bookDir, "Assassin's Apprentice.cover.jpg"); + + var bookFile = new BookFile { Id = 40, EditionId = 30, Path = bookFilePath, MediaType = "ebook" }; + + var rootFolderService = Stub(out var rootStub); + rootStub.Handlers["All"] = _ => new List { new RootFolder { Id = 1, Path = rootPath } }; + rootStub.Handlers["GetBestRootFolder"] = _ => new RootFolder { Id = 1, Path = rootPath }; + + var source = new NzbDrone.Core.Notifications.Grimmory.Grimmory(proxy, commandQueue, rootFolderService, new CacheManager(), LogManager.GetLogger("test")) { Definition = new NotificationDefinition { Id = 1, Name = "Grimmory", Settings = settings } }; @@ -161,20 +175,30 @@ private static Context CreateContext(int targetDefinitionId = 2) }; context.Target = target; - var factory = Stub(out var factoryStub); - factoryStub.Handlers["GetAvailableProviders"] = _ => new List { source, target }; + var providers = new List { source, target }; - var rootPath = @"C:\books".AsOsAgnostic(); - var bookDir = Path.Combine(rootPath, "Robin Hobb", "Assassin's Apprentice"); - var bookFilePath = Path.Combine(bookDir, "Assassin's Apprentice.epub"); - context.SidecarPath = Path.Combine(bookDir, "Assassin's Apprentice.metadata.json"); - context.CoverSidecarPath = Path.Combine(bookDir, "Assassin's Apprentice.cover.jpg"); + if (withSibling) + { + var siblingProxy = new ScriptedGrimmoryProxy(); + context.SiblingProxy = siblingProxy; - var bookFile = new BookFile { Id = 40, EditionId = 30, Path = bookFilePath, MediaType = "ebook" }; + var siblingSettings = new GrimmorySettings + { + Url = "http://grimmory-b:6060", + Username = "chaptarr", + Password = "secret", + EbookLibraryId = EbookLibraryId, + PushMetadata = true + }; + + providers.Add(new NzbDrone.Core.Notifications.Grimmory.Grimmory(siblingProxy, commandQueue, rootFolderService, new CacheManager(), LogManager.GetLogger("test")) + { + Definition = new NotificationDefinition { Id = 3, Name = "Grimmory B", Settings = siblingSettings } + }); + } - var rootFolderService = Stub(out var rootStub); - rootStub.Handlers["All"] = _ => new List { new RootFolder { Id = 1, Path = rootPath } }; - rootStub.Handlers["GetBestRootFolder"] = _ => new RootFolder { Id = 1, Path = rootPath }; + var factory = Stub(out var factoryStub); + factoryStub.Handlers["GetAvailableProviders"] = _ => providers; var mediaFileService = Stub(out var mediaFileStub); mediaFileStub.Handlers["GetFilesWithBasePath"] = args => string.Equals((string)args[0], bookDir, StringComparison.OrdinalIgnoreCase) @@ -325,6 +349,36 @@ public void should_ignore_sidecar_without_matching_book_file() context.Forwarder.Dispose(); } + [Test] + public void should_forward_edit_to_sibling_grimmory_connection() + { + var context = CreateContext(withSibling: true); + context.Proxy.BooksByPath[RelativePath] = BuildGrimmoryBook(); + context.SiblingProxy.BooksByPath[RelativePath] = new GrimmoryBook + { + Id = 500, + LibraryId = EbookLibraryId, + PrimaryFile = BuildGrimmoryBook().PrimaryFile + }; + + context.Forwarder.QueueSidecar(context.SidecarPath); + context.Forwarder.ForwardPending(); + + Assert.That(context.SiblingProxy.MetadataUpdates, Has.Count.EqualTo(1)); + + var (bookId, metadata) = context.SiblingProxy.MetadataUpdates[0]; + + Assert.Multiple(() => + { + Assert.That(bookId, Is.EqualTo(500)); + Assert.That(metadata["description"], Is.EqualTo("Edited in Grimmory.")); + Assert.That(metadata["seriesName"], Is.EqualTo("Farseer")); + Assert.That(context.Proxy.MetadataUpdates, Is.Empty, "the source instance must not receive its own edit back"); + }); + + context.Forwarder.Dispose(); + } + [Test] public void should_skip_target_sharing_the_sources_definition() { diff --git a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs index 14f09fda..77b4c2a1 100644 --- a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs +++ b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs @@ -106,7 +106,7 @@ private static Context CreateContext(bool pushMetadata = true, bool pushCovers = return null; }; - var provider = new NzbDrone.Core.Notifications.Grimmory.Grimmory(proxy, commandQueue, new CacheManager(), LogManager.GetLogger("test")) + var provider = new NzbDrone.Core.Notifications.Grimmory.Grimmory(proxy, commandQueue, null, new CacheManager(), LogManager.GetLogger("test")) { Definition = new NotificationDefinition { Id = 1, Name = "Grimmory", Settings = settings } }; diff --git a/src/NzbDrone.Core/Notifications/Grimmory/Grimmory.cs b/src/NzbDrone.Core/Notifications/Grimmory/Grimmory.cs index f46b097b..f0fcc388 100644 --- a/src/NzbDrone.Core/Notifications/Grimmory/Grimmory.cs +++ b/src/NzbDrone.Core/Notifications/Grimmory/Grimmory.cs @@ -8,20 +8,23 @@ using NzbDrone.Core.Books; using NzbDrone.Core.MediaFiles; using NzbDrone.Core.Messaging.Commands; +using NzbDrone.Core.RootFolders; namespace NzbDrone.Core.Notifications.Grimmory { - public class Grimmory : NotificationBase + public class Grimmory : NotificationBase, IExternalLibraryEditTarget { private readonly IGrimmoryProxy _proxy; private readonly IManageCommandQueue _commandQueueManager; + private readonly IRootFolderService _rootFolderService; private readonly Logger _logger; private readonly ICached _pendingLibrariesCache; - public Grimmory(IGrimmoryProxy proxy, IManageCommandQueue commandQueueManager, ICacheManager cacheManager, Logger logger) + public Grimmory(IGrimmoryProxy proxy, IManageCommandQueue commandQueueManager, IRootFolderService rootFolderService, ICacheManager cacheManager, Logger logger) { _proxy = proxy; _commandQueueManager = commandQueueManager; + _rootFolderService = rootFolderService; _logger = logger; _pendingLibrariesCache = cacheManager.GetRollingCache(GetType(), "pendingLibraries", TimeSpan.FromDays(1)); } @@ -240,6 +243,101 @@ public override object RequestAction(string action, IDictionary return new { }; } + public bool AcceptsExternalLibraryEdits => Settings.PushMetadata || Settings.PushCovers; + + // Applies an edit made in another library service - typically a sibling Grimmory + // connection, since the forwarder excludes the source itself - so every instance + // converges on the same values. Identity fields stay Chaptarr's per the forwarder + // convention, locked fields on this instance keep winning, and the target rewrites + // its own sidecar in response, so the push is recorded to absorb that echo. + public void PushExternalLibraryEdit(Book book, List files, ExternalLibraryEditPayload payload) + { + if (book == null || payload == null || files == null || files.Empty()) + { + return; + } + + var libraryId = book.MediaType == BookMediaType.Ebook ? Settings.EbookLibraryId : Settings.AudiobookLibraryId; + + if (libraryId <= 0) + { + return; + } + + var grimmoryBook = files + .Select(f => GetRootRelativePath(f?.Path)) + .Where(p => p.IsNotNullOrWhiteSpace()) + .Select(p => _proxy.FindBookByPath(Settings, libraryId, p, bypassCache: true)) + .FirstOrDefault(b => b != null); + + if (grimmoryBook == null) + { + return; + } + + var metadata = new Dictionary(); + + if (Settings.PushMetadata) + { + void Add(string field, object value) + { + if (value != null && (!(value is string s) || s.IsNotNullOrWhiteSpace())) + { + metadata[field] = value; + } + } + + Add("description", payload.Description); + Add("publisher", payload.Publisher); + Add("seriesName", payload.SeriesName); + Add("seriesNumber", payload.SeriesPosition); + Add("language", payload.Languages?.FirstOrDefault()); + Add("isbn13", payload.Identifiers?.GetValueOrDefault("isbn")); + Add("asin", payload.Identifiers?.GetValueOrDefault("asin")); + Add("goodreadsId", payload.Identifiers?.GetValueOrDefault("goodreads")); + + if (payload.PublishedDate.HasValue) + { + metadata["publishedDate"] = payload.PublishedDate.Value.ToString("yyyy-MM-dd"); + } + + if (payload.Genres?.Any() == true) + { + metadata["categories"] = payload.Genres; + } + + if (metadata.Any()) + { + _proxy.UpdateBookMetadata(Settings, grimmoryBook.Id, metadata); + GrimmoryPushRegistry.RecordPush(book.Id); + } + } + + if (Settings.PushCovers && payload.CoverBytes?.Length > 0) + { + _proxy.UploadBookCover(Settings, grimmoryBook.Id, payload.CoverBytes, "cover.jpg"); + } + + _logger.Debug("Applied external library edit of '{0}' to Grimmory book {1} on {2}", book.Title, grimmoryBook.Id, Settings.Url); + } + + private string GetRootRelativePath(string path) + { + if (path.IsNullOrWhiteSpace()) + { + return null; + } + + var rootFolder = _rootFolderService.GetBestRootFolder(path); + + if (rootFolder?.Path == null || rootFolder.Path.PathEquals(path)) + { + return null; + } + + return rootFolder.Path.GetRelativePath(path); + } + private string QueueKey => $"{Settings.Url}:{Settings.Username}"; private void QueueRefresh(long libraryId, string reason) From 273fdf82b4c34fd87a46fe2916c59876361664e3 Mon Sep 17 00:00:00 2001 From: Benjamin Tobalt Date: Wed, 9 Sep 2026 18:36:54 -0400 Subject: [PATCH 14/18] Record the Grimmory push before the metadata update Grimmory writes its sidecar while the update request is still in flight, so the filesystem event could reach the forwarder before the registry entry existed and get forwarded back out as a real edit. --- src/NzbDrone.Core/Notifications/Grimmory/Grimmory.cs | 4 +++- .../Notifications/Grimmory/GrimmoryPushService.cs | 11 ++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/NzbDrone.Core/Notifications/Grimmory/Grimmory.cs b/src/NzbDrone.Core/Notifications/Grimmory/Grimmory.cs index f0fcc388..acae01d7 100644 --- a/src/NzbDrone.Core/Notifications/Grimmory/Grimmory.cs +++ b/src/NzbDrone.Core/Notifications/Grimmory/Grimmory.cs @@ -308,8 +308,10 @@ void Add(string field, object value) if (metadata.Any()) { - _proxy.UpdateBookMetadata(Settings, grimmoryBook.Id, metadata); + // Recorded before the update so the target's sidecar rewrite, which can + // reach the forwarder before this call returns, is absorbed as an echo. GrimmoryPushRegistry.RecordPush(book.Id); + _proxy.UpdateBookMetadata(Settings, grimmoryBook.Id, metadata); } } diff --git a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs index c1a9713c..56e42319 100644 --- a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs +++ b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs @@ -211,15 +211,16 @@ private bool PushBook(int bookId, List fields, List connection if (metadata.Any()) { + // Recorded before the update: Grimmory writes the sidecar during the call, + // so the filesystem event can reach the forwarder before the call returns. + // Only a metadata update makes Grimmory rewrite the sidecar - a cover-only + // push leaves no entry that could swallow the person's next edit. + GrimmoryPushRegistry.RecordPush(book.Id); + // Locked fields are deliberately left alone: Grimmory skips them even for // the writer that locked them, so a re-push only lands on fields someone // has unlocked in Grimmory (or never locked). Locks stay authoritative. _proxy.UpdateBookMetadata(settings, grimmoryBook.Id, metadata); - - // Only a metadata update makes Grimmory rewrite the sidecar, so only then - // is there an echo for the forwarder to consume. A cover-only push leaves - // no registry entry that could swallow the person's next edit. - GrimmoryPushRegistry.RecordPush(book.Id); } if (fields.Contains("cover", StringComparer.OrdinalIgnoreCase)) From 7145232978f1810257d64500f9d4872e60435278 Mon Sep 17 00:00:00 2001 From: Benjamin Tobalt Date: Wed, 9 Sep 2026 19:02:45 -0400 Subject: [PATCH 15/18] Trim the Grimmory connector to the codebase conventions Strip the reviewer-facing narration from the Grimmory connector, keeping only the comments that record an external constraint: Grimmory rewriting the sidecar during a metadata update, its async refresh, its handling of locked fields, and the AudioBookShelf rescan the forwarder debounces past. Drop IGrimmoryProxy.GetLibraryBooks, which no caller outside the proxy used, and inline the single-use edition lookup in GrimmoryPushService. File the two Grimmory push strings in their alphabetical place in en.json instead of the middle of the GoTo* run, and translate the push dialog's field labels the way QuickstartMatchingSection does rather than hardcoding English. Publish Date reuses the existing PublishedDate key. --- .../src/Grimmory/GrimmoryPushModalContent.js | 4 +-- .../System/Quickstart/QuickstartConnector.js | 1 - .../Quickstart/QuickstartGrimmorySection.js | 5 ---- .../Notifications/Grimmory/GrimmoryFixture.cs | 1 - .../GrimmoryLibraryChangeForwarderFixture.cs | 1 - .../Grimmory/GrimmoryPushServiceFixture.cs | 1 - src/NzbDrone.Core/Localization/Core/en.json | 6 ++-- .../MediaFiles/DiskScanService.cs | 7 ++--- .../MediaFiles/MediaFileDeletionService.cs | 4 +-- .../Notifications/Grimmory/Grimmory.cs | 9 ++---- .../GrimmoryLibraryChangeForwarder.cs | 17 ++++------- .../Notifications/Grimmory/GrimmoryProxy.cs | 7 ++--- .../Grimmory/GrimmoryPushRegistry.cs | 15 ++++------ .../Grimmory/GrimmoryPushService.cs | 30 +++++++------------ .../Grimmory/PushGrimmoryMetadataCommand.cs | 4 +-- 15 files changed, 36 insertions(+), 76 deletions(-) diff --git a/frontend/src/Grimmory/GrimmoryPushModalContent.js b/frontend/src/Grimmory/GrimmoryPushModalContent.js index cb11786e..6b84822f 100644 --- a/frontend/src/Grimmory/GrimmoryPushModalContent.js +++ b/frontend/src/Grimmory/GrimmoryPushModalContent.js @@ -17,7 +17,7 @@ const grimmoryFields = [ { name: 'series', label: 'Series' }, { name: 'description', label: 'Description' }, { name: 'publisher', label: 'Publisher' }, - { name: 'publisheddate', label: 'Publish Date' }, + { name: 'publisheddate', label: 'PublishedDate' }, { name: 'language', label: 'Language' }, { name: 'tags', label: 'Tags' }, { name: 'identifiers', label: 'Identifiers' } @@ -93,7 +93,7 @@ class GrimmoryPushModalContent extends Component { onChange={this.onFieldChange} />
-
{field.label}
+
{translate(field.label)}
{preview}
); diff --git a/frontend/src/System/Quickstart/QuickstartConnector.js b/frontend/src/System/Quickstart/QuickstartConnector.js index 35a43cde..2c5c255e 100644 --- a/frontend/src/System/Quickstart/QuickstartConnector.js +++ b/frontend/src/System/Quickstart/QuickstartConnector.js @@ -53,7 +53,6 @@ function createMapStateToProps() { notification.implementationName === 'AudioBookShelf' ); - // Find first Grimmory notification const grimmoryNotification = notifications.find((notification) => notification.implementationName === 'Grimmory' ); diff --git a/frontend/src/System/Quickstart/QuickstartGrimmorySection.js b/frontend/src/System/Quickstart/QuickstartGrimmorySection.js index 984225f5..7c182fac 100644 --- a/frontend/src/System/Quickstart/QuickstartGrimmorySection.js +++ b/frontend/src/System/Quickstart/QuickstartGrimmorySection.js @@ -25,7 +25,6 @@ class QuickstartGrimmorySection extends Component { } componentDidMount() { - // Pre-fetch the schema so it's ready when user clicks if (!this.props.notificationsState.isSchemaPopulated) { this.props.fetchNotificationSchema(); } @@ -61,7 +60,6 @@ class QuickstartGrimmorySection extends Component { } = this.props; if (grimmoryNotification) { - // Edit existing Grimmory notification this.setState({ isEditNotificationModalOpen: true, schemaSelectionError: false @@ -112,8 +110,6 @@ class QuickstartGrimmorySection extends Component { schemaSelectionError: false }); - // Refresh notifications to ensure we have the latest state - // This will update the button text and state after deletion if (this.props.fetchNotifications) { this.props.fetchNotifications(); } @@ -141,7 +137,6 @@ class QuickstartGrimmorySection extends Component { }; onTestConnectionSuccess = () => { - // Mark this section as interacted when test connection succeeds const { markSectionInteracted } = this.props; if (markSectionInteracted) { markSectionInteracted({ section: 'grimmory' }); diff --git a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs index c79f0fe9..22120ba3 100644 --- a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs +++ b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs @@ -246,7 +246,6 @@ public ValidationFailure Test(GrimmorySettings settings) return null; } - public List GetLibraryBooks(GrimmorySettings settings, long libraryId, bool bypassCache = false) => new List(); public GrimmoryBook FindBookByPath(GrimmorySettings settings, long libraryId, string relativePath, bool bypassCache = false) => null; public void UpdateBookMetadata(GrimmorySettings settings, long bookId, Dictionary metadata) { } public void UploadBookCover(GrimmorySettings settings, long bookId, byte[] image, string fileName) { } diff --git a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs index 8e36b1b7..001b2f7e 100644 --- a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs +++ b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryLibraryChangeForwarderFixture.cs @@ -64,7 +64,6 @@ private class ScriptedGrimmoryProxy : IGrimmoryProxy public List GetLibraries(GrimmorySettings settings) => new List(); public void RefreshLibrary(GrimmorySettings settings, long libraryId) { } - public List GetLibraryBooks(GrimmorySettings settings, long libraryId, bool bypassCache = false) => BooksByPath.Values.ToList(); public GrimmoryBook FindBookByPath(GrimmorySettings settings, long libraryId, string relativePath, bool bypassCache = false) { diff --git a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs index 77b4c2a1..b7a0b144 100644 --- a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs +++ b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs @@ -59,7 +59,6 @@ private class FakeGrimmoryProxy : IGrimmoryProxy public List GetLibraries(GrimmorySettings settings) => new List(); public void RefreshLibrary(GrimmorySettings settings, long libraryId) { } - public List GetLibraryBooks(GrimmorySettings settings, long libraryId, bool bypassCache = false) => BooksByPath.Values.ToList(); public GrimmoryBook FindBookByPath(GrimmorySettings settings, long libraryId, string relativePath, bool bypassCache = false) { diff --git a/src/NzbDrone.Core/Localization/Core/en.json b/src/NzbDrone.Core/Localization/Core/en.json index af8f3368..1af42ecc 100644 --- a/src/NzbDrone.Core/Localization/Core/en.json +++ b/src/NzbDrone.Core/Localization/Core/en.json @@ -442,6 +442,7 @@ "CountIndexersSelected": "{selectedCount} indexer(s) selected", "CountMore": "{count} more", "Country": "Country", + "Cover": "Cover", "CreateEmptyAuthorFolders": "Create empty author folders", "CreateEmptyAuthorFoldersHelpText": "Create missing author folders during disk scan", "CreateGroup": "Create group", @@ -818,8 +819,6 @@ "GlobalProxy": "Default Proxy", "GlobalProxyHelpText": "Default proxy used when proxy routing applies. In Indexers Only mode it is used for indexers without a specific override. In Proxy Everything mode it is also used for metadata, covers, notifications, updates, and other app HTTP requests.", "GoToAuthorListing": "Go to author listing", - "GrimmoryPush": "Grimmory Push", - "GrimmoryPushDescriptionInterp": "Choose which fields to push to Grimmory for {0} book(s). Pushed fields are locked in Grimmory so its own metadata refreshes do not overwrite them; anything left unticked is untouched.", "GoToInteractiveSearch": "Go to Interactive Search", "GoToInterp": "Go to {0}", "GoodreadsImportListBookshelfRequired": "Select at least one bookshelf.", @@ -830,6 +829,8 @@ "GrabReleaseMessageText": "Chaptarr was unable to determine which author and book this release was for. Chaptarr may be unable to automatically import this release. Do you want to grab '{0}'?", "GrabSelected": "Download Now", "GraphicAudio": "Graphic Audio", + "GrimmoryPush": "Grimmory Push", + "GrimmoryPushDescriptionInterp": "Choose which fields to push to Grimmory for {0} book(s). Pushed fields are locked in Grimmory so its own metadata refreshes do not overwrite them; anything left unticked is untouched.", "Group": "Group", "Hardcover": "Hardcover", "HardcoverApiKeyPlaceholder": "Paste your Hardcover API key here", @@ -859,6 +860,7 @@ "ISBN": "ISBN", "IconForCutoffUnmet": "Icon for Cutoff Unmet", "IconTooltip": "Scheduled", + "Identifiers": "Identifiers", "IfYouDontAddAnImportListExclusionAndTheAuthorHasAMetadataProfileOtherThanNoneThenThisBookMayBeReaddedDuringTheNextAuthorRefresh": "If you don't add an import list exclusion and the author has a metadata profile other than 'None' then this book may be re-added during the next author refresh.", "IgnoreDeletedBooks": "Ignore Deleted Books", "IgnoreDownload": "Ignore Download", diff --git a/src/NzbDrone.Core/MediaFiles/DiskScanService.cs b/src/NzbDrone.Core/MediaFiles/DiskScanService.cs index 9020f824..ab2229f1 100644 --- a/src/NzbDrone.Core/MediaFiles/DiskScanService.cs +++ b/src/NzbDrone.Core/MediaFiles/DiskScanService.cs @@ -185,11 +185,8 @@ public void Scan(List folders = null, FilterFilesType filter = FilterFil } else if (!result.ScannedFilePaths.Any()) { - // An empty result is ambiguous: a mount that dropped out from under the - // scan, or files genuinely deleted (e.g. a book removed in an external - // library app). For a subfolder of a root that provably still has content - // it is the latter, and skipping would leave the pruned-on-disk book - // tracked forever; only a root-level empty scan keeps the mount guard. + // An empty result means either a dropped mount or a genuine delete; + // only a root-level scan can still be a dropped mount. var subfolderOfHealthyRoot = !rootFolder.Path.PathEquals(folder) && _diskProvider.FolderExists(rootFolder.Path) && _diskProvider.GetDirectories(rootFolder.Path).Any(); diff --git a/src/NzbDrone.Core/MediaFiles/MediaFileDeletionService.cs b/src/NzbDrone.Core/MediaFiles/MediaFileDeletionService.cs index f87ba438..57a25a70 100644 --- a/src/NzbDrone.Core/MediaFiles/MediaFileDeletionService.cs +++ b/src/NzbDrone.Core/MediaFiles/MediaFileDeletionService.cs @@ -306,8 +306,8 @@ public void HandleAsync(BookDeletedEvent message) CleanupEmptyFolders(author, folder); } - // Notification providers queue work on OnBookDelete and drain it when this event - // signals the files are actually off the disk; author deletes already publish it. + // Providers queue work on OnBookDelete and drain it on this event; author + // deletes already publish it. _eventAggregator.PublishEvent(new DeleteCompletedEvent()); } diff --git a/src/NzbDrone.Core/Notifications/Grimmory/Grimmory.cs b/src/NzbDrone.Core/Notifications/Grimmory/Grimmory.cs index acae01d7..6d025e86 100644 --- a/src/NzbDrone.Core/Notifications/Grimmory/Grimmory.cs +++ b/src/NzbDrone.Core/Notifications/Grimmory/Grimmory.cs @@ -245,11 +245,6 @@ public override object RequestAction(string action, IDictionary public bool AcceptsExternalLibraryEdits => Settings.PushMetadata || Settings.PushCovers; - // Applies an edit made in another library service - typically a sibling Grimmory - // connection, since the forwarder excludes the source itself - so every instance - // converges on the same values. Identity fields stay Chaptarr's per the forwarder - // convention, locked fields on this instance keep winning, and the target rewrites - // its own sidecar in response, so the push is recorded to absorb that echo. public void PushExternalLibraryEdit(Book book, List files, ExternalLibraryEditPayload payload) { if (book == null || payload == null || files == null || files.Empty()) @@ -308,8 +303,8 @@ void Add(string field, object value) if (metadata.Any()) { - // Recorded before the update so the target's sidecar rewrite, which can - // reach the forwarder before this call returns, is absorbed as an echo. + // Grimmory writes the sidecar during the update, so the entry has to + // exist before the call for the forwarder to absorb the echo. GrimmoryPushRegistry.RecordPush(book.Id); _proxy.UpdateBookMetadata(Settings, grimmoryBook.Id, metadata); } diff --git a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryLibraryChangeForwarder.cs b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryLibraryChangeForwarder.cs index cd0a5858..04ba02d5 100644 --- a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryLibraryChangeForwarder.cs +++ b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryLibraryChangeForwarder.cs @@ -15,24 +15,17 @@ namespace NzbDrone.Core.Notifications.Grimmory { - // Forwards metadata and cover edits made in Grimmory to connections that implement - // IExternalLibraryEditTarget. Grimmory's database is remote, but with sidecar - // write-on-update enabled it rewrites ".metadata.json" (and, when configured, - // ".cover.jpg") next to the book after every edit - so, like the calibre forwarder - // watching metadata.db, watching the root folders for sidecar writes is the change - // signal. No polling. Chaptarr's own pushes also rewrite the sidecar; those are filtered - // through GrimmoryPushRegistry rather than by author, so edits a person makes in Grimmory - // are forwarded even when they use the connection's own account. + // Grimmory's database is remote, but with sidecar write-on-update enabled it rewrites + // ".metadata.json" (and ".cover.jpg") next to the book after every edit, so + // watching the root folders is the only change signal available. public class GrimmoryLibraryChangeForwarder : IHandle, IHandle>, IHandle>, IDisposable { - // Long enough that a target's own reaction to the same edit settles first: with - // save-to-original-file enabled Grimmory rewrites the book alongside the sidecar, and - // AudioBookShelf rescans the rewritten file ~30s later, rebuilding item metadata - a - // push that lands before that rescan is silently overwritten by it. + // With save-to-original-file enabled Grimmory rewrites the book alongside the sidecar + // and AudioBookShelf rescans it ~30s later, overwriting anything pushed before that. private static readonly TimeSpan DebounceDelay = TimeSpan.FromSeconds(90); private static readonly string[] SidecarSuffixes = { ".metadata.json", ".cover.jpg" }; diff --git a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryProxy.cs b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryProxy.cs index ec43029f..130a2a0f 100644 --- a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryProxy.cs +++ b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryProxy.cs @@ -17,7 +17,6 @@ public interface IGrimmoryProxy { List GetLibraries(GrimmorySettings settings); void RefreshLibrary(GrimmorySettings settings, long libraryId); - List GetLibraryBooks(GrimmorySettings settings, long libraryId, bool bypassCache = false); GrimmoryBook FindBookByPath(GrimmorySettings settings, long libraryId, string relativePath, bool bypassCache = false); void UpdateBookMetadata(GrimmorySettings settings, long bookId, Dictionary metadata); void UploadBookCover(GrimmorySettings settings, long bookId, byte[] image, string fileName); @@ -67,7 +66,7 @@ public void RefreshLibrary(GrimmorySettings settings, long libraryId) _logger.Debug("Triggered Grimmory refresh for library {0}", libraryId); } - public List GetLibraryBooks(GrimmorySettings settings, long libraryId, bool bypassCache = false) + private List GetLibraryBooks(GrimmorySettings settings, long libraryId, bool bypassCache) { var cacheKey = $"{settings.Url}:{settings.Username}:{libraryId}"; @@ -176,7 +175,6 @@ public ValidationFailure Test(GrimmorySettings settings) { return new ValidationFailure(nameof(GrimmorySettings.AudiobookLibraryId), "The selected audiobook library was not found in Grimmory"); } - } catch (GrimmoryAuthenticationException) { @@ -280,8 +278,7 @@ private string Login(GrimmorySettings settings) private static HttpRequestBuilder BuildRequest(GrimmorySettings settings, string relativePath, string token) { - // Status codes are handled in ExecuteWithAuth so a 401/403 can trigger a re-login - // instead of surfacing as an HttpException from the client. + // SuppressHttpError so ExecuteWithAuth sees a 401/403 and can re-login. return new HttpRequestBuilder(HttpUri.CombinePath(settings.Url, relativePath)) { SuppressHttpError = true diff --git a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushRegistry.cs b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushRegistry.cs index 134a7fe8..7038ee4d 100644 --- a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushRegistry.cs +++ b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushRegistry.cs @@ -4,10 +4,8 @@ namespace NzbDrone.Core.Notifications.Grimmory { - // Remembers books Chaptarr itself just pushed to Grimmory so the sidecar watcher can tell - // Chaptarr's own writes (Grimmory rewrites the sidecar after every metadata update, ours - // included) apart from edits a person made in Grimmory. Static because both the push - // service and the forwarder are singletons and notification instances are transient. + // Grimmory rewrites the sidecar after every metadata update, Chaptarr's own pushes + // included, so the watcher needs to know which writes were ours. public static class GrimmoryPushRegistry { private static readonly ConcurrentDictionary RecentPushes = new ConcurrentDictionary(); @@ -15,7 +13,7 @@ public static class GrimmoryPushRegistry private static readonly TimeSpan Window = TimeSpan.FromMinutes(10); // The filesystem raises several events for one sidecar write, so the echo of a push is - // absorbed for this long after it is first consumed. Settable so tests need not wait. + // absorbed for this long after it is first consumed. public static TimeSpan EchoShadow { get; set; } = TimeSpan.FromSeconds(15); public static void RecordPush(int bookId) @@ -30,11 +28,8 @@ public static bool WasRecentlyPushed(int bookId) return RecentPushes.TryGetValue(bookId, out var pushed) && DateTime.UtcNow - pushed <= Window; } - // Decided per filesystem event, at arrival: the first sidecar event after a push is - // its echo (Grimmory rewrites the sidecar in response to the push) and consumes the - // entry; further events inside the shadow are duplicate notifications for that same - // write. Anything later is a real edit and must be forwarded - deciding per batch - // instead would let an echo and a genuine edit coalesce and be discarded together. + // Decided per event at arrival rather than per batch: batching would let an echo and + // a genuine edit coalesce and be discarded together. public static bool ShouldSuppressSidecarEvent(int bookId) { Sweep(); diff --git a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs index 56e42319..b260942e 100644 --- a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs +++ b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs @@ -82,9 +82,8 @@ public static List ToggleFields(GrimmorySettings settings) return fields; } - // Fires when a book's cover/metadata is updated in Chaptarr (e.g. through the UI). - // Author-scoped cover events are deliberately ignored - they fire during routine - // author refreshes and would fan out into pushes for every book of the author. + // Author-scoped cover events are ignored: they fire during routine author refreshes + // and would fan out into a push for every book of the author. public void Handle(MediaCoversUpdatedEvent message) { var book = message.Book; @@ -186,7 +185,8 @@ private bool PushBook(int bookId, List fields, List connection } var author = _authorService.GetAuthor(book.AuthorId); - var edition = ResolveEdition(book); + var editions = _editionService.GetEditionsByBook(book.Id); + var edition = editions.FirstOrDefault(e => e.Monitored) ?? editions.FirstOrDefault(); var anyPushed = false; foreach (var connection in connections) @@ -211,15 +211,12 @@ private bool PushBook(int bookId, List fields, List connection if (metadata.Any()) { - // Recorded before the update: Grimmory writes the sidecar during the call, - // so the filesystem event can reach the forwarder before the call returns. - // Only a metadata update makes Grimmory rewrite the sidecar - a cover-only - // push leaves no entry that could swallow the person's next edit. + // Only a metadata update makes Grimmory rewrite the sidecar, and it writes + // it during the call, so the entry has to exist before the call. GrimmoryPushRegistry.RecordPush(book.Id); - // Locked fields are deliberately left alone: Grimmory skips them even for - // the writer that locked them, so a re-push only lands on fields someone - // has unlocked in Grimmory (or never locked). Locks stay authoritative. + // Grimmory skips locked fields even for the writer that locked them, so a + // re-push only lands on fields someone has unlocked there. _proxy.UpdateBookMetadata(settings, grimmoryBook.Id, metadata); } @@ -264,8 +261,8 @@ private GrimmoryBook FindGrimmoryBook(GrimmorySettings settings, long libraryId, return null; } - // Freshly imported books only exist in Grimmory once its (async) refresh has - // scanned them, so re-fetch the library list until the book shows up. + // A freshly imported book only exists in Grimmory once its async refresh has + // scanned it, so re-fetch until it shows up. Thread.Sleep(WaitForBookInterval); bypassCache = true; } @@ -283,13 +280,6 @@ private string GetRootRelativePath(string path) return rootFolder.Path.GetRelativePath(path); } - private Edition ResolveEdition(Book book) - { - var editions = _editionService.GetEditionsByBook(book.Id); - - return editions.FirstOrDefault(e => e.Monitored) ?? editions.FirstOrDefault(); - } - private Dictionary BuildMetadata(Book book, Author author, Edition edition, List fields) { var metadata = new Dictionary(); diff --git a/src/NzbDrone.Core/Notifications/Grimmory/PushGrimmoryMetadataCommand.cs b/src/NzbDrone.Core/Notifications/Grimmory/PushGrimmoryMetadataCommand.cs index 91d36cd9..21584f81 100644 --- a/src/NzbDrone.Core/Notifications/Grimmory/PushGrimmoryMetadataCommand.cs +++ b/src/NzbDrone.Core/Notifications/Grimmory/PushGrimmoryMetadataCommand.cs @@ -9,8 +9,8 @@ public class PushGrimmoryMetadataCommand : Command public List Fields { get; set; } = new List(); - // Set for pushes queued right after an import, when Grimmory may not have scanned the - // new files yet - the executor then waits for the book to appear before giving up. + // Grimmory's refresh is async, so a push queued right after an import has to wait + // for the book to appear. public bool WaitForBook { get; set; } public override bool SendUpdatesToClient => true; From f84d9a446c5a1225490c0cd88864f56308ca5e8b Mon Sep 17 00:00:00 2001 From: Benjamin Tobalt Date: Wed, 9 Sep 2026 19:14:38 -0400 Subject: [PATCH 16/18] Mirror Chaptarr-started pushes to the other library edit targets Grimmory rewrites its sidecar in response to a push and the forwarder drops that event as an echo, so Audiobookshelf and the other connections never saw covers or metadata that were pushed from Chaptarr. --- .../Grimmory/GrimmoryPushServiceFixture.cs | 77 ++++++++- .../Grimmory/GrimmoryPushService.cs | 150 +++++++++++++++++- 2 files changed, 218 insertions(+), 9 deletions(-) diff --git a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs index b7a0b144..4f075d27 100644 --- a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs +++ b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs @@ -72,12 +72,32 @@ public GrimmoryBook FindBookByPath(GrimmorySettings settings, long libraryId, st public ValidationFailure Test(GrimmorySettings settings) => null; } + private class TestEditTarget : NotificationBase, IExternalLibraryEditTarget + { + public List<(Book Book, ExternalLibraryEditPayload Payload)> Pushes { get; } = new List<(Book, ExternalLibraryEditPayload)>(); + + public override string Name => "TestTarget"; + public override string Link => string.Empty; + public bool AcceptsExternalLibraryEdits => true; + + public void PushExternalLibraryEdit(Book book, List files, ExternalLibraryEditPayload payload) + { + Pushes.Add((book, payload)); + } + + public override ValidationResult Test() + { + return new ValidationResult(); + } + } + private class Context { public FakeGrimmoryProxy Proxy; public GrimmoryPushService Service; public List PushedCommands = new List(); public GrimmorySettings Settings; + public TestEditTarget Target; } private static Context CreateContext(bool pushMetadata = true, bool pushCovers = true, string coverPath = null) @@ -110,8 +130,13 @@ private static Context CreateContext(bool pushMetadata = true, bool pushCovers = Definition = new NotificationDefinition { Id = 1, Name = "Grimmory", Settings = settings } }; + context.Target = new TestEditTarget + { + Definition = new NotificationDefinition { Id = 2, Name = "TestTarget", Settings = new GrimmorySettings() } + }; + var factory = Stub(out var factoryStub); - factoryStub.Handlers["GetAvailableProviders"] = _ => new List { provider }; + factoryStub.Handlers["GetAvailableProviders"] = _ => new List { provider, context.Target }; var book = new Book { @@ -249,6 +274,56 @@ public void should_not_record_cover_only_push_in_registry() Assert.That(GrimmoryPushRegistry.WasRecentlyPushed(10), Is.False); } + [Test] + public void should_mirror_push_to_other_edit_targets() + { + var coverFile = Path.GetTempFileName(); + File.WriteAllBytes(coverFile, new byte[] { 1, 2, 3 }); + + try + { + var context = CreateContext(coverPath: coverFile); + context.Proxy.BooksByPath["Robin Hobb/Assassin's Apprentice/Assassin's Apprentice.epub"] = GrimmoryBookAt("Robin Hobb/Assassin's Apprentice/Assassin's Apprentice.epub"); + + context.Service.Execute(new PushGrimmoryMetadataCommand + { + BookIds = new List { 10 }, + Fields = new List { "description", "publisher", "tags", "cover" } + }); + + Assert.That(context.Target.Pushes, Has.Count.EqualTo(1)); + + var payload = context.Target.Pushes[0].Payload; + + Assert.Multiple(() => + { + Assert.That(payload.Description, Is.EqualTo("Edition overview.")); + Assert.That(payload.Publisher, Is.EqualTo("Voyager")); + Assert.That(payload.Genres, Is.EqualTo(new List { "fantasy" })); + Assert.That(payload.CoverBytes, Is.EqualTo(new byte[] { 1, 2, 3 })); + Assert.That(payload.Title, Is.Null); + }); + } + finally + { + File.Delete(coverFile); + } + } + + [Test] + public void should_not_mirror_push_when_nothing_was_pushed_to_grimmory() + { + var context = CreateContext(); + + context.Service.Execute(new PushGrimmoryMetadataCommand + { + BookIds = new List { 10 }, + Fields = new List { "title" } + }); + + Assert.That(context.Target.Pushes, Is.Empty); + } + [Test] public void should_skip_when_book_not_found_in_grimmory() { diff --git a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs index b260942e..61f9006a 100644 --- a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs +++ b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs @@ -229,9 +229,47 @@ private bool PushBook(int bookId, List fields, List connection anyPushed = true; } + if (anyPushed) + { + PushToOtherTargets(book, files, edition, fields, connections); + } + return anyPushed; } + // Grimmory rewrites its sidecar in response to this push and the forwarder drops that + // event as an echo, so the other connections have to be told here or they keep showing + // the pre-push values. + private void PushToOtherTargets(Book book, List files, Edition edition, List fields, List connections) + { + var alreadyPushed = new HashSet(connections.Select(c => c.Definition.Id)); + + var targets = _notificationFactory.GetAvailableProviders() + .OfType() + .Where(t => t.AcceptsExternalLibraryEdits && !alreadyPushed.Contains(t.Definition.Id)) + .ToList(); + + if (targets.Empty()) + { + return; + } + + var payload = BuildEditPayload(book, edition, fields); + + foreach (var target in targets) + { + try + { + target.PushExternalLibraryEdit(book, files, payload); + _logger.Debug("Mirrored the push of '{0}' to {1}", book.Title, target.Definition.Name); + } + catch (Exception ex) + { + _logger.Warn(ex, "Failed to mirror the push of '{0}' to {1}", book.Title, target.Definition.Name); + } + } + } + private GrimmoryBook FindGrimmoryBook(GrimmorySettings settings, long libraryId, List files, bool waitForBook) { var deadline = waitForBook ? DateTime.UtcNow + WaitForBookTimeout : DateTime.UtcNow; @@ -360,25 +398,121 @@ void Add(string field, string grimmoryField, object value) return metadata; } - private void PushCover(GrimmorySettings settings, long grimmoryBookId, Book book, Edition edition) + private ExternalLibraryEditPayload BuildEditPayload(Book book, Edition edition, List fields) { - var cover = (edition?.Images ?? book.Images)?.FirstOrDefault(i => i.CoverType == MediaCoverTypes.Cover); + var wanted = new HashSet(fields, StringComparer.OrdinalIgnoreCase); + var payload = new ExternalLibraryEditPayload(); - if (cover == null) + if (wanted.Contains("title")) { - _logger.Debug("No cover known for '{0}'; skipping cover push", book.Title); - return; + payload.Title = edition?.Title ?? book.Title; } - var coverPath = _coverMapper.GetCoverPath(book.Id, MediaCoverEntity.Book, cover.CoverType, cover.Extension); + if (wanted.Contains("description")) + { + payload.Description = edition?.Overview ?? book.Overview; + } + + if (wanted.Contains("publisher")) + { + payload.Publisher = edition?.Publisher; + } - if (coverPath.IsNullOrWhiteSpace() || !File.Exists(coverPath)) + if (wanted.Contains("language") && edition?.Language.IsNotNullOrWhiteSpace() == true) + { + payload.Languages = new List { edition.Language }; + } + + var releaseDate = edition?.ReleaseDate ?? book.ReleaseDate; + + if (wanted.Contains("publisheddate") && releaseDate.HasValue && releaseDate.Value > DateTime.MinValue) { - _logger.Debug("Cover file for '{0}' not present at {1}; skipping cover push", book.Title, coverPath); + payload.PublishedDate = releaseDate; + } + + if (wanted.Contains("series")) + { + var seriesLink = book.SeriesLinks?.FirstOrDefault(l => l?.Series?.Value?.Title.IsNotNullOrWhiteSpace() == true); + + if (seriesLink != null) + { + payload.SeriesName = seriesLink.Series.Value.Title; + + if (double.TryParse(seriesLink.Position, out var position)) + { + payload.SeriesPosition = position; + } + } + } + + if (wanted.Contains("tags") && book.Genres?.Any() == true) + { + payload.Genres = book.Genres; + } + + if (wanted.Contains("identifiers")) + { + var identifiers = new Dictionary(); + + if (edition?.Isbn13.IsNotNullOrWhiteSpace() == true) + { + identifiers["isbn"] = edition.Isbn13; + } + + if (edition?.Asin.IsNotNullOrWhiteSpace() == true) + { + identifiers["asin"] = edition.Asin; + } + + if (edition?.ForeignEditionId.IsNotNullOrWhiteSpace() == true) + { + identifiers["goodreads"] = edition.ForeignEditionId; + } + + if (identifiers.Any()) + { + payload.Identifiers = identifiers; + } + } + + if (wanted.Contains("cover")) + { + var coverPath = GetCoverPath(book, edition); + + if (coverPath != null) + { + payload.CoverBytes = File.ReadAllBytes(coverPath); + } + } + + return payload; + } + + private void PushCover(GrimmorySettings settings, long grimmoryBookId, Book book, Edition edition) + { + var coverPath = GetCoverPath(book, edition); + + if (coverPath == null) + { + _logger.Debug("No cover file on disk for '{0}'; skipping cover push", book.Title); return; } _proxy.UploadBookCover(settings, grimmoryBookId, File.ReadAllBytes(coverPath), Path.GetFileName(coverPath)); } + + private string GetCoverPath(Book book, Edition edition) + { + var cover = (edition?.Images ?? book.Images)?.FirstOrDefault(i => i.CoverType == MediaCoverTypes.Cover); + + if (cover == null) + { + return null; + } + + var path = _coverMapper.GetCoverPath(book.Id, MediaCoverEntity.Book, cover.CoverType, cover.Extension); + + return path.IsNotNullOrWhiteSpace() && File.Exists(path) ? path : null; + } } } From 8b819441a2dc2c4be08357c0f015fb18387ed624 Mon Sep 17 00:00:00 2001 From: Benjamin Tobalt Date: Wed, 9 Sep 2026 19:27:43 -0400 Subject: [PATCH 17/18] Lock the authors, categories and cover Chaptarr pushes Those three were written without their lock flag, so Grimmory's own metadata refresh could overwrite them. The cover is uploaded before the lock is set because Grimmory rejects an upload outright once the cover is locked, and an already-locked cover is left alone for the same reason. --- .../Grimmory/GrimmoryPushServiceFixture.cs | 38 +++++++++++++++-- .../Notifications/Grimmory/GrimmoryProxy.cs | 3 ++ .../Grimmory/GrimmoryPushService.cs | 42 +++++++++---------- 3 files changed, 59 insertions(+), 24 deletions(-) diff --git a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs index 4f075d27..98084343 100644 --- a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs +++ b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryPushServiceFixture.cs @@ -223,7 +223,7 @@ public void should_push_metadata_with_locks_to_matched_book() context.Service.Execute(new PushGrimmoryMetadataCommand { BookIds = new List { 10 }, - Fields = new List { "title", "description", "authors", "identifiers" } + Fields = new List { "title", "description", "authors", "identifiers", "tags" } }); Assert.That(context.Proxy.MetadataUpdates, Has.Count.EqualTo(1)); @@ -237,6 +237,9 @@ public void should_push_metadata_with_locks_to_matched_book() Assert.That(metadata["titleLocked"], Is.True); Assert.That(metadata["description"], Is.EqualTo("Edition overview.")); Assert.That(metadata["authors"], Is.EqualTo(new List { "Robin Hobb" })); + Assert.That(metadata["authorsLocked"], Is.True); + Assert.That(metadata["categories"], Is.EqualTo(new List { "fantasy" })); + Assert.That(metadata["categoriesLocked"], Is.True); Assert.That(metadata["isbn13"], Is.EqualTo("9780007562252")); Assert.That(metadata["isbn13Locked"], Is.True); Assert.That(metadata.ContainsKey("publisher"), Is.False); @@ -260,7 +263,7 @@ public void should_record_push_in_registry_for_echo_suppression() } [Test] - public void should_not_record_cover_only_push_in_registry() + public void should_not_record_push_in_registry_when_nothing_was_sent() { var context = CreateContext(); context.Proxy.BooksByPath["Robin Hobb/Assassin's Apprentice/Assassin's Apprentice.epub"] = GrimmoryBookAt("Robin Hobb/Assassin's Apprentice/Assassin's Apprentice.epub"); @@ -274,6 +277,34 @@ public void should_not_record_cover_only_push_in_registry() Assert.That(GrimmoryPushRegistry.WasRecentlyPushed(10), Is.False); } + [Test] + public void should_leave_a_locked_cover_alone() + { + var coverFile = Path.GetTempFileName(); + File.WriteAllBytes(coverFile, new byte[] { 1, 2, 3 }); + + try + { + var context = CreateContext(coverPath: coverFile); + var grimmoryBook = GrimmoryBookAt("Robin Hobb/Assassin's Apprentice/Assassin's Apprentice.epub"); + grimmoryBook.Metadata = new GrimmoryBookMetadata { CoverLocked = true }; + context.Proxy.BooksByPath["Robin Hobb/Assassin's Apprentice/Assassin's Apprentice.epub"] = grimmoryBook; + + context.Service.Execute(new PushGrimmoryMetadataCommand + { + BookIds = new List { 10 }, + Fields = new List { "cover" } + }); + + Assert.That(context.Proxy.CoverUploads, Is.Empty); + Assert.That(context.Proxy.MetadataUpdates, Is.Empty); + } + finally + { + File.Delete(coverFile); + } + } + [Test] public void should_mirror_push_to_other_edit_targets() { @@ -356,7 +387,8 @@ public void should_upload_cover_when_cover_field_selected_and_file_exists() }); Assert.That(context.Proxy.CoverUploads, Has.Count.EqualTo(1)); - Assert.That(context.Proxy.MetadataUpdates, Is.Empty); + Assert.That(context.Proxy.MetadataUpdates, Has.Count.EqualTo(1)); + Assert.That(context.Proxy.MetadataUpdates[0].Metadata.Keys, Is.EqualTo(new[] { "coverLocked" })); } finally { diff --git a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryProxy.cs b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryProxy.cs index 130a2a0f..9103c684 100644 --- a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryProxy.cs +++ b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryProxy.cs @@ -392,6 +392,9 @@ public class GrimmoryBookMetadata [JsonProperty("categories")] public List Categories { get; set; } + [JsonProperty("coverLocked")] + public bool? CoverLocked { get; set; } + [JsonProperty("coverUpdatedOn")] public DateTime? CoverUpdatedOn { get; set; } diff --git a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs index 61f9006a..113203ec 100644 --- a/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs +++ b/src/NzbDrone.Core/Notifications/Grimmory/GrimmoryPushService.cs @@ -208,21 +208,32 @@ private bool PushBook(int bookId, List fields, List connection } var metadata = BuildMetadata(book, author, edition, fields); + var coverPath = fields.Contains("cover", StringComparer.OrdinalIgnoreCase) ? GetCoverPath(book, edition) : null; - if (metadata.Any()) + // Grimmory rejects a cover upload outright once the cover is locked, where it + // skips a locked metadata field silently. + var pushCover = coverPath != null && grimmoryBook.Metadata?.CoverLocked != true; + + if (metadata.Any() || pushCover) { - // Only a metadata update makes Grimmory rewrite the sidecar, and it writes - // it during the call, so the entry has to exist before the call. + // Grimmory writes the sidecar during these calls, so the entry has to exist + // before the first of them. GrimmoryPushRegistry.RecordPush(book.Id); + } - // Grimmory skips locked fields even for the writer that locked them, so a - // re-push only lands on fields someone has unlocked there. - _proxy.UpdateBookMetadata(settings, grimmoryBook.Id, metadata); + if (pushCover) + { + _proxy.UploadBookCover(settings, grimmoryBook.Id, File.ReadAllBytes(coverPath), Path.GetFileName(coverPath)); + + // Locked only once the upload has landed, for the same reason. + metadata["coverLocked"] = true; } - if (fields.Contains("cover", StringComparer.OrdinalIgnoreCase)) + if (metadata.Any()) { - PushCover(settings, grimmoryBook.Id, book, edition); + // Grimmory skips locked fields even for the writer that locked them, so a + // re-push only lands on fields someone has unlocked there. + _proxy.UpdateBookMetadata(settings, grimmoryBook.Id, metadata); } _logger.Debug("Pushed '{0}' to Grimmory book {1} on {2}", book.Title, grimmoryBook.Id, settings.Url); @@ -350,6 +361,7 @@ void Add(string field, string grimmoryField, object value) if (wanted.Contains("authors") && author?.Name.IsNotNullOrWhiteSpace() == true) { metadata["authors"] = new List { author.Name }; + metadata["authorsLocked"] = true; } if (wanted.Contains("series")) @@ -372,6 +384,7 @@ void Add(string field, string grimmoryField, object value) if (wanted.Contains("tags") && book.Genres?.Any() == true) { metadata["categories"] = book.Genres; + metadata["categoriesLocked"] = true; } if (wanted.Contains("identifiers")) @@ -488,19 +501,6 @@ private ExternalLibraryEditPayload BuildEditPayload(Book book, Edition edition, return payload; } - private void PushCover(GrimmorySettings settings, long grimmoryBookId, Book book, Edition edition) - { - var coverPath = GetCoverPath(book, edition); - - if (coverPath == null) - { - _logger.Debug("No cover file on disk for '{0}'; skipping cover push", book.Title); - return; - } - - _proxy.UploadBookCover(settings, grimmoryBookId, File.ReadAllBytes(coverPath), Path.GetFileName(coverPath)); - } - private string GetCoverPath(Book book, Edition edition) { var cover = (edition?.Images ?? book.Images)?.FirstOrDefault(i => i.CoverType == MediaCoverTypes.Cover); From 11a7e3b37c128aaa53f2b672f8e0476dbd3a676d Mon Sep 17 00:00:00 2001 From: Benjamin Tobalt Date: Wed, 9 Sep 2026 22:16:09 -0400 Subject: [PATCH 18/18] Refresh and push for books that arrive via library scans A file adopted by a disk scan raises BookImportedEvent with NewDownload false, which NotificationService dropped before any provider saw it. A book added through Grimmory and picked up by a scan therefore got no library refresh and no metadata or cover push until something else touched it. Add the NotifyOnLibraryImports opt-in to INotification and NotificationBase and let NotificationService deliver library imports to the providers that declare it, along with an OnLibraryFileAdded hook for providers that work per file. Grimmory declares it whenever it is already configured to push, so a scan pickup goes through the existing OnReleaseImport path: refresh the matching library, then push the selected fields once Grimmory's own scan has the book. The three shared notification files are byte-identical to the ones on feature/calibre-content-server-connector so the two branches still merge without conflict. --- .../Notifications/Grimmory/GrimmoryFixture.cs | 16 ++++++ .../Notifications/Grimmory/Grimmory.cs | 2 + .../Notifications/INotification.cs | 3 ++ .../Notifications/NotificationBase.cs | 7 +++ .../Notifications/NotificationService.cs | 53 ++++++++++++++++++- 5 files changed, 80 insertions(+), 1 deletion(-) diff --git a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs index 22120ba3..b5922e54 100644 --- a/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs +++ b/src/Chaptarr.Core.Test/Notifications/Grimmory/GrimmoryFixture.cs @@ -37,6 +37,22 @@ public void should_not_refresh_at_event_time_and_refresh_on_process_queue() Assert.That(subject.HasPendingQueue, Is.False); } + [Test] + public void should_notify_on_library_imports_only_when_pushing() + { + var subject = CreateSubject(new FakeGrimmoryProxy()); + var settings = (GrimmorySettings)subject.Definition.Settings; + + Assert.That(subject.NotifyOnLibraryImports, Is.False); + + settings.PushMetadata = true; + Assert.That(subject.NotifyOnLibraryImports, Is.True); + + settings.PushMetadata = false; + settings.PushCovers = true; + Assert.That(subject.NotifyOnLibraryImports, Is.True); + } + [Test] public void should_dedupe_multiple_events_into_single_refresh() { diff --git a/src/NzbDrone.Core/Notifications/Grimmory/Grimmory.cs b/src/NzbDrone.Core/Notifications/Grimmory/Grimmory.cs index 6d025e86..7fcf1603 100644 --- a/src/NzbDrone.Core/Notifications/Grimmory/Grimmory.cs +++ b/src/NzbDrone.Core/Notifications/Grimmory/Grimmory.cs @@ -32,6 +32,8 @@ public Grimmory(IGrimmoryProxy proxy, IManageCommandQueue commandQueueManager, I public override string Name => "Grimmory"; public override string Link => "https://github.com/grimmory-tools/grimmory"; + public override bool NotifyOnLibraryImports => Settings.PushMetadata || Settings.PushCovers; + private class GrimmoryUpdateQueue { public HashSet PendingLibraries { get; } = new HashSet(); diff --git a/src/NzbDrone.Core/Notifications/INotification.cs b/src/NzbDrone.Core/Notifications/INotification.cs index 2bbda7d8..595efbb8 100644 --- a/src/NzbDrone.Core/Notifications/INotification.cs +++ b/src/NzbDrone.Core/Notifications/INotification.cs @@ -8,6 +8,7 @@ namespace NzbDrone.Core.Notifications public interface INotification : IProvider { string Link { get; } + bool NotifyOnLibraryImports { get; } void OnGrab(GrabMessage grabMessage); void OnReleaseImport(BookDownloadMessage message); @@ -22,6 +23,7 @@ public interface INotification : IProvider void OnDownloadFailure(DownloadFailedMessage message); void OnImportFailure(BookDownloadMessage message); void OnBookRetag(BookRetagMessage message); + void OnLibraryFileAdded(BookFile bookFile, Book book); void ProcessQueue(); bool HasPendingQueue { get; } bool SupportsOnGrab { get; } @@ -39,5 +41,6 @@ public interface INotification : IProvider bool SupportsOnDownloadFailure { get; } bool SupportsOnImportFailure { get; } bool SupportsOnBookRetag { get; } + bool SupportsOnLibraryFileAdded { get; } } } diff --git a/src/NzbDrone.Core/Notifications/NotificationBase.cs b/src/NzbDrone.Core/Notifications/NotificationBase.cs index ed322aa8..78115658 100644 --- a/src/NzbDrone.Core/Notifications/NotificationBase.cs +++ b/src/NzbDrone.Core/Notifications/NotificationBase.cs @@ -38,6 +38,8 @@ public abstract class NotificationBase : INotification public abstract string Name { get; } + public virtual bool NotifyOnLibraryImports => false; + public Type ConfigContract => typeof(TSettings); public virtual ProviderMessage Message => null; @@ -93,6 +95,10 @@ public virtual void OnImportFailure(BookDownloadMessage message) { } + public virtual void OnLibraryFileAdded(BookFile bookFile, Book book) + { + } + public virtual void OnBookRetag(BookRetagMessage message) { } @@ -121,6 +127,7 @@ public virtual void ProcessQueue() public bool SupportsOnDownloadFailure => HasConcreteImplementation("OnDownloadFailure"); public bool SupportsOnImportFailure => HasConcreteImplementation("OnImportFailure"); public bool SupportsOnBookRetag => HasConcreteImplementation("OnBookRetag"); + public bool SupportsOnLibraryFileAdded => HasConcreteImplementation("OnLibraryFileAdded"); public bool SupportsOnApplicationUpdate => HasConcreteImplementation("OnApplicationUpdate"); protected TSettings Settings => (TSettings)Definition.Settings; diff --git a/src/NzbDrone.Core/Notifications/NotificationService.cs b/src/NzbDrone.Core/Notifications/NotificationService.cs index 4616921c..da663b65 100644 --- a/src/NzbDrone.Core/Notifications/NotificationService.cs +++ b/src/NzbDrone.Core/Notifications/NotificationService.cs @@ -25,6 +25,7 @@ public class NotificationService IHandle, IHandle, IHandle, + IHandle, IHandle, IHandle, IHandle, @@ -193,7 +194,9 @@ public void Handle(BookGrabbedEvent message) public void Handle(BookImportedEvent message) { - if (!message.NewDownload) + var isLibraryImport = !message.NewDownload; + + if (isLibraryImport && _notificationFactory.OnReleaseImportEnabled().All(n => !n.NotifyOnLibraryImports)) { _logger.Info("Skipping OnReleaseImport for '{0}' (BookId={1}): import was not from a tracked download", message.Book?.Title ?? "", @@ -227,6 +230,11 @@ public void Handle(BookImportedEvent message) { try { + if (isLibraryImport && !notification.NotifyOnLibraryImports) + { + continue; + } + if (ShouldHandleAuthor(notification.Definition, author)) { if (downloadMessage.OldFiles.Empty() || ((NotificationDefinition)notification.Definition).OnUpgrade) @@ -359,6 +367,49 @@ public void Handle(BookDeletedEvent message) } } + public void Handle(BookFileAddedEvent message) + { + var bookFile = message.BookFile; + + if (bookFile?.Path == null || bookFile.EditionId <= 0) + { + return; + } + + var book = bookFile.Edition?.Book ?? _editionService.GetEdition(bookFile.EditionId)?.Book; + + if (book == null) + { + return; + } + + var author = book.Author ?? bookFile.Author; + + foreach (var notification in _notificationFactory.OnReleaseImportEnabled()) + { + if (!notification.NotifyOnLibraryImports) + { + continue; + } + + try + { + if (author != null && !ShouldHandleAuthor(notification.Definition, author)) + { + continue; + } + + notification.OnLibraryFileAdded(bookFile, book); + _notificationStatusService.RecordSuccess(notification.Definition.Id); + } + catch (Exception ex) + { + _notificationStatusService.RecordFailure(notification.Definition.Id); + _logger.Warn(ex, "Unable to send library-file notification to: " + notification.Definition.Name); + } + } + } + public void Handle(BookFileDeletedEvent message) { var deleteMessage = new BookFileDeleteMessage();