diff --git a/Jellyfin/backend/Api/AnimeMarkersController.cs b/Jellyfin/backend/Api/AnimeMarkersController.cs index 9f721e9..8f21f4a 100644 --- a/Jellyfin/backend/Api/AnimeMarkersController.cs +++ b/Jellyfin/backend/Api/AnimeMarkersController.cs @@ -421,18 +421,27 @@ public async Task> GetPreview( _diagnostics.Write($"preview series=\"{series.Name}\" (from the admin page)"); - var show = _resolver.MatchSeries(series); + // Sync the id to a englisch/romanji title. + await _resolver.RefreshAliasesAsync(new[] { series }, cancellationToken).ConfigureAwait(false); + + var match = _resolver.MatchSeriesDetailed(series); + var show = match.Show; if (show == null) { - _diagnostics.Write(" -> matched nothing on AnimeFillerList"); + _diagnostics.Write(" -> matched nothing on AnimeFillerList, by title or by provider id"); return Ok(new { series = series.Name, matched = false, - hint = "No AnimeFillerList show has this title. Check the site's spelling for it." + hint = "No AnimeFillerList show has this title, and none of its provider ids lead to one. Check its AniList, AniDB or MyAnimeList id." }); } + if (match.AliasKey != null) + { + _diagnostics.Write($" -> matched {show.Slug} through {match.AliasKey}"); + } + var entry = _resolver.GetCachedEntry(series, CacheMaxAge); if (entry == null) { @@ -515,6 +524,7 @@ public async Task> RefreshSeries( } await _client.EnsureCatalogAsync(cancellationToken).ConfigureAwait(false); + await _resolver.RefreshAliasesAsync(new[] { series }, cancellationToken).ConfigureAwait(false); var show = _resolver.MatchSeries(series); if (show == null) diff --git a/Jellyfin/backend/Pages/configPage.html b/Jellyfin/backend/Pages/configPage.html index 8d6f957..f9161eb 100644 --- a/Jellyfin/backend/Pages/configPage.html +++ b/Jellyfin/backend/Pages/configPage.html @@ -714,7 +714,8 @@

Anime Filler & Recap Markers

Nightly task that matches your anime against AnimeFillerList so Moonfin clients can badge filler episodes. No account or API key is needed. Series are matched by - title, and only shows the site actually lists are fetched. + title, falling back to the English and romaji titles behind their ids for libraries in other languages, + and only shows the site actually lists are fetched.
diff --git a/Jellyfin/backend/PluginServiceRegistrator.cs b/Jellyfin/backend/PluginServiceRegistrator.cs index 5c65810..5bb261d 100644 --- a/Jellyfin/backend/PluginServiceRegistrator.cs +++ b/Jellyfin/backend/PluginServiceRegistrator.cs @@ -32,6 +32,7 @@ public void RegisterServices(IServiceCollection serviceCollection, IServerApplic serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); diff --git a/Jellyfin/backend/Services/AnimeIdMappingService.cs b/Jellyfin/backend/Services/AnimeIdMappingService.cs index 14995b4..dd360b5 100644 --- a/Jellyfin/backend/Services/AnimeIdMappingService.cs +++ b/Jellyfin/backend/Services/AnimeIdMappingService.cs @@ -136,6 +136,11 @@ private async Task DownloadAsync(CancellationToken cancellationToken) } } + /// + /// Loads a table from a given file, so tests can check the parsing without a download. + /// + internal void LoadFrom(string path) => _index = Parse(path); + private static MappingIndex Parse(string path) { var index = new MappingIndex(); @@ -157,7 +162,11 @@ private static MappingIndex Parse(string path) Add(index.ByAniDb, ReadInt(entry, "anidb_id"), malId.Value); Add(index.ByAniSearch, ReadInt(entry, "anisearch_id"), malId.Value); Add(index.ByKitsu, ReadInt(entry, "kitsu_id"), malId.Value); - Add(index.ByImdb, ReadString(entry, "imdb_id"), malId.Value); + // The table lists IMDb ids as an array, since one entry can have several titles. + foreach (var imdbId in ReadStrings(entry, "imdb_id")) + { + Add(index.ByImdb, imdbId, malId.Value); + } // One TVDB or TMDB id covers a whole show, while MyAnimeList splits it per // season, so the first entry wins. That is the earliest season, and the recap @@ -173,6 +182,48 @@ private static MappingIndex Parse(string path) return index; } + /// + /// Returns true when the mapping table has been loaded, either from disk or from a download. + /// + public bool IsLoaded => _index != null; + + /// + /// Resolves one provider id, given by the canonical names + /// uses ("anidb", "tvdb", ...), to a MyAnimeList id. Returns null when it isnt in the table. + /// + public int? ResolveProvider(string provider, string value) + { + var index = _index; + if (index == null || string.IsNullOrWhiteSpace(value)) + { + return null; + } + + var trimmed = value.Trim(); + if (provider == "imdb") + { + return index.ByImdb.TryGetValue(trimmed, out var fromImdb) ? fromImdb : null; + } + + if (!int.TryParse(trimmed, out var id)) + { + return null; + } + + var map = provider switch + { + "anilist" => index.ByAniList, + "anidb" => index.ByAniDb, + "anisearch" => index.ByAniSearch, + "kitsu" => index.ByKitsu, + "tvdb" => index.ByTvdb, + "tmdb" => index.ByTmdb, + _ => null + }; + + return map != null && map.TryGetValue(id, out var malId) ? malId : null; + } + /// /// Resolves a MyAnimeList id for a series, using the provider ids the library already has. Returns null if no mapping is found. /// @@ -256,10 +307,31 @@ private static void Add(Dictionary map, string? key, int malId) ? parsed : null; - private static string? ReadString(JsonElement element, string property) => - element.TryGetProperty(property, out var value) && value.ValueKind == JsonValueKind.String - ? value.GetString() - : null; + /// + /// Reads a property that is either one string or an array of them. + /// + private static IEnumerable ReadStrings(JsonElement element, string property) + { + if (!element.TryGetProperty(property, out var value)) + { + yield break; + } + + if (value.ValueKind == JsonValueKind.String) + { + yield return value.GetString(); + } + else if (value.ValueKind == JsonValueKind.Array) + { + foreach (var item in value.EnumerateArray()) + { + if (item.ValueKind == JsonValueKind.String) + { + yield return item.GetString(); + } + } + } + } private static bool TryGetInt(BaseItem series, out int value, params string[] keys) { diff --git a/Jellyfin/backend/Services/AnimeMarkerModels.cs b/Jellyfin/backend/Services/AnimeMarkerModels.cs index bcf7c8c..5f8d4f4 100644 --- a/Jellyfin/backend/Services/AnimeMarkerModels.cs +++ b/Jellyfin/backend/Services/AnimeMarkerModels.cs @@ -83,3 +83,24 @@ public class AnimeMarkerCacheEntry [JsonPropertyName("cachedAt")] public DateTimeOffset CachedAt { get; set; } } + +/// +/// The titles AniList knows one provider id by. Lets a series whose library name is +/// localized still be found under the English or romaji title AnimeFillerList uses. +/// +public class AnimeTitleAliasEntry +{ + /// + /// The MyAnimeList id behind the titles, when AniList has one. + /// + [JsonPropertyName("malId")] + public int? MalId { get; set; } + + /// + /// English title, then romaji, then synonyms. Empty when AniList had no entry. + [JsonPropertyName("titles")] + public List Titles { get; set; } = new(); + + [JsonPropertyName("fetchedAt")] + public DateTimeOffset FetchedAt { get; set; } +} diff --git a/Jellyfin/backend/Services/AnimeMarkerResolver.cs b/Jellyfin/backend/Services/AnimeMarkerResolver.cs index b79afe7..dafad6c 100644 --- a/Jellyfin/backend/Services/AnimeMarkerResolver.cs +++ b/Jellyfin/backend/Services/AnimeMarkerResolver.cs @@ -15,6 +15,7 @@ public class AnimeMarkerResolver private readonly IMediaSourceManager? _mediaSourceManager; private readonly AnimeFillerListClient _client; private readonly AnimeMarkerCacheService _cache; + private readonly AnimeTitleAliasService _aliases; private readonly ILogger _logger; public AnimeMarkerResolver( @@ -22,12 +23,14 @@ public AnimeMarkerResolver( IMediaSourceManager mediaSourceManager, AnimeFillerListClient client, AnimeMarkerCacheService cache, + AnimeTitleAliasService aliases, ILogger logger) { _libraryManager = libraryManager; _mediaSourceManager = mediaSourceManager; _client = client; _cache = cache; + _aliases = aliases; _logger = logger; } @@ -136,13 +139,50 @@ private IEnumerable QuerySeries(Guid? parentId) } /// - /// Finds the AnimeFillerList show for a series, by its library name and its original - /// title. Returns null when neither matches, which is the right answer for the large - /// majority of a general library. + /// Finds the AnimeFillerList show for a series. Returns null when nothing matches, which + /// is the right answer for the large majority of a general library. /// - public AnimeFillerShow? MatchSeries(Series series) => + public AnimeFillerShow? MatchSeries(Series series) => MatchSeriesDetailed(series).Show; + + /// + /// The same, along with how it matched. The library name and original title are tried + /// first, then the English and romaji titles behind the series' provider ids, which is + /// what finds a show whose library name is localized. Those titles only exist once + /// has fetched them. + /// + public SeriesMatchResult MatchSeriesDetailed(Series series) + { + var byTitle = MatchByLibraryTitle(series); + if (byTitle != null) + { + return new SeriesMatchResult(byTitle, null, null); + } + + foreach (var (key, alias) in _aliases.GetAliases(series)) + { + var show = AnimeTitleMatcher.MatchPrimary(_client.Index, alias.Titles); + if (show != null) + { + return new SeriesMatchResult(show, key, alias.MalId); + } + } + + return new SeriesMatchResult(null, null, null); + } + + private AnimeFillerShow? MatchByLibraryTitle(Series series) => AnimeTitleMatcher.Match(_client.Index, WithProductionYear(series), series.Name, series.OriginalTitle); + /// + /// Fetches the provider-id titles for every series whose library title matches nothing. + /// Series that already match by title are left alone, so a library that works today + /// will not change. + /// + public Task RefreshAliasesAsync(IEnumerable series, CancellationToken cancellationToken) => + _aliases.RefreshAsync( + series.Where(item => MatchByLibraryTitle(item) == null).ToList(), + cancellationToken); + /// /// The series' name with its production year appended, when it has one. /// @@ -155,16 +195,19 @@ private IEnumerable QuerySeries(Guid? parentId) /// Every series that matches a show on the site, paired with the show. Used both by the /// sync task to decide what to fetch and by the diagnostics endpoint. /// - public List BuildMatches() + public List BuildMatches() => BuildMatches(GetCandidateSeries()); + + /// The same, over series the caller has already gathered. + public List BuildMatches(IEnumerable candidates) { var matches = new List(); - foreach (var series in GetCandidateSeries()) + foreach (var series in candidates) { - var show = MatchSeries(series); - if (show != null) + var match = MatchSeriesDetailed(series); + if (match.Show != null) { - matches.Add(new SeriesMatch(series, show)); + matches.Add(new SeriesMatch(series, match.Show, match.MalId)); } } @@ -572,8 +615,18 @@ public class AudioMarkerResult public Dictionary Seasons { get; } = new(); } -/// One library series and the AnimeFillerList show it matched. -public record SeriesMatch(Series Series, AnimeFillerShow Show); +/// +/// One library series and the AnimeFillerList show it matched. The MyAnimeList id is set +/// when the match came from a provider id's titles, and is the entry the recap pass should +/// use, since it is the one that actually named the show. +/// +public record SeriesMatch(Series Series, AnimeFillerShow Show, int? MalId = null); + +/// +/// The show a series matched, if any, and the provider id key ("anidb:266") whose titles +/// matched it. The key is null for a match on the library title. +/// +public record SeriesMatchResult(AnimeFillerShow? Show, string? AliasKey, int? MalId); /// One library episode and the absolute number the site would use for it. public record NumberedEpisode(Episode Episode, int AbsoluteNumber); diff --git a/Jellyfin/backend/Services/AnimeMarkerSyncTask.cs b/Jellyfin/backend/Services/AnimeMarkerSyncTask.cs index cda2982..bdbb4ad 100644 --- a/Jellyfin/backend/Services/AnimeMarkerSyncTask.cs +++ b/Jellyfin/backend/Services/AnimeMarkerSyncTask.cs @@ -75,7 +75,19 @@ public async Task ExecuteAsync(IProgress progress, CancellationToken can progress.Report(5); - var matches = _resolver.BuildMatches(); + // A localized library name never matches the site's English titles, so the titles + // behind those series' provider ids are fetched before matching. + var candidates = _resolver.GetCandidateSeries(); + try + { + await _resolver.RefreshAliasesAsync(candidates, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning(ex, "Anime markers sync: the provider id title lookup failed, matching on library titles only"); + } + + var matches = _resolver.BuildMatches(candidates); var showsBySlug = matches .GroupBy(match => match.Show.Slug, StringComparer.OrdinalIgnoreCase) .ToDictionary(group => group.Key, group => group.First().Show, StringComparer.OrdinalIgnoreCase); @@ -247,7 +259,11 @@ private async Task FetchRecapsAsync(List matches, Stopwatch clock, continue; } - var malId = await _recapService.TryResolveMalIdAsync(match.Series, cancellationToken).ConfigureAwait(false); + // A match made through a provider id's titles already knows which entry named + // the show. Resolving again could land on another one, such as a movie sharing + // the series' ids. + var malId = match.MalId + ?? await _recapService.TryResolveMalIdAsync(match.Series, cancellationToken).ConfigureAwait(false); if (malId == null) { _diagnostics.Write( diff --git a/Jellyfin/backend/Services/AnimeTitleAliasService.cs b/Jellyfin/backend/Services/AnimeTitleAliasService.cs new file mode 100644 index 0000000..add89f2 --- /dev/null +++ b/Jellyfin/backend/Services/AnimeTitleAliasService.cs @@ -0,0 +1,382 @@ +using System.Net.Http; +using System.Text; +using System.Text.Json; +using MediaBrowser.Controller.Entities; +using Microsoft.Extensions.Logging; + +namespace Moonfin.Server.Services; + +/// +/// Looks up the English and romaji titles behind a series' anime provider ids, so a series +/// whose library name is localized ("El detectiu Conan") can still be matched to the English +/// title AnimeFillerList lists it under. The titles come from AniList and are cached per +/// provider id, keyed like "anidb:266", so the request path never goes to the network. +/// +public class AnimeTitleAliasService : FileBackedCacheService +{ + private const string AniListEndpoint = "https://graphql.anilist.co"; + + private const string MediaQuery = + "query($ids:[Int]){Page(perPage:50){media({0}:$ids,type:ANIME){id idMal title{romaji english} synonyms}}}"; + + private const int BatchSize = 50; + + private static readonly TimeSpan MaxAge = TimeSpan.FromDays(30); + + /// + /// AniList allows 30 requests a minute while it is degraded, which is most of the time. + /// + private static readonly TimeSpan MinRequestSpacing = TimeSpan.FromSeconds(2.5); + + /// + /// The providers whose ids can lead to a show, in the order their titles are tried. + /// The anime databases come first because they name one entry, where a TVDB or TMDB + /// id covers a whole show that MyAnimeList splits into several. + /// + private static readonly (string Name, string[] Keys)[] Providers = + { + ("mal", new[] { "MyAnimeList", "Mal" }), + ("anilist", new[] { "AniList" }), + ("anidb", new[] { "AniDB" }), + ("kitsu", new[] { "Kitsu", "KitsuIo" }), + ("anisearch", new[] { "AniSearch" }), + ("tvdb", new[] { "Tvdb", "TheTVDB" }), + ("tmdb", new[] { "Tmdb", "TheMovieDb" }), + ("imdb", new[] { "Imdb" }) + }; + + private readonly IHttpClientFactory _httpClientFactory; + private readonly AnimeIdMappingService _mapping; + private readonly AnimeMarkerDiagnosticLog _diagnostics; + private readonly ILogger _logger; + private readonly SemaphoreSlim _refreshLock = new(1, 1); + + private DateTimeOffset _lastRequestAt = DateTimeOffset.MinValue; + + public AnimeTitleAliasService( + IHttpClientFactory httpClientFactory, + AnimeIdMappingService mapping, + AnimeMarkerDiagnosticLog diagnostics, + ILogger logger) + : base(logger, "anime_title_aliases.json", "Anime title aliases") + { + _httpClientFactory = httpClientFactory; + _mapping = mapping; + _diagnostics = diagnostics; + _logger = logger; + } + + /// + /// The series' provider ids as cache keys. + /// + public static IEnumerable<(string Key, string Provider, string Value)> ProviderKeys(BaseItem series) + { + var seen = new HashSet(StringComparer.Ordinal); + + foreach (var (name, keys) in Providers) + { + foreach (var (providerKey, providerValue) in series.ProviderIds) + { + if (string.IsNullOrWhiteSpace(providerValue) || + !keys.Any(key => string.Equals(key, providerKey, StringComparison.OrdinalIgnoreCase))) + { + continue; + } + + var value = providerValue.Trim(); + var key = name + ":" + value; + if (seen.Add(key)) + { + yield return (key, name, value); + } + } + } + } + + /// + /// The cached titles for each of the series' provider ids that has any, best first. + /// Never goes to the network, so a series nobody has refreshed yet simply has none. + /// + public IEnumerable<(string Key, AnimeTitleAliasEntry Entry)> GetAliases(BaseItem series) + { + var cache = EnsureLoaded(); + + foreach (var (key, _, _) in ProviderKeys(series)) + { + if (cache.TryGetValue(key, out var entry) && entry.Titles.Count > 0) + { + yield return (key, entry); + } + } + } + + /// + /// Fetches titles for every provider id of these series that isnt cached yet, or whose + /// cached titles are older than a month. Ids that cant be tied to an anime entry offline + /// cost nothing, so a whole library can be handed over. Returns how many ids were stored. + /// + public async Task RefreshAsync(IEnumerable seriesList, CancellationToken cancellationToken) + { + await _refreshLock.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + await _mapping.EnsureLoadedAsync(cancellationToken).ConfigureAwait(false); + + var cache = EnsureLoaded(); + var now = DateTimeOffset.UtcNow; + + // Keys wanting titles, grouped by which AniList id field can answer them. An + // AniList id is asked about directly. Anything else needs a MyAnimeList id, + // either stated outright or found through the offline mapping table. + var byAniListId = new Dictionary>(); + var byMalId = new Dictionary>(); + + foreach (var series in seriesList) + { + foreach (var (key, provider, value) in ProviderKeys(series)) + { + if (cache.TryGetValue(key, out var cached) && now - cached.FetchedAt < MaxAge) + { + continue; + } + + if (provider == "anilist" && int.TryParse(value, out var aniListId)) + { + AddKey(byAniListId, aniListId, key); + continue; + } + + int? malId = provider == "mal" + ? int.TryParse(value, out var stated) ? stated : null + : _mapping.ResolveProvider(provider, value); + + if (malId != null) + { + AddKey(byMalId, malId.Value, key); + } + } + } + + if (byAniListId.Count == 0 && byMalId.Count == 0) + { + return 0; + } + + var stored = 0; + stored += await FetchAndStoreAsync("id_in", byAniListId, media => media.Id, cancellationToken).ConfigureAwait(false); + stored += await FetchAndStoreAsync("idMal_in", byMalId, media => media.MalId, cancellationToken).ConfigureAwait(false); + + if (stored > 0) + { + await FlushAsync().ConfigureAwait(false); + _logger.LogInformation("Anime markers: titles looked up for {Count} provider ids", stored); + } + + return stored; + } + finally + { + _refreshLock.Release(); + } + } + + private async Task FetchAndStoreAsync( + string field, + Dictionary> keysById, + Func idOf, + CancellationToken cancellationToken) + { + var stored = 0; + var cache = EnsureLoaded(); + + foreach (var batch in keysById.Keys.Chunk(BatchSize)) + { + cancellationToken.ThrowIfCancellationRequested(); + + var media = await QueryAsync(field, batch, cancellationToken).ConfigureAwait(false); + if (media == null) + { + // A failed request stores nothing, so the next run asks again. + continue; + } + + var found = new Dictionary(); + foreach (var item in media) + { + if (idOf(item) is { } id) + { + found.TryAdd(id, item); + } + } + + foreach (var id in batch) + { + // AniList answered and had no entry: stored empty, so it isnt asked again + // until the entry goes stale. + found.TryGetValue(id, out var item); + var entry = new AnimeTitleAliasEntry + { + MalId = item?.MalId, + Titles = item?.Titles ?? new List(), + FetchedAt = DateTimeOffset.UtcNow + }; + + foreach (var key in keysById[id]) + { + cache[key] = entry; + stored++; + _diagnostics.Write(entry.Titles.Count == 0 + ? $"alias {key}: AniList has no entry for it" + : $"alias {key}: MAL {entry.MalId?.ToString(System.Globalization.CultureInfo.InvariantCulture) ?? "?"} " + + $"known as {string.Join(" | ", entry.Titles.Take(5))}"); + } + } + } + + return stored; + } + + private async Task?> QueryAsync(string field, int[] ids, CancellationToken cancellationToken) + { + var sinceLast = DateTimeOffset.UtcNow - _lastRequestAt; + if (sinceLast < MinRequestSpacing) + { + await Task.Delay(MinRequestSpacing - sinceLast, cancellationToken).ConfigureAwait(false); + } + + _lastRequestAt = DateTimeOffset.UtcNow; + + try + { + var client = _httpClientFactory.CreateClient(); + client.Timeout = TimeSpan.FromSeconds(20); + client.DefaultRequestHeaders.UserAgent.ParseAdd("Moonfin/1.0"); + + var body = JsonSerializer.Serialize(new + { + query = MediaQuery.Replace("{0}", field, StringComparison.Ordinal), + variables = new { ids } + }); + + using var content = new StringContent(body, Encoding.UTF8, "application/json"); + using var response = await client.PostAsync(AniListEndpoint, content, cancellationToken).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + { + _diagnostics.Write($"alias AniList {field} lookup of {ids.Length} ids -> HTTP {(int)response.StatusCode}"); + return null; + } + + var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + return ParseMedia(json); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogDebug(ex, "AniList title lookup failed"); + _diagnostics.Write($"alias AniList {field} lookup of {ids.Length} ids -> {ex.GetType().Name}: {ex.Message}"); + return null; + } + } + + /// + /// Reads a Page.media response. Returns null when the response isnt one, so a malformed + /// answer isnt mistaken for AniList having no entries. + /// + public static List? ParseMedia(string json) + { + using var document = JsonDocument.Parse(json); + + if (!document.RootElement.TryGetProperty("data", out var data) || + data.ValueKind != JsonValueKind.Object || + !data.TryGetProperty("Page", out var page) || + page.ValueKind != JsonValueKind.Object || + !page.TryGetProperty("media", out var mediaList) || + mediaList.ValueKind != JsonValueKind.Array) + { + return null; + } + + var result = new List(); + + foreach (var media in mediaList.EnumerateArray()) + { + if (media.ValueKind != JsonValueKind.Object) + { + continue; + } + + var titles = new List(); + if (media.TryGetProperty("title", out var title) && title.ValueKind == JsonValueKind.Object) + { + AddTitle(titles, title, "english"); + AddTitle(titles, title, "romaji"); + } + + if (media.TryGetProperty("synonyms", out var synonyms) && synonyms.ValueKind == JsonValueKind.Array) + { + foreach (var synonym in synonyms.EnumerateArray()) + { + if (synonym.ValueKind == JsonValueKind.String) + { + AddTitle(titles, synonym.GetString()); + } + } + } + + result.Add(new AniListMedia(ReadInt(media, "id"), ReadInt(media, "idMal"), titles)); + } + + return result; + } + + private static void AddTitle(List titles, JsonElement parent, string property) + { + if (parent.TryGetProperty(property, out var value) && value.ValueKind == JsonValueKind.String) + { + AddTitle(titles, value.GetString()); + } + } + + /// + /// Keeps a title only when it folds to a usable key. Titles in a non-Latin script fold + /// to nothing and could never match, so they arent worth keeping on disk. + /// + private static void AddTitle(List titles, string? title) + { + if (string.IsNullOrWhiteSpace(title) || AnimeTitleMatcher.Normalize(title).Length == 0) + { + return; + } + + var trimmed = title.Trim(); + if (!titles.Contains(trimmed, StringComparer.OrdinalIgnoreCase)) + { + titles.Add(trimmed); + } + } + + private static int? ReadInt(JsonElement element, string property) => + element.TryGetProperty(property, out var value) && + value.ValueKind == JsonValueKind.Number && + value.TryGetInt32(out var parsed) + ? parsed + : null; + + private static void AddKey(Dictionary> map, int id, string key) + { + if (!map.TryGetValue(id, out var keys)) + { + keys = new List(); + map[id] = keys; + } + + if (!keys.Contains(key)) + { + keys.Add(key); + } + } + + /// + /// One AniList entry: its ids and its usable titles, English first. + /// + public record AniListMedia(int? Id, int? MalId, List Titles); +} diff --git a/Jellyfin/backend/Services/AnimeTitleMatcher.cs b/Jellyfin/backend/Services/AnimeTitleMatcher.cs index 18ba35a..28af195 100644 --- a/Jellyfin/backend/Services/AnimeTitleMatcher.cs +++ b/Jellyfin/backend/Services/AnimeTitleMatcher.cs @@ -182,6 +182,29 @@ public static Dictionary BuildIndex(IEnumerable + /// The same, but trying only the titles themselves and not the names inside their + /// parentheses. Used for AniList's titles, where "Konpeki no Hitsugi (Jolly Roger)" + /// is a subtitle rather than another name for the show. + /// + public static AnimeFillerShow? MatchPrimary( + IReadOnlyDictionary index, + IEnumerable candidateTitles) + { + foreach (var title in candidateTitles) + { + foreach (var key in PrimaryVariants(title)) + { + if (index.TryGetValue(key, out var show)) + { + return show; + } + } + } + + return null; + } + private static string FoldDiacritics(string value) { var decomposed = value.Normalize(NormalizationForm.FormD); diff --git a/Jellyfin/tests/Moonfin.Server.Tests/AnimeTitleAliasTests.cs b/Jellyfin/tests/Moonfin.Server.Tests/AnimeTitleAliasTests.cs new file mode 100644 index 0000000..0178c97 --- /dev/null +++ b/Jellyfin/tests/Moonfin.Server.Tests/AnimeTitleAliasTests.cs @@ -0,0 +1,126 @@ +using MediaBrowser.Controller.Entities.TV; +using Moonfin.Server.Services; +using Xunit; + +namespace Moonfin.Server.Tests; + +/// +/// Tests the fallback that matches a series with a localized library name through the +/// English and romaji titles AniList holds for its provider ids (issue #310). +/// +public class AnimeTitleAliasTests +{ + private const string AniListResponse = """ + {"data":{"Page":{"media":[ + {"id":223,"idMal":223,"title":{"romaji":"Dragon Ball","english":"Dragon Ball"}, + "synonyms":["Dragonball","Bola de Drac","דרגון בול ","DB"]}, + {"id":235,"idMal":235,"title":{"romaji":"Meitantei Conan","english":"Detective Conan"}, + "synonyms":["Case Closed","Detectiu Conan","名侦探柯南","Détective Conan"]}, + {"id":2171,"idMal":2171, + "title":{"romaji":"Meitantei Conan: Konpeki no Hitsugi (Jolly Roger)","english":"Detective Conan: Jolly Roger in the Deep Azure"}, + "synonyms":["Detective Conan 2007","Detective Conan Movie 11"]} + ]}}} + """; + + private static readonly AnimeFillerShow DetectiveConan = new() { Slug = "detective-conan", Title = "Detective Conan" }; + private static readonly AnimeFillerShow DragonBall = new() { Slug = "dragon-ball", Title = "Dragon Ball" }; + private static readonly AnimeFillerShow JollyRoger = new() { Slug = "jolly-roger", Title = "Jolly Roger" }; + + private static Dictionary Index() => + AnimeTitleMatcher.BuildIndex(new[] { DetectiveConan, DragonBall, JollyRoger }); + + private static Dictionary Parsed() => + AnimeTitleAliasService.ParseMedia(AniListResponse)!.ToDictionary(media => media.Id!.Value); + + [Fact] + public void ParseMedia_ListsEnglishThenRomajiThenSynonyms_AndDropsNonLatinTitles() + { + var conan = Parsed()[235]; + + Assert.Equal(235, conan.MalId); + Assert.Equal( + new[] { "Detective Conan", "Meitantei Conan", "Case Closed", "Detectiu Conan", "Détective Conan" }, + conan.Titles); + } + + [Fact] + public void ParseMedia_CollapsesAnEnglishTitleThatRepeatsTheRomaji() + { + Assert.Equal(new[] { "Dragon Ball", "Dragonball", "Bola de Drac", "DB" }, Parsed()[223].Titles); + } + + [Theory] + [InlineData("{\"data\":null}")] + [InlineData("{\"errors\":[{\"message\":\"Too Many Requests.\"}],\"data\":null}")] + public void ParseMedia_ReturnsNullForAnAnswerThatIsntAPage(string json) + { + Assert.Null(AnimeTitleAliasService.ParseMedia(json)); + } + + [Theory] + [InlineData(235, "detective-conan")] + [InlineData(223, "dragon-ball")] + public void TheTitlesOfTheRightEntryFindTheShow(int aniListId, string slug) + { + Assert.Equal(slug, AnimeTitleMatcher.MatchPrimary(Index(), Parsed()[aniListId].Titles)?.Slug); + } + + [Fact] + public void AMovieEntrysSubtitleInParenthesesDoesntPassForAShow() + { + // The plain matcher would read "(Jolly Roger)" as another name for the show. + Assert.Null(AnimeTitleMatcher.MatchPrimary(Index(), Parsed()[2171].Titles)); + } + + [Theory] + [InlineData("imdb", "tt0131179")] + [InlineData("tmdb", "30983")] + [InlineData("tvdb", "72454")] + [InlineData("anidb", "266")] + public void TheMappingTableResolvesTheIdsAStockJellyfinLibraryHas(string provider, string value) + { + // The real table's entry for Detective Conan. IMDb ids come as an array there. + const string Table = """ + [{"type":"TV","anidb_id":266,"anilist_id":235,"imdb_id":["tt0131179"],"kitsu_id":210, + "mal_id":235,"themoviedb_id":{"tv":30983},"tvdb_id":72454}] + """; + + var path = Path.Combine(Path.GetTempPath(), "moonfin-mapping-" + Guid.NewGuid().ToString("N") + ".json"); + File.WriteAllText(path, Table); + try + { + var mapping = new AnimeIdMappingService( + new ThrowingHttpClientFactory(), new NoOpLogger()); + mapping.LoadFrom(path); + + Assert.Equal(235, mapping.ResolveProvider(provider, value)); + } + finally + { + File.Delete(path); + } + } + + [Fact] + public void ProviderKeys_PutsTheAnimeDatabasesBeforeTheGeneralOnes() + { + var series = new Series + { + Name = "El detectiu Conan", + ProviderIds = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["Imdb"] = "tt0131179", + ["Tvdb"] = "72454", + ["Tmdb"] = "30983", + ["AniList"] = "2171", + ["AniDB"] = "266", + ["Zap2It"] = "SH03296957", + ["TvdbSlug"] = "detective-conan" + } + }; + + Assert.Equal( + new[] { "anilist:2171", "anidb:266", "tvdb:72454", "tmdb:30983", "imdb:tt0131179" }, + AnimeTitleAliasService.ProviderKeys(series).Select(key => key.Key)); + } +}