From 3913ac1be55ae551da739e89d4568101f8bd52d1 Mon Sep 17 00:00:00 2001 From: OneHotTake Date: Wed, 30 Sep 2026 06:57:06 -0500 Subject: [PATCH 1/4] Preserve MDBList media types and release years --- Services/ListFetcher.cs | 35 +++++++++++++++++++++----- Tests/MdblistIdentityTests.cs | 47 +++++++++++++++++++++++++++++++++++ docs/EXTERNAL_LISTS.md | 15 +++++++++++ 3 files changed, 91 insertions(+), 6 deletions(-) create mode 100644 Tests/MdblistIdentityTests.cs diff --git a/Services/ListFetcher.cs b/Services/ListFetcher.cs index 7ec037f..7293916 100644 --- a/Services/ListFetcher.cs +++ b/Services/ListFetcher.cs @@ -149,16 +149,16 @@ private static async Task FetchMdblistAsync( // Legacy format: [ ... ] if (doc.RootElement.ValueKind == JsonValueKind.Object) { - var combined = new List(); + var combined = new List<(JsonElement Item, string MediaType)>(); if (doc.RootElement.TryGetProperty("movies", out var movies)) - foreach (var item in movies.EnumerateArray()) combined.Add(item); + foreach (var item in movies.EnumerateArray()) combined.Add((item, "movie")); if (doc.RootElement.TryGetProperty("shows", out var shows)) - foreach (var item in shows.EnumerateArray()) combined.Add(item); + foreach (var item in shows.EnumerateArray()) combined.Add((item, "series")); if (combined.Count == 0) break; foreach (var item in combined) { - var li = ParseMdblistItem(item); + var li = ParseMdblistItem(item.Item, item.MediaType); if (li != null) all.Add(li); } @@ -194,7 +194,7 @@ private static async Task FetchMdblistAsync( return new ListFetchResult { Ok = true, Items = all }; } - private static ListItem? ParseMdblistItem(JsonElement item) + internal static ListItem? ParseMdblistItem(JsonElement item, string? bucketType = null) { string? imdbId = null; if (item.TryGetProperty("imdb_id", out var imdbEl) && imdbEl.ValueKind == JsonValueKind.String) @@ -205,7 +205,30 @@ private static async Task FetchMdblistAsync( ? titleEl.GetString() ?? "Unknown" : "Unknown"; - return new ListItem(title, imdbId.ToLowerInvariant(), null, null); + // MDBList's public /json arrays use mediatype=show and release_year. + // Losing these fields silently routes shows through the movie resolver. + var rawType = item.TryGetProperty("mediatype", out var typeEl) + || item.TryGetProperty("media_type", out typeEl) + || item.TryGetProperty("type", out typeEl) + ? typeEl.ValueKind == JsonValueKind.String ? typeEl.GetString()?.ToLowerInvariant() : null + : null; + var explicitType = rawType switch + { + "movie" => "movie", + "show" or "series" or "tv" => "series", + _ => null + }; + if (bucketType != null && explicitType != null && bucketType != explicitType) return null; + var mediaType = explicitType ?? bucketType; + if (mediaType is not ("movie" or "series")) return null; + + int? year = null; + if (item.TryGetProperty("release_year", out var yearEl) || item.TryGetProperty("year", out yearEl)) + { + if (yearEl.ValueKind == JsonValueKind.Number && yearEl.TryGetInt32(out var numericYear)) year = numericYear; + else if (yearEl.ValueKind == JsonValueKind.String && int.TryParse(yearEl.GetString(), out numericYear)) year = numericYear; + } + return new ListItem(title, imdbId.ToLowerInvariant(), year, mediaType); } // ── Trakt ──────────────────────────────────────────────────────────────── diff --git a/Tests/MdblistIdentityTests.cs b/Tests/MdblistIdentityTests.cs new file mode 100644 index 0000000..f3be5a4 --- /dev/null +++ b/Tests/MdblistIdentityTests.cs @@ -0,0 +1,47 @@ +using System.Text.Json; +using InfiniteDrive.Services; +using Xunit; + +namespace InfiniteDrive.Tests; + +public sealed class MdblistIdentityTests +{ + [Theory] + [InlineData("show", "series")] + [InlineData("series", "series")] + [InlineData("movie", "movie")] + public void PublicArrayPreservesMediaTypeAndReleaseYear(string sourceType, string expected) + { + using var doc = JsonDocument.Parse($$"""{"title":"Sugar","imdb_id":"tt16418808","mediatype":"{{sourceType}}","release_year":2024}"""); + var item = ListFetcher.ParseMdblistItem(doc.RootElement); + Assert.NotNull(item); + Assert.Equal(expected, item.MediaType); + Assert.Equal(2024, item.Year); + } + + [Theory] + [InlineData("movie")] + [InlineData("series")] + public void GroupedResponseSuppliesTypeWhenIndividualItemOmitsIt(string bucket) + { + using var doc = JsonDocument.Parse("""{"title":"Example","imdb_id":"tt1234567","year":"2024"}"""); + var item = ListFetcher.ParseMdblistItem(doc.RootElement, bucket); + Assert.NotNull(item); + Assert.Equal(bucket, item.MediaType); + Assert.Equal(2024, item.Year); + } + + [Fact] + public void UntypedArrayEntryNeverDefaultsToMovie() + { + using var doc = JsonDocument.Parse("""{"title":"Loki","imdb_id":"tt9140554"}"""); + Assert.Null(ListFetcher.ParseMdblistItem(doc.RootElement)); + } + + [Fact] + public void ContradictoryBucketAndItemTypeAreRejected() + { + using var doc = JsonDocument.Parse("""{"title":"Loki","imdb_id":"tt9140554","mediatype":"show"}"""); + Assert.Null(ListFetcher.ParseMdblistItem(doc.RootElement, "movie")); + } +} diff --git a/docs/EXTERNAL_LISTS.md b/docs/EXTERNAL_LISTS.md index 3adbc6e..b8595d9 100644 --- a/docs/EXTERNAL_LISTS.md +++ b/docs/EXTERNAL_LISTS.md @@ -18,6 +18,21 @@ intent; user lists belong to their Emby user and may create native playlist state. Adding or refreshing a list fetches it through `ListFetcher`, normalizes provider IDs, and records its sync/error state. +## MDBList media identity + +MDBList entries retain their IMDb ID, media type and release year. Public JSON +arrays use `mediatype` (`show` becomes `series`) and `release_year`; grouped +responses use the `movies` and `shows` buckets when an item omits its type. +A contradictory type or an untyped array entry is rejected rather than routed +to the movie resolver. + +This parser change prevents misclassification of newly imported rows. Refreshing +a list does **not** automatically convert an existing movie row into a series. +Previously misclassified rows need a separate, backed-up repair of their media +type and managed destination. Existing movie STRMs must never be renamed into +fake episodes. Preserve list memberships, user blocks, retry history and owned +media; verify episode imports separately from playback. + ## Catalog-less behavior Catalog synchronization evaluates list state before deciding that no providers From d78139e8e54551fe764308ce77510610239613cf Mon Sep 17 00:00:00 2001 From: OneHotTake Date: Wed, 30 Sep 2026 07:27:18 -0500 Subject: [PATCH 2/4] Normalize media identity across external list providers --- Services/ListFetcher.cs | 140 ++++++++++++++++++----------- Tests/ExternalListIdentityTests.cs | 119 ++++++++++++++++++++++++ docs/EXTERNAL_LISTS.md | 68 +++++++++++--- 3 files changed, 259 insertions(+), 68 deletions(-) create mode 100644 Tests/ExternalListIdentityTests.cs diff --git a/Services/ListFetcher.cs b/Services/ListFetcher.cs index 7293916..bcea843 100644 --- a/Services/ListFetcher.cs +++ b/Services/ListFetcher.cs @@ -212,12 +212,7 @@ private static async Task FetchMdblistAsync( || item.TryGetProperty("type", out typeEl) ? typeEl.ValueKind == JsonValueKind.String ? typeEl.GetString()?.ToLowerInvariant() : null : null; - var explicitType = rawType switch - { - "movie" => "movie", - "show" or "series" or "tv" => "series", - _ => null - }; + var explicitType = NormalizeMediaType(rawType); if (bucketType != null && explicitType != null && bucketType != explicitType) return null; var mediaType = explicitType ?? bucketType; if (mediaType is not ("movie" or "series")) return null; @@ -287,7 +282,7 @@ private static async Task FetchTraktAsync( } } - private static ListFetchResult ParseTraktResponse(string json) + internal static ListFetchResult ParseTraktResponse(string json) { var items = new List(); try @@ -298,14 +293,24 @@ private static ListFetchResult ParseTraktResponse(string json) foreach (var entry in doc.RootElement.EnumerateArray()) { - // Wrapped format: { "movie": {...} } or { "show": {...} } + if (entry.ValueKind != JsonValueKind.Object) continue; + var hasType = entry.TryGetProperty("type", out var typeEl); + var declaredType = hasType && typeEl.ValueKind == JsonValueKind.String + ? NormalizeMediaType(typeEl.GetString()) : null; + if (hasType && declaredType == null) continue; + + // Wrapper identity is authoritative; do not promote episode + // history entries into their parent show or accept conflicts. JsonElement? mediaObj = null; - if (entry.TryGetProperty("movie", out var movieEl) && movieEl.ValueKind == JsonValueKind.Object) - mediaObj = movieEl; - else if (entry.TryGetProperty("show", out var showEl) && showEl.ValueKind == JsonValueKind.Object) - mediaObj = showEl; - else if (entry.TryGetProperty("ids", out _)) - mediaObj = entry; // Flat format + string? mediaType = null; + var hasMovie = entry.TryGetProperty("movie", out var movieEl) && movieEl.ValueKind == JsonValueKind.Object; + var hasShow = entry.TryGetProperty("show", out var showEl) && showEl.ValueKind == JsonValueKind.Object; + if (hasMovie && hasShow) continue; + if (hasMovie) { mediaObj = movieEl; mediaType = "movie"; } + else if (hasShow) { mediaObj = showEl; mediaType = "series"; } + else if (entry.TryGetProperty("ids", out _) && declaredType != null) + { mediaObj = entry; mediaType = declaredType; } + if (declaredType != null && mediaType != declaredType) continue; if (mediaObj == null) continue; @@ -326,7 +331,7 @@ private static ListFetchResult ParseTraktResponse(string json) if (m.TryGetProperty("year", out var yearEl) && yearEl.ValueKind == JsonValueKind.Number) year = yearEl.GetInt32(); - items.Add(new ListItem(title, imdbId.ToLowerInvariant(), year, null)); + items.Add(new ListItem(title, imdbId.ToLowerInvariant(), year, mediaType)); } } catch @@ -379,6 +384,19 @@ private static async Task FetchTmdbAsync( return Fail($"TMDB returned HTTP {(int)resp.StatusCode}."); var json = await resp.Content.ReadAsStringAsync(ct); + return ParseTmdbResponse(json); + } + catch (OperationCanceledException) { throw; } + catch (Exception ex) + { + return Fail($"Could not fetch TMDB list: {ex.Message}"); + } + } + + internal static ListFetchResult ParseTmdbResponse(string json) + { + try + { using var doc = JsonDocument.Parse(json); if (!doc.RootElement.TryGetProperty("items", out var itemsEl)) @@ -393,26 +411,23 @@ private static async Task FetchTmdbAsync( : null; if (string.IsNullOrWhiteSpace(title)) continue; - var mediaType = entry.TryGetProperty("media_type", out var mt) - ? mt.GetString() ?? "movie" : "movie"; - - if (!entry.TryGetProperty("id", out var idEl) || idEl.ValueKind != JsonValueKind.Number) - continue; - - var rawId = $"tmdb_{idEl.GetInt32()}"; - + var hasType = entry.TryGetProperty("media_type", out var mt); + var mediaType = hasType && mt.ValueKind == JsonValueKind.String + ? NormalizeMediaType(mt.GetString()) + : !hasType && entry.TryGetProperty("title", out var movieTitle) + && movieTitle.ValueKind == JsonValueKind.String + && !string.IsNullOrWhiteSpace(movieTitle.GetString()) ? "movie" : null; + // V3 movie lists omit media_type; a name-only entry is ambiguous. + if (mediaType == null) continue; + if (!entry.TryGetProperty("id", out var idEl) || idEl.ValueKind != JsonValueKind.Number + || !idEl.TryGetInt32(out var id) || id <= 0) continue; + var rawId = $"tmdb_{id}"; int? year = null; - if (entry.TryGetProperty("release_date", out var rd) && rd.ValueKind == JsonValueKind.String) + var dateField = mediaType == "series" ? "first_air_date" : "release_date"; + if (entry.TryGetProperty(dateField, out var dateEl) && dateEl.ValueKind == JsonValueKind.String) { - var dateStr = rd.GetString(); - if (dateStr?.Length >= 4 && int.TryParse(dateStr.Substring(0, 4), out var y)) - year = y; - } - else if (entry.TryGetProperty("first_air_date", out var fad) && fad.ValueKind == JsonValueKind.String) - { - var dateStr = fad.GetString(); - if (dateStr?.Length >= 4 && int.TryParse(dateStr.Substring(0, 4), out var y)) - year = y; + var date = dateEl.GetString(); + if (date?.Length >= 4 && int.TryParse(date.Substring(0, 4), out var y)) year = y; } items.Add(new ListItem(title, rawId, year, mediaType)); @@ -426,11 +441,8 @@ private static async Task FetchTmdbAsync( return new ListFetchResult { Ok = true, Items = items, DisplayName = listName }; } - catch (OperationCanceledException) { throw; } - catch (Exception ex) - { - return Fail($"Could not fetch TMDB list: {ex.Message}"); - } + catch (Exception ex) when (ex is JsonException or InvalidOperationException or KeyNotFoundException) + { return Fail("Could not parse TMDB response."); } } // ── AniList ────────────────────────────────────────────────────────────── @@ -466,6 +478,7 @@ private static async Task FetchAnilistAsync( title { romaji english native } startDate { year } type + format } } } @@ -493,6 +506,19 @@ private static async Task FetchAnilistAsync( } var json = await resp.Content.ReadAsStringAsync(ct); + return ParseAnilistResponse(json, username); + } + catch (OperationCanceledException) { throw; } + catch (Exception ex) + { + return Fail($"Could not fetch AniList: {ex.Message}"); + } + } + + internal static ListFetchResult ParseAnilistResponse(string json, string username) + { + try + { using var doc = JsonDocument.Parse(json); // Check for GraphQL errors @@ -509,7 +535,7 @@ private static async Task FetchAnilistAsync( .GetProperty("MediaListCollection") .GetProperty("lists"); - var items = new List<(string title, string? titleEnglish, string? titleRomaji, int? year, int anilistId)>(); + var items = new List(); foreach (var list in lists.EnumerateArray()) { @@ -517,6 +543,16 @@ private static async Task FetchAnilistAsync( foreach (var entry in entries.EnumerateArray()) { var media = entry.GetProperty("media"); + if (!media.TryGetProperty("type", out var typeEl) || typeEl.GetString() != "ANIME") continue; + var format = media.TryGetProperty("format", out var formatEl) && formatEl.ValueKind == JsonValueKind.String + ? formatEl.GetString() : null; + var mediaType = format switch + { + "MOVIE" => "movie", + "TV" or "TV_SHORT" or "OVA" or "ONA" or "SPECIAL" => "series", + _ => null + }; + if (mediaType == null) continue; var titleObj = media.GetProperty("title"); var titleEnglish = titleObj.TryGetProperty("english", out var te) ? te.GetString() : null; var titleRomaji = titleObj.TryGetProperty("romaji", out var tr) ? tr.GetString() : null; @@ -531,7 +567,7 @@ private static async Task FetchAnilistAsync( && idEl.ValueKind == JsonValueKind.Number ? idEl.GetInt32() : 0; - items.Add((title, titleEnglish, titleRomaji, year, anilistId)); + if (anilistId > 0) items.Add(new ListItem(title, $"anilist:{anilistId}", year, mediaType)); } } @@ -541,25 +577,21 @@ private static async Task FetchAnilistAsync( // Return items with native AniList IDs (anilist:XXX) — resolution // happens downstream via IdResolverService in UserCatalogSyncService. // The official Emby AniList plugin uses these numeric IDs. - var result = items - .Where(i => i.anilistId > 0) - .Select(i => new ListItem(i.title, $"anilist:{i.anilistId}", i.year, "tv")) - .ToList(); - - if (result.Count == 0) - return Fail($"AniList user '{username}' has no anime with valid IDs."); - - return new ListFetchResult { Ok = true, Items = result }; - } - catch (OperationCanceledException) { throw; } - catch (Exception ex) - { - return Fail($"Could not fetch AniList: {ex.Message}"); + return new ListFetchResult { Ok = true, Items = items }; } + catch (Exception ex) when (ex is JsonException or InvalidOperationException or KeyNotFoundException) + { return Fail("Could not parse AniList response."); } } // ── Helpers ────────────────────────────────────────────────────────────── + private static string? NormalizeMediaType(string? type) => type?.ToLowerInvariant() switch + { + "movie" => "movie", + "show" or "series" or "tv" => "series", + _ => null + }; + private static ListFetchResult Fail(string error) => new() { Ok = false, diff --git a/Tests/ExternalListIdentityTests.cs b/Tests/ExternalListIdentityTests.cs new file mode 100644 index 0000000..ecf596e --- /dev/null +++ b/Tests/ExternalListIdentityTests.cs @@ -0,0 +1,119 @@ +using System.Linq; +using System.Text.Json; +using InfiniteDrive.Models; +using InfiniteDrive.Services; +using Xunit; + +namespace InfiniteDrive.Tests; + +public sealed class ExternalListIdentityTests +{ + [Theory] + [InlineData("movie", "movie", false)] + [InlineData("show", "series", true)] + public void TraktWrappersKeepIdentityYearAndImportFamily(string wrapper, string type, bool episodic) + { + var result = ListFetcher.ParseTraktResponse("""[{"type":"__WRAPPER__","__WRAPPER__":{"title":"Example","year":2024,"ids":{"imdb":"tt1234567"}}}]""".Replace("__WRAPPER__", wrapper)); + var item = Assert.Single(result.Items); + Assert.Equal("tt1234567", item.AioId); + Assert.Equal(2024, item.Year); + Assert.Equal(type, item.MediaType); + Assert.Equal(episodic, ImportInventory.IsSeries(new CatalogItem { MediaType = item.MediaType! })); + } + + [Theory] + [InlineData("movie", "movie")] + [InlineData("show", "series")] + public void TraktFlatEntriesRequireAnExplicitType(string rawType, string expected) + { + var result = ListFetcher.ParseTraktResponse("""[{"title":"Example","year":2024,"type":"__RAWTYPE__","ids":{"imdb":"tt1234567"}}]""".Replace("__RAWTYPE__", rawType)); + Assert.Equal(expected, Assert.Single(result.Items).MediaType); + } + + [Theory] + [InlineData("""[{"title":"Unknown type","ids":{"imdb":"tt1234567"}}]""")] + [InlineData("""[{"type":"episode","show":{"title":"Parent","ids":{"imdb":"tt1234567"}},"episode":{"season":1,"number":2}}]""")] + [InlineData("""[{"type":"show","movie":{"title":"Contradiction","ids":{"imdb":"tt1234567"}}}]""")] + public void TraktDoesNotGuessOrPromoteUnsupportedItems(string json) + { + Assert.False(ListFetcher.ParseTraktResponse(json).Ok); + } + + [Fact] + public void TraktMixedListDoesNotRouteShowsThroughMovies() + { + var result = ListFetcher.ParseTraktResponse("""[{"movie":{"title":"Film","year":2023,"ids":{"imdb":"tt1234567"}}},{"show":{"title":"Show","year":2024,"ids":{"imdb":"tt1234568"}}}]"""); + Assert.Equal(new[] { "movie", "series" }, result.Items.Select(x => x.MediaType)); + } + + [Fact] + public void TmdbTvKeepsTvYearAndUsesEpisodeImporter() + { + var result = ListFetcher.ParseTmdbResponse("""{"name":"Mixed list","items":[{"id":203744,"name":"Sugar","media_type":"tv","release_date":"","first_air_date":"2024-04-05"}]}"""); + var item = Assert.Single(result.Items); + Assert.Equal("tmdb_203744", item.AioId); + Assert.Equal("series", item.MediaType); + Assert.Equal(2024, item.Year); + Assert.True(ImportInventory.IsSeries(new CatalogItem { MediaType = item.MediaType! })); + Assert.Equal("Mixed list", result.DisplayName); + } + + [Fact] + public void TmdbV3MovieRetainsUntypedMovieCompatibility() + { + var result = ListFetcher.ParseTmdbResponse("""{"items":[{"id":157336,"title":"Interstellar","release_date":"2014-11-05"}]}"""); + var item = Assert.Single(result.Items); + Assert.Equal("tmdb_157336", item.AioId); + Assert.Equal("movie", item.MediaType); + Assert.Equal(2014, item.Year); + } + + [Theory] + [InlineData("""{"items":[{"id":42,"name":"Ambiguous"}]}""")] + [InlineData("""{"items":[{"id":42,"name":"A person","media_type":"person"}]}""")] + public void TmdbRejectsAmbiguousOrUnsupportedTypes(string json) + { + Assert.False(ListFetcher.ParseTmdbResponse(json).Ok); + } + + [Theory] + [InlineData("MOVIE", "movie", false)] + [InlineData("TV", "series", true)] + [InlineData("TV_SHORT", "series", true)] + [InlineData("OVA", "series", true)] + [InlineData("ONA", "series", true)] + [InlineData("SPECIAL", "series", true)] + public void AnilistFormatDistinguishesFilmsFromEpisodes(string format, string type, bool episodic) + { + var result = ListFetcher.ParseAnilistResponse("""{"data":{"MediaListCollection":{"lists":[{"entries":[{"media":{"id":21519,"type":"ANIME","format":"__FORMAT__","title":{"english":"Example","romaji":"Rei"},"startDate":{"year":2016}}}]}]}}}""".Replace("__FORMAT__", format), "example-user"); + var item = Assert.Single(result.Items); + Assert.Equal("anilist:21519", item.AioId); + Assert.Equal(2016, item.Year); + Assert.Equal(type, item.MediaType); + Assert.Equal(episodic, ImportInventory.IsSeries(new CatalogItem { MediaType = item.MediaType! })); + } + + [Theory] + [InlineData("ANIME", "UNKNOWN")] + [InlineData("MANGA", "TV")] + [InlineData("ANIME", "MUSIC")] + public void AnilistRejectsUnsupportedFormatOrNonAnime(string type, string format) + { + var result = ListFetcher.ParseAnilistResponse("""{"data":{"MediaListCollection":{"lists":[{"entries":[{"media":{"id":1,"type":"__TYPE__","format":"__FORMAT__","title":{"english":"Unsupported"}}}]}]}}}""".Replace("__TYPE__", type).Replace("__FORMAT__", format), "example-user"); + Assert.False(result.Ok); + } + + [Theory] + [InlineData("{}")] + [InlineData("{\"data\":{\"MediaListCollection\":null}}")] + public void AnilistMalformedCollectionIsReportedAsAFailure(string json) + { + Assert.False(ListFetcher.ParseAnilistResponse(json, "example-user").Ok); + } + + [Fact] + public void AnilistGraphqlErrorDoesNotBecomeAnEmptySuccessfulList() + { + Assert.False(ListFetcher.ParseAnilistResponse("""{"errors":[{"message":"Not found"}]}""", "example-user").Ok); + } +} diff --git a/docs/EXTERNAL_LISTS.md b/docs/EXTERNAL_LISTS.md index b8595d9..f400397 100644 --- a/docs/EXTERNAL_LISTS.md +++ b/docs/EXTERNAL_LISTS.md @@ -18,20 +18,60 @@ intent; user lists belong to their Emby user and may create native playlist state. Adding or refreshing a list fetches it through `ListFetcher`, normalizes provider IDs, and records its sync/error state. -## MDBList media identity - -MDBList entries retain their IMDb ID, media type and release year. Public JSON -arrays use `mediatype` (`show` becomes `series`) and `release_year`; grouped -responses use the `movies` and `shows` buckets when an item omits its type. -A contradictory type or an untyped array entry is rejected rather than routed -to the movie resolver. - -This parser change prevents misclassification of newly imported rows. Refreshing -a list does **not** automatically convert an existing movie row into a series. -Previously misclassified rows need a separate, backed-up repair of their media -type and managed destination. Existing movie STRMs must never be renamed into -fake episodes. Preserve list memberships, user blocks, retry history and owned -media; verify episode imports separately from playback. +## Media identity + +Each supported parser returns `movie` or `series`, along with its provider ID +and available release year. These values select different metadata and import +paths: a show must reach the episode importer. Provider terms such as `show` and +`tv` cannot be passed through unchanged or replaced with a movie default. + +| Source | Identity handling | +|---|---| +| MDBList | Public JSON arrays use `mediatype` and `release_year`. `show` becomes `series`; grouped `movies`/`shows` responses supply the type when an item omits it. Contradictory buckets and untyped array entries are skipped. | +| Trakt | `movie` and `show` wrappers supply the type; an explicit `type` must agree. Flat entries require an explicit supported type. Episode/history and other unsupported entries are skipped rather than promoted to a whole series. | +| TMDB | Explicit `media_type=tv` becomes `series` and uses `first_air_date` for the year. Movies use `release_date`. V3 movie list entries without a type retain movie compatibility when they have a movie `title`; an untyped name-only entry is skipped. | +| AniList | The query requests both `type` and `format`. `ANIME`/`MOVIE` becomes a movie; TV, TV_SHORT, OVA, ONA and SPECIAL become series. Manga, music and unknown formats are skipped. Native `anilist:` IDs remain available for downstream ID enrichment. | + +AniList's `ANIME` describes its media category; it does not mean every entry is a +television series. Episodic AniList entries use the series import path and series +library destination. This does not enable a new absolute-numbered anime policy. +Season/episode identity still depends on the configured metadata provider. + +These parser repairs are on the review branch and are **not part of release +0.42.8**. On September 30, 2026, synthetic response tests reproduced the original +Trakt, TMDB and AniList defects before the fixes. The tests exercise the production +parsers and check that their output agrees with the episode import contract. +The repaired pinned-ABI build passed all 157 tests and published successfully. +They do not establish live provider availability, successful enrichment or playback. + +### Existing misclassified rows + +Refreshing a list does **not** automatically convert an existing movie row into +a series. Parser fixes protect newly imported rows; existing rows retain their +media type. Previously misclassified rows need a separate, backed-up repair of +their media type and managed destination. Existing movie STRMs must never be +renamed into fake episodes. Preserve list memberships, user blocks, retry history +and owned media; verify episode imports separately from playback. + +## Other implementations + +We checked other list consumers while investigating these repairs: + +- [Kometa's MDBList parser](https://github.com/Kometa-Team/Kometa/blob/master/modules/mdblist.py) + reads `mediatype`/`type` and assigns separate movie/show ID families. Its + [TMDB list builder](https://github.com/Kometa-Team/Kometa/blob/master/modules/tmdb.py) + also distinguishes movies from shows. +- [HomeScreenCompanion's list fetcher](https://github.com/soderlund91/HomeScreenCompanion/blob/main/HomeScreenCompanion/ListFetcher.cs) + collapses MDBList and Trakt results to IMDb IDs. + Its [consumer](https://github.com/soderlund91/HomeScreenCompanion/blob/main/HomeScreenCompanion/HomeScreenCompanionTask.cs) + looks up existing Movie/Series items for list selection. That does not establish + that ignoring the type is safe when creating a new import. +- [Kometa's AniList user-list reader](https://github.com/Kometa-Team/Kometa/blob/master/modules/anilist.py) + returns AniList IDs for later conversion. It does not assign every entry a + television import type at the list-parser boundary. + +These are source comparisons, not runtime tests of those projects. Their +selection and conversion workflows differ from InfiniteDrive's STRM creation. ## Catalog-less behavior From 5878f5733bf5527001ef4d75c926f08b7694fc55 Mon Sep 17 00:00:00 2001 From: OneHotTake Date: Wed, 30 Sep 2026 07:50:01 -0500 Subject: [PATCH 3/4] Protect managed catalog identities and prepare 0.42.9 --- Data/DatabaseManager.cs | 67 ++++++++++-- InfiniteDrive.csproj | 6 +- README.md | 2 +- Tasks/CatalogSyncTask.cs | 4 +- Tests/CatalogIdentityPersistenceTests.cs | 123 +++++++++++++++++++++++ docs/EXTERNAL_LISTS.md | 27 ++++- docs/README.md | 2 +- plugin.json | 2 +- 8 files changed, 209 insertions(+), 24 deletions(-) create mode 100644 Tests/CatalogIdentityPersistenceTests.cs diff --git a/Data/DatabaseManager.cs b/Data/DatabaseManager.cs index cbd76da..db3f43b 100644 --- a/Data/DatabaseManager.cs +++ b/Data/DatabaseManager.cs @@ -118,11 +118,13 @@ INSERT INTO catalog_items @tvdb_id, @raw_meta_json, @catalog_type, @videos_json, @episodes_expanded, @last_expanded_at, @last_verified_at, @source_manifest_url, @selected_versions_json, @last_version_refresh_at) ON CONFLICT(aio_id, source) DO UPDATE SET - tmdb_id = excluded.tmdb_id, + tmdb_id = COALESCE(excluded.tmdb_id, catalog_items.tmdb_id), unique_ids_json = COALESCE(excluded.unique_ids_json, catalog_items.unique_ids_json), title = excluded.title, - year = excluded.year, - media_type = excluded.media_type, + year = COALESCE(excluded.year, catalog_items.year), + media_type = CASE WHEN (catalog_items.strm_path IS NOT NULL AND catalog_items.strm_path<>'') + OR (catalog_items.item_state=3 AND catalog_items.local_source='library') + THEN catalog_items.media_type ELSE excluded.media_type END, seasons_json = COALESCE(excluded.seasons_json, catalog_items.seasons_json), strm_path = CASE WHEN catalog_items.item_state = 3 THEN NULL ELSE COALESCE(excluded.strm_path, catalog_items.strm_path) END, @@ -158,11 +160,7 @@ private void BindCatalogItemParams(IStatement cmd, CatalogItem item) BindNullableText(cmd, "@unique_ids_json", item.UniqueIdsJson); BindText(cmd, "@title", item.Title); BindNullableInt(cmd, "@year", item.Year); - var rawMediaType = item.MediaType; - var validMediaType = string.IsNullOrEmpty(rawMediaType) ? "movie" : rawMediaType; - if (validMediaType != "movie" && validMediaType != "series" && validMediaType != "anime" && validMediaType != "episode" && validMediaType != "other") - validMediaType = "movie"; - BindText(cmd, "@media_type", validMediaType); + BindText(cmd, "@media_type", ValidCatalogMediaType(item.MediaType)); BindText(cmd, "@source", item.Source); BindNullableText(cmd, "@source_list_id", item.SourceListId); BindNullableText(cmd, "@seasons_json", item.SeasonsJson); @@ -199,9 +197,47 @@ private void BindCatalogItemParams(IStatement cmd, CatalogItem item) BindNullableText(cmd, "@last_version_refresh_at", item.LastVersionRefreshAt); } + private static string ValidCatalogMediaType(string? type) => + type is "movie" or "series" or "anime" or "episode" or "other" ? type : "movie"; + + // A stale worker or source refresh must not change the import family of a + // row that already has managed media. Such conversions require a backed-up + // destination repair, not a generic catalog upsert. + private static bool ManagedCatalogIdentityCompatible(IDatabaseConnection conn, CatalogItem item) + { + using var stmt = conn.PrepareStatement(@" + SELECT media_type FROM catalog_items + WHERE aio_id=@aio_id AND source=@source + AND ((strm_path IS NOT NULL AND strm_path<>'') + OR (item_state=3 AND local_source='library'));"); + BindText(stmt, "@aio_id", item.AioId); + BindText(stmt, "@source", item.Source); + foreach (var row in stmt.AsRows()) + { + var existing = row.GetString(0); + var incoming = ValidCatalogMediaType(item.MediaType); + if (existing == incoming || (existing is "series" or "anime" && incoming is "series" or "anime")) continue; + return false; + } + return true; + } + public async Task UpsertCatalogItemAsync(CatalogItem item, CancellationToken cancellationToken = default) { - await ExecuteWriteAsync(UpsertCatalogItemSql, cmd => BindCatalogItemParams(cmd, item)); + await _dbWriteGate.WaitAsync(cancellationToken); + try + { + using var conn = OpenConnection(); + conn.RunInTransaction(c => + { + if (!ManagedCatalogIdentityCompatible(c, item)) + throw new InvalidOperationException("managed_identity_conflict"); + using var stmt = c.PrepareStatement(UpsertCatalogItemSql); + BindCatalogItemParams(stmt, item); + while (stmt.MoveNext()) { } + }); + } + finally { _dbWriteGate.Release(); } } /// @@ -259,10 +295,11 @@ UPDATE catalog_items /// Batch upsert: wraps all items in a single SQLite transaction. /// Orders of magnitude faster than calling UpsertCatalogItemAsync in a loop. /// - public async Task BulkUpsertCatalogItemsAsync(IEnumerable items, CancellationToken cancellationToken = default) + public async Task BulkUpsertCatalogItemsAsync(IEnumerable items, CancellationToken cancellationToken = default) { var itemList = items as IList ?? items.ToList(); - if (itemList.Count == 0) return; + if (itemList.Count == 0) return 0; + var accepted = 0; await _dbWriteGate.WaitAsync(cancellationToken); try @@ -272,9 +309,16 @@ public async Task BulkUpsertCatalogItemsAsync(IEnumerable items, Ca { foreach (var item in itemList) { + cancellationToken.ThrowIfCancellationRequested(); + if (!ManagedCatalogIdentityCompatible(c, item)) + { + _logger.LogWarning("[InfiniteDrive] Skipped catalog update for {AioId}: managed_identity_conflict", item.AioId); + continue; + } using var stmt = c.PrepareStatement(UpsertCatalogItemSql); BindCatalogItemParams(stmt, item); while (stmt.MoveNext()) { } + accepted++; } }); } @@ -282,6 +326,7 @@ public async Task BulkUpsertCatalogItemsAsync(IEnumerable items, Ca { _dbWriteGate.Release(); } + return accepted; } /// diff --git a/InfiniteDrive.csproj b/InfiniteDrive.csproj index af46649..bcb11fc 100755 --- a/InfiniteDrive.csproj +++ b/InfiniteDrive.csproj @@ -2,9 +2,9 @@ net8.0 - 0.42.8.0 - 0.42.8.0 - 0.42.8.0 + 0.42.9.0 + 0.42.9.0 + 0.42.9.0 enable InfiniteDrive InfiniteDrive diff --git a/README.md b/README.md index dee5119..27c6d2d 100755 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ imports titles and repairs missing files. A stream will appear. Probably. -**Current release: 0.42.8**, built and tested against **Emby 4.10.0.40**. +**Current release: 0.42.9**, built and tested against **Emby 4.10.0.40**. ## How we run it diff --git a/Tasks/CatalogSyncTask.cs b/Tasks/CatalogSyncTask.cs index c07c049..ec3c8b6 100755 --- a/Tasks/CatalogSyncTask.cs +++ b/Tasks/CatalogSyncTask.cs @@ -453,9 +453,9 @@ private async Task UpsertItemsAsync( try { - await db.BulkUpsertCatalogItemsAsync(items, cancellationToken); + var accepted = await db.BulkUpsertCatalogItemsAsync(items, cancellationToken); _logger.LogInformation( - "[InfiniteDrive] Bulk upserted {Count} catalog items", items.Count); + "[InfiniteDrive] Bulk upserted {Count}/{Total} catalog items", accepted, items.Count); } catch (Exception ex) { diff --git a/Tests/CatalogIdentityPersistenceTests.cs b/Tests/CatalogIdentityPersistenceTests.cs new file mode 100644 index 0000000..65728fd --- /dev/null +++ b/Tests/CatalogIdentityPersistenceTests.cs @@ -0,0 +1,123 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using InfiniteDrive.Data; +using InfiniteDrive.Models; +using Microsoft.Extensions.Logging.Abstractions; +using Xunit; + +namespace InfiniteDrive.Tests; + +public sealed class CatalogIdentityPersistenceTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task WrongTypeRefreshCannotResetManagedIdentityOrSafeguards(bool bulk) + { + using var h = new Harness(); + var original = h.Series(); + original.RetryCount = 2; + original.NextRetryAt = 1900000000; + original.BlockedAt = "2026-09-30T12:00:00Z"; + original.BlockedBy = "test-user"; + await h.Db.UpsertCatalogItemAsync(original); + var stale = new CatalogItem { AioId = original.AioId, Source = original.Source, + MediaType = "movie", Title = original.Title, ItemState = ItemState.Queued }; + if (bulk) Assert.Equal(0, await h.Db.BulkUpsertCatalogItemsAsync(new[] { stale })); + else + { + var error = await Assert.ThrowsAsync(() => h.Db.UpsertCatalogItemAsync(stale)); + Assert.Equal("managed_identity_conflict", error.Message); + } + var actual = (await h.Db.GetCatalogItemByAioIdAsync(original.AioId))!; + Assert.Equal(original.Id, actual.Id); + Assert.Equal("series", actual.MediaType); + Assert.Equal(2026, actual.Year); + Assert.Equal("252105", actual.TmdbId); + Assert.Equal(original.StrmPath, actual.StrmPath); + Assert.Equal(2, actual.RetryCount); + Assert.Equal(original.NextRetryAt, actual.NextRetryAt); + Assert.Equal(original.BlockedAt, actual.BlockedAt); + Assert.Equal(original.BlockedBy, actual.BlockedBy); + Assert.Equal(original.UpdatedAt, actual.UpdatedAt); + } + + [Fact] + public async Task SameTypeSparseRefreshKeepsKnownYearAndTmdb() + { + using var h = new Harness(); var original = h.Series(); + await h.Db.UpsertCatalogItemAsync(original); + await h.Db.UpsertCatalogItemAsync(new CatalogItem { AioId = original.AioId, + Source = original.Source, MediaType = "series", Title = "Updated title" }); + var actual = (await h.Db.GetCatalogItemByAioIdAsync(original.AioId))!; + Assert.Equal("Updated title", actual.Title); + Assert.Equal(2026, actual.Year); Assert.Equal("252105", actual.TmdbId); + Assert.Equal(original.StrmPath, actual.StrmPath); + } + + [Fact] + public async Task BulkIdentityConflictSkipsOnlyConflictingRow() + { + using var h = new Harness(); var original = h.Series(); + await h.Db.UpsertCatalogItemAsync(original); + var unrelated = new CatalogItem { AioId = "tt0000002", Source = "aiostreams", MediaType = "movie", Title = "Original" }; + await h.Db.UpsertCatalogItemAsync(unrelated); unrelated.Title = "Valid update"; + var wrong = new CatalogItem { AioId = original.AioId, Source = original.Source, MediaType = "movie", Title = "Wrong" }; + Assert.Equal(1, await h.Db.BulkUpsertCatalogItemsAsync(new[] { unrelated, wrong })); + Assert.Equal("Valid update", (await h.Db.GetCatalogItemByAioIdAsync(unrelated.AioId))!.Title); + Assert.Equal("series", (await h.Db.GetCatalogItemByAioIdAsync(original.AioId))!.MediaType); + } + + [Fact] + public async Task OwnedRetirementRemainsRetiredAndCannotChangeFamily() + { + using var h = new Harness(); var original = h.Series(); + await h.Db.UpsertCatalogItemAsync(original); + await h.Db.RetireCatalogItemForOwnedMediaAsync(original.Id, "/owned/series"); + var wrong = new CatalogItem { AioId = original.AioId, Source = original.Source, MediaType = "movie", Title = "Wrong" }; + await Assert.ThrowsAsync(() => h.Db.UpsertCatalogItemAsync(wrong)); + var actual = (await h.Db.GetCatalogItemByAioIdAsync(original.AioId))!; + Assert.Equal(ItemState.Retired, actual.ItemState); Assert.Null(actual.StrmPath); + Assert.Equal("library", actual.LocalSource); Assert.Equal("/owned/series", actual.LocalPath); + Assert.Equal("series", actual.MediaType); + } + + [Theory] + [InlineData("anime", "series")] + [InlineData("series", "anime")] + public async Task CompatibleEpisodeFamilyRefreshPreservesManagedDestinationType(string existing, string incoming) + { + using var h = new Harness(); var original = h.Series(); original.MediaType = existing; + await h.Db.UpsertCatalogItemAsync(original); + var refresh = new CatalogItem { AioId = original.AioId, Source = original.Source, MediaType = incoming, Title = "New title" }; + Assert.Equal(1, await h.Db.BulkUpsertCatalogItemsAsync(new[] { refresh })); + var actual = (await h.Db.GetCatalogItemByAioIdAsync(original.AioId))!; + Assert.Equal(existing, actual.MediaType); Assert.Equal(original.StrmPath, actual.StrmPath); + Assert.Equal("New title", actual.Title); + } + + [Fact] + public async Task UnpublishedRowsAndSeparateSourcesCanStillAcceptCatalogIntent() + { + using var h = new Harness(); var pending = h.Series(); pending.StrmPath = null; + await h.Db.UpsertCatalogItemAsync(pending); + pending.MediaType = "movie"; await h.Db.UpsertCatalogItemAsync(pending); + Assert.Equal("movie", (await h.Db.GetCatalogItemByAioIdAsync(pending.AioId))!.MediaType); + var separate = h.Series(); separate.Source = "aiostreams"; + await h.Db.UpsertCatalogItemAsync(separate); + Assert.Equal("series", (await h.Db.GetImportCatalogByIdAsync(separate.Id))!.MediaType); + } + + private sealed class Harness : IDisposable + { + private readonly string _path = Path.Combine(Path.GetTempPath(), "catalog-identity-" + Guid.NewGuid().ToString("N")); + public DatabaseManager Db { get; } + public Harness() { Directory.CreateDirectory(_path); Db = new DatabaseManager(_path, NullLogger.Instance); Db.Initialise(); } + public CatalogItem Series() => new() { AioId = "tt32149571", Source = "external_list", Title = "Golden Axe", + MediaType = "series", Year = 2026, TmdbId = "252105", StrmPath = "/managed/shows/Golden Axe", + LocalPath = "/managed/shows/Golden Axe", LocalSource = "strm", ItemState = ItemState.Queued, + UpdatedAt = "2026-09-30T12:04:46Z" }; + public void Dispose() { try { Directory.Delete(_path, true); } catch { } } + } +} diff --git a/docs/EXTERNAL_LISTS.md b/docs/EXTERNAL_LISTS.md index f400397..dfae90e 100644 --- a/docs/EXTERNAL_LISTS.md +++ b/docs/EXTERNAL_LISTS.md @@ -37,22 +37,39 @@ television series. Episodic AniList entries use the series import path and serie library destination. This does not enable a new absolute-numbered anime policy. Season/episode identity still depends on the configured metadata provider. -These parser repairs are on the review branch and are **not part of release -0.42.8**. On September 30, 2026, synthetic response tests reproduced the original +Release **0.42.9** includes these parser repairs. On September 30, 2026, +synthetic response tests reproduced the original Trakt, TMDB and AniList defects before the fixes. The tests exercise the production parsers and check that their output agrees with the episode import contract. -The repaired pinned-ABI build passed all 157 tests and published successfully. -They do not establish live provider availability, successful enrichment or playback. +The repaired 0.42.9 pinned-ABI build passed all 165 tests and published +successfully, including eight real SQLite cases for identity persistence. These +checks do not establish live provider availability, successful enrichment or playback. ### Existing misclassified rows Refreshing a list does **not** automatically convert an existing movie row into a series. Parser fixes protect newly imported rows; existing rows retain their -media type. Previously misclassified rows need a separate, backed-up repair of +media type. Generic single and bulk catalog upserts reject a media-type change +for a row with a managed STRM path or an owned-media retirement. This prevents a +stale worker DTO from undoing a repaired destination; it reports +`managed_identity_conflict` rather than pretending to convert the row. Bulk sync +skips only conflicting rows and reports the accepted count. Compatible +anime/series updates retain the existing managed destination type. A refresh +that omits year or TMDB ID keeps the existing known values. + +Previously misclassified rows need a separate, backed-up repair of their media type and managed destination. Existing movie STRMs must never be renamed into fake episodes. Preserve list memberships, user blocks, retry history and owned media; verify episode imports separately from playback. +### September 30, 2026 — 0.42.9 + +- Preserve movie/series identity and release year across supported list providers. +- Reject incompatible managed catalog upserts; preserve known year/TMDB values + when a refresh omits them. +- Keep existing rows, files, import journals, blocks and owned-media retirement + safeguards. There is no automatic database migration or STRM wipe. + ## Other implementations We checked other list consumers while investigating these repairs: diff --git a/docs/README.md b/docs/README.md index 9323c31..82cda38 100644 --- a/docs/README.md +++ b/docs/README.md @@ -26,4 +26,4 @@ old test plans and dated reports. These are not current configuration instructions. The implementation and current guides take precedence over historical claims. -Updated September 29, 2026 for the 0.42.8 source tree. +Updated September 30, 2026 for the 0.42.9 source tree. diff --git a/plugin.json b/plugin.json index 5a810fb..f7b6016 100755 --- a/plugin.json +++ b/plugin.json @@ -5,7 +5,7 @@ "overview": "InfiniteDrive discovers your AIOStreams catalog, writes .strm files, and resolves debrid URLs on demand. Like the Infinite Improbability Drive: a stream will appear. Probably. No external processes needed.", "owner": "InfiniteDrive", "category": "General", - "version": "0.42.8.0", + "version": "0.42.9.0", "targetAbi": "4.10.0.40", "framework": "net8.0", "imageUrl": "" From 38facad3041d94fa3ec4504782c578dd898fdfe5 Mon Sep 17 00:00:00 2001 From: OneHotTake Date: Wed, 30 Sep 2026 07:53:40 -0500 Subject: [PATCH 4/4] Initialize SQLite regression tests independently of test order --- Tests/CatalogIdentityPersistenceTests.cs | 2 +- Tests/ImportReconciliationTests.cs | 2 +- Tests/SqliteTestRuntime.cs | 17 +++++++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 Tests/SqliteTestRuntime.cs diff --git a/Tests/CatalogIdentityPersistenceTests.cs b/Tests/CatalogIdentityPersistenceTests.cs index 65728fd..96dda7c 100644 --- a/Tests/CatalogIdentityPersistenceTests.cs +++ b/Tests/CatalogIdentityPersistenceTests.cs @@ -113,7 +113,7 @@ private sealed class Harness : IDisposable { private readonly string _path = Path.Combine(Path.GetTempPath(), "catalog-identity-" + Guid.NewGuid().ToString("N")); public DatabaseManager Db { get; } - public Harness() { Directory.CreateDirectory(_path); Db = new DatabaseManager(_path, NullLogger.Instance); Db.Initialise(); } + public Harness() { SqliteTestRuntime.EnsureInitialized(); Directory.CreateDirectory(_path); Db = new DatabaseManager(_path, NullLogger.Instance); Db.Initialise(); } public CatalogItem Series() => new() { AioId = "tt32149571", Source = "external_list", Title = "Golden Axe", MediaType = "series", Year = 2026, TmdbId = "252105", StrmPath = "/managed/shows/Golden Axe", LocalPath = "/managed/shows/Golden Axe", LocalSource = "strm", ItemState = ItemState.Queued, diff --git a/Tests/ImportReconciliationTests.cs b/Tests/ImportReconciliationTests.cs index 86c4eca..c5c014a 100644 --- a/Tests/ImportReconciliationTests.cs +++ b/Tests/ImportReconciliationTests.cs @@ -15,7 +15,7 @@ namespace InfiniteDrive.Tests; public sealed class ImportReconciliationTests { - static ImportReconciliationTests() => SQLitePCLEx.raw.SetProvider(new SQLitePCLEx.SQLite3Provider_sqlite3()); + static ImportReconciliationTests() => SqliteTestRuntime.EnsureInitialized(); private static readonly DateTimeOffset Now = DateTimeOffset.Parse("2026-09-29T18:00:00Z"); private static ImportEpisode Ep(int n = 1) => new() { Key = $"aired:1:{n}", Season = 1, Episode = n, Released = Now.AddDays(-1), Eligible = true }; diff --git a/Tests/SqliteTestRuntime.cs b/Tests/SqliteTestRuntime.cs new file mode 100644 index 0000000..70f70b5 --- /dev/null +++ b/Tests/SqliteTestRuntime.cs @@ -0,0 +1,17 @@ +namespace InfiniteDrive.Tests; + +internal static class SqliteTestRuntime +{ + private static readonly object Gate = new(); + private static bool _initialized; + + internal static void EnsureInitialized() + { + lock (Gate) + { + if (_initialized) return; + SQLitePCLEx.raw.SetProvider(new SQLitePCLEx.SQLite3Provider_sqlite3()); + _initialized = true; + } + } +}