From d748e0996f27deda861db44b732c144b10233a30 Mon Sep 17 00:00:00 2001 From: Thoroslives Date: Fri, 3 Jul 2026 07:08:46 +1000 Subject: [PATCH 1/5] Add a persistent index of ThePosterDB user uploads for incremental scrapes --- README.md | 12 +- artwork_uploader.py | 3 +- core/config.py | 10 ++ core/constants.py | 1 + models/arguments.py | 2 + models/options.py | 2 + scrapers/theposterdb_scraper.py | 225 ++++++++++++++++++++++++++- services/__init__.py | 4 +- services/asset_index.py | 266 ++++++++++++++++++++++++++++++++ static/web_interface.js | 3 + templates/web_interface.html | 4 + tests/test_asset_index.py | 139 +++++++++++++++++ utils/utils.py | 4 +- 13 files changed, 667 insertions(+), 8 deletions(-) create mode 100644 services/asset_index.py create mode 100644 tests/test_asset_index.py diff --git a/README.md b/README.md index c2b0912..451383c 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ We (optionally) store an artwork ID in a Plex label against each movie, show, ep If you really want to upload artwork again, use the ```--force``` option at the command line, in the bulk file, or when entering the URL in the Web UI. ### ThePosterDB scraping -There are also a couple of new options for thePosterDb, which will allow you to also grab additional sets and additional posters from the same page. This is sometimes useful for big sets like the Marvel or Disney movies, where you'll otherwise need to specify multiple sets. This is against the terms of service of theposterdb.com so we encourage you to login, download the files you want, and upload them using this tool, rather than scraping. Once an API is available we'll switch over ASAP. +There are also a couple of new options for thePosterDb, which will allow you to also grab additional sets and additional posters from the same page. This is sometimes useful for big sets like the Marvel or Disney movies, where you'll otherwise need to specify multiple sets. This is against the terms of service of theposterdb.com so we encourage you to login, download the files you want, and upload them using this tool, rather than scraping. Once an API is available we'll switch over ASAP. If you regularly scrape the same ThePosterDB users, enable ```cache_user_scrapes``` so repeat scrapes only fetch new uploads instead of re-crawling the whole catalogue every time. ### Per-URL filtering and artwork excludes And there are other options such as per-URL filtering, fixing missing things that I found while I was using the tool (where I wanted to apply episode title cards but didn't like the season artwork for example). And if you don't like a particular piece of artwork or poster from a set, you can now exclude it. You can also exclude entire seasons or individual episodes, that way the app doesn't have to provess all the previous seasons for which you already have artwork applied. @@ -159,7 +159,7 @@ Basic scheduler, so that you can leave this running and update all your artwork The basic version is available now on the bulk imports page, click on the clock to enable or disable per file. -It's there for when we have API access (and works for scrapers in the meantime) but is limited to running once a day which should be fine. +It's there for when we have API access (and works for scrapers in the meantime) but is limited to running once a day which should be fine. If you enable ```cache_user_scrapes```, a daily scheduled user scrape only fetches uploads that are new since the last run, so scheduled full-catalogue runs stay fast. If you enable ```skip_locked_artwork```, a scheduled user or bulk scrape will only fill items that are still on their default artwork and leave anything you've set by hand untouched, so it's safe to leave running against a curated library. @@ -232,6 +232,12 @@ This is optional - if you don't do this, a new config.json will be created when ```"skip_locked_artwork"``` - Setting this to ```true``` will skip any artwork whose target field (poster, background or square art) is locked in Plex, unless ```--force``` is used. Plex locks a field whenever artwork is deliberately set - manually or by an upload - so this makes scheduled bulk imports and user scrapes fill items still on default artwork while leaving anything you've already set alone. - Setting to ```false``` (the default) keeps the existing behaviour where artwork is applied regardless of locks. +```"cache_user_scrapes"``` +- Setting this to ```true``` keeps a local index of each ThePosterDB user's uploads (a small SQLite file in your config directory), so scraping a user page again only fetches pages until it reaches uploads it has already seen - full-catalogue re-runs drop from hundreds of page requests to a couple, which is also much kinder to ThePosterDB. Every ```user_cache_refresh_days``` days the next scrape of that user re-crawls every page to pick up edited or deleted uploads. +- Setting to ```false``` (the default) crawls every page on every user scrape, exactly as before. + +```"user_cache_refresh_days"``` +- The number of days between full re-crawls of a cached user's uploads (default ```7```). Only used when ```cache_user_scrapes``` is ```true```. ```"auto_manage_bulk_files"``` - Setting this to ```true``` will automatically add, label and sort URLs from the scrape tab into the currently loaded bulk import file. At the moment it won't auto-save, but I might add that later. @@ -387,6 +393,8 @@ The script supports various command-line arguments for flexible use. ```--stage```, in conjuction with --kometa (or if ```save_to_kometa```is true in ```config.json```), will download assets for TV show seasons and episodes not yet available in Plex. This can be useful if you run the script before a particular season or episode is downloaded by your automation. If ```stage_assets``` is set to ```true``` in ```config.json``` then this argument is not necessary. This option does not apply to the Specials season (Season 0). +```--no-cache``` will crawl every page of a ThePosterDB user this run, ignoring the cached index (the index is still refreshed). Handy to force a full refresh of a user when you have ```cache_user_scrapes``` enabled. Works on the command line, in bulk files and in the Web UI. + ### Using these options in files and GUI These options can also be used in the URL scraper GUI, and in your bulk file, just add them straight after the URL in each line, for example diff --git a/artwork_uploader.py b/artwork_uploader.py index 47b91ed..d18e3db 100644 --- a/artwork_uploader.py +++ b/artwork_uploader.py @@ -670,7 +670,8 @@ def update_scheduled_jobs(): year=args.year, kometa=args.kometa, stage=args.stage, - temp=args.temp) # Arguments per url to process + temp=args.temp, + no_cache=args.no_cache) # Arguments per url to process # Create config as a global object config = Config() diff --git a/core/config.py b/core/config.py index f593424..ec0e669 100644 --- a/core/config.py +++ b/core/config.py @@ -31,6 +31,8 @@ class Config: stage_assets: Whether to download assets for seasons and episodes that are not in Plex yet (except Specials) track_artwork_ids: Whether to track artwork IDs using Plex labels skip_locked_artwork: Whether to skip artwork whose target field is locked in Plex (already set) + cache_user_scrapes: Whether to keep a persistent index of ThePosterDB users' uploads so repeat scrapes only fetch new ones + user_cache_refresh_days: Days between full re-crawls of a cached user's uploads (catches edits and deletions) auto_manage_bulk_files: Whether to auto-organize bulk files reset_overlay: Whether to reset Kometa overlay labels on upload schedules: List of scheduled bulk import jobs @@ -55,6 +57,8 @@ def __init__(self, config_path: str = "config/config.json") -> None: self.stage_assets: bool = False self.track_artwork_ids: bool = True self.skip_locked_artwork: bool = False + self.cache_user_scrapes: bool = False + self.user_cache_refresh_days: int = 7 self.auto_manage_bulk_files: bool = True self.reset_overlay: bool = False self.schedules: List[Dict[str, Any]] = [] @@ -93,6 +97,8 @@ def load(self) -> None: self.bulk_txt = config.get("bulk_txt", "bulk_import.txt") self.track_artwork_ids = config.get("track_artwork_ids", True) self.skip_locked_artwork = config.get("skip_locked_artwork", False) + self.cache_user_scrapes = config.get("cache_user_scrapes", False) + self.user_cache_refresh_days = config.get("user_cache_refresh_days", 7) self.auto_manage_bulk_files = config.get("auto_manage_bulk_files", True) self.reset_overlay = config.get("reset_overlay", False) self.schedules = config.get("schedules", []) @@ -120,6 +126,8 @@ def create(self) -> None: "stage_assets": False, "track_artwork_ids": True, "skip_locked_artwork": False, + "cache_user_scrapes": False, + "user_cache_refresh_days": 7, "auto_manage_bulk_files": True, "reset_overlay": True, "schedules": [], @@ -161,6 +169,8 @@ def save(self) -> None: "bulk_txt": self.bulk_txt, "track_artwork_ids": self.track_artwork_ids, "skip_locked_artwork": self.skip_locked_artwork, + "cache_user_scrapes": self.cache_user_scrapes, + "user_cache_refresh_days": self.user_cache_refresh_days, "auto_manage_bulk_files": self.auto_manage_bulk_files, "reset_overlay": self.reset_overlay, "schedules": self.schedules, diff --git a/core/constants.py b/core/constants.py index 26fdee2..6c38996 100644 --- a/core/constants.py +++ b/core/constants.py @@ -20,6 +20,7 @@ # File paths DEFAULT_CONFIG_PATH = "config.json" +ASSET_INDEX_PATH = "config/asset_index.db" DEFAULT_BULK_IMPORTS_DIR = "bulk_imports" DEFAULT_BULK_IMPORT_FILE = "bulk_import.txt" diff --git a/models/arguments.py b/models/arguments.py index 880d078..6f268fc 100644 --- a/models/arguments.py +++ b/models/arguments.py @@ -15,6 +15,7 @@ # --kometa Saves artwork to Kometa asset directory (specified in config file) instead of uploading to Plex. # --stage Downloads artwork for seasons and episodes that are not in Plex yet (except Specials). # --temp Uses a temporary directory (specified in config file) instead of the Kometa asset directory. +# --no-cache Ignore the cached ThePosterDB user uploads index for this run and crawl every page. # --------------------------------------------------------- def parse_arguments(): @@ -35,5 +36,6 @@ def parse_arguments(): parser.add_argument("--kometa", action='store_true', help="Saves artwork to Kometa asset directory (specified in config file) instead of uploading to Plex.") parser.add_argument("--stage", action='store_true', help="Downloads artwork for seasons and episodes that are not in Plex yet (except Specials).") parser.add_argument("--temp", action='store_true', help="Uses a temporary directory (specified in config file) instead of the Kometa asset directory.") + parser.add_argument("--no-cache", action='store_true', help="Ignore the cached ThePosterDB user uploads index for this run and crawl every page (the run still refreshes the index).") return parser.parse_args() diff --git a/models/options.py b/models/options.py index 439f649..da15e4a 100644 --- a/models/options.py +++ b/models/options.py @@ -21,6 +21,7 @@ class Options: temp: Use temporary directory instead of Kometa asset directory force: Force re-upload even if artwork hasn't changed skip_locked: Skip artwork when the target Plex field is locked (already set) + no_cache: Ignore the cached user uploads index for this run and crawl every page filters: List of artwork types to include (e.g., ['show_cover', 'title_card']) exclude: List of artwork IDs to skip year: Override year for Plex matching @@ -34,6 +35,7 @@ class Options: temp: bool = False force: bool = False skip_locked: bool = False + no_cache: bool = False filters: List[str] = field(default_factory=list) exclude: Optional[List[str]] = None year: Optional[int] = None diff --git a/scrapers/theposterdb_scraper.py b/scrapers/theposterdb_scraper.py index bd32f9a..ae7587f 100644 --- a/scrapers/theposterdb_scraper.py +++ b/scrapers/theposterdb_scraper.py @@ -13,6 +13,11 @@ from core.constants import TPDB_API_ASSETS_URL, TPDB_USER_UPLOADS_PER_PAGE, BOOTSTRAP_COLORS, ANSI_RESET, ANSI_BOLD from models.artwork_types import MovieArtworkList, TVArtworkList, CollectionArtworkList +import sqlite3 +from datetime import datetime, timezone +from urllib.parse import urlparse +from services.asset_index import AssetIndex, page_is_fully_known, full_crawl_due + class ThePosterDBScraper: @@ -40,6 +45,10 @@ def __init__(self, url: str, callbacks: Optional[ProcessingCallbacks]) -> None: self.user_uploads: int = 0 self.user_pages: int = 0 + # When set to a list, get_posters records every parsed asset (pre-filter) into it so a + # user crawl can populate the persistent index. None (default) disables recording. + self.catalog: Optional[list] = None + # Set options - otherwise will use defaults of False def set_options(self, options: Options) -> None: @@ -73,6 +82,16 @@ def scrape(self) -> None: self.callbacks.debug(f"There are {self.user_uploads} assets and {self.user_pages} pages for user {self.author}") self.callbacks.progress(0, 0, f"Collecting assets from TPDb user {self.author}") + # With the persistent index enabled, only fetch pages until we reach uploads we + # have already seen, then apply the user's full catalogue from the index. + if self.config.cache_user_scrapes: + try: + self._scrape_user_cached() + return + except sqlite3.Error as cache_error: + self.callbacks.debug(f"Asset index unavailable ({cache_error}); crawling every page", "ThePosterDBScraper/scrape") + self._reset_user_collections() + collected = 0 for user_page in range(self.user_pages): if globals.cancel_scrape: @@ -152,13 +171,15 @@ def scrape_user_info(self) -> None: except (AttributeError, KeyError, ValueError, TypeError) as e: raise ScraperException(f"Can't get user information, please check the URL you're using") from e - def scrape_user_page (self, page) -> None: + def scrape_user_page(self, page, catalog=None) -> bool: try: page_url = f"{self.url}?section=uploads&page={page + 1}" child_scraper = ThePosterDBScraper(page_url, self.callbacks) child_scraper.set_options(self.options) child_scraper.is_child = True + if catalog is not None: + child_scraper.catalog = catalog child_scraper.scrape() for artwork in child_scraper.collection_artwork: @@ -173,9 +194,11 @@ def scrape_user_page (self, page) -> None: self.filtered += child_scraper.filtered self.errored += child_scraper.errored self.total += child_scraper.total + return True except Exception as e: - self.callbacks.debug(f"Failed to scrape user asset page {page}: {str(e)}") + self.callbacks.debug(f"Failed to scrape user asset page {page}: {str(e)}", "ThePosterDBScraper/scrape_user_page") + return False def get_set_title(self, soup: Any) -> None: try: @@ -236,6 +259,14 @@ def get_posters(self, poster_div: Any) -> None: file_type = "show_cover" else: file_type = "season_cover" + + if self.catalog is not None: + self.catalog.append({ + "id": poster_id, "title": title, "year": year, + "season": season if isinstance(season, int) else None, + "media_type": file_type, "author": self.author, + "url": f"{TPDB_API_ASSETS_URL}/{poster_id}", + }) if (self.options.has_no_filters() and file_type in self.config.tpdb_filters) or self.options.has_filter(file_type): if not self.options.is_excluded(poster_id, season if isinstance(season, int) else None, None): @@ -272,6 +303,12 @@ def get_posters(self, poster_div: Any) -> None: ) elif media_type == MediaType.MOVIE.value: title, year = media_metadata.parse_movie(title_p) + if self.catalog is not None: + self.catalog.append({ + "id": poster_id, "title": title, "year": year, "season": None, + "media_type": "movie_poster", "author": self.author, + "url": f"{TPDB_API_ASSETS_URL}/{poster_id}", + }) if (self.options.has_no_filters() and "movie_poster" in self.config.tpdb_filters) or self.options.has_filter("movie_poster"): if not self.options.is_excluded(poster_id): self.callbacks.debug(f"{i+1}. ✅ Including movie poster for '{title} ({year})'.") @@ -293,6 +330,12 @@ def get_posters(self, poster_div: Any) -> None: self.filtered += 1 self.callbacks.debug(f"{i+1}. ⏩ Skipping movie poster for '{title} ({year})' based on filters.") elif media_type == MediaType.COLLECTION.value: + if self.catalog is not None: + self.catalog.append({ + "id": poster_id, "title": title_p, "year": None, "season": None, + "media_type": "collection_poster", "author": self.author, + "url": f"{TPDB_API_ASSETS_URL}/{poster_id}", + }) if (self.options.has_no_filters() and "collection_poster" in self.config.tpdb_filters) or self.options.has_filter("collection_poster"): if not self.options.is_excluded(poster_id): self.callbacks.debug(f"{i+1}. ✅ Including collection poster for '{title_p}'.") @@ -312,6 +355,12 @@ def get_posters(self, poster_div: Any) -> None: self.filtered += 1 self.callbacks.debug(f"{i+1}. ⏩ Skipping collection poster for '{title_p}' based on filters.") else: + if self.catalog is not None: + self.catalog.append({ + "id": poster_id, "title": title_p, "year": None, "season": None, + "media_type": "unknown", "author": self.author, + "url": f"{TPDB_API_ASSETS_URL}/{poster_id}", + }) self.errored += 1 self.callbacks.debug(f"⏩ Skipping artwork item - unknown media type: {title_p} | {poster_url}") self.callbacks.log(f"{f'⚠️ {self.title} • ' if self.title is not None else '⚠️ '}{self.author} | Skipping asset (unknown media type): {title_p}") @@ -353,3 +402,175 @@ def scrape_additional_sets(self) -> None: def scrape_posters(self, soup: Any) -> None: poster_div = soup.find('div', class_='row d-flex flex-wrap m-0 w-100 mx-n1 mt-n1') return self.get_posters(poster_div) + + def _reset_user_collections(self) -> None: + """Clear the collected artwork lists and skip counters for a user scrape.""" + self.movie_artwork = [] + self.tv_artwork = [] + self.collection_artwork = [] + self.skipped = 0 + self.exclusions = 0 + self.filtered = 0 + self.errored = 0 + self.total = 0 + + def _user_key(self) -> str: + """Stable key for a user URL: the handle after /user/, lower-cased so a web-entered + (lower-cased) URL and a CLI-cased one share a single index entry.""" + parts = [segment for segment in urlparse(self.url).path.split("/") if segment] + if "user" in parts: + index = parts.index("user") + if index + 1 < len(parts): + return parts[index + 1].casefold() + return "" + + def _scrape_user_cached(self) -> None: + """Incrementally crawl a user's uploads using the persistent index, then apply the + full catalogue from the index. Raises sqlite3.Error if the index is unusable, so the + caller can fall back to a full crawl.""" + index = AssetIndex() + user_key = self._user_key() + crawl_started_at = datetime.now(timezone.utc).isoformat() + state = index.crawl_state(user_key) + full = self.options.no_cache or full_crawl_due( + state["last_full_crawl"] if state else None, self.config.user_cache_refresh_days) + self.callbacks.debug( + f"Cache {'full crawl' if full else 'incremental crawl'} for user {self.author}", + "ThePosterDBScraper/scrape") + + new_rows, seen_ids, clean = self._crawl_user_pages(index, user_key, full) + + # Cross-check ThePosterDB's own upload counter: if an incremental crawl's arithmetic + # doesn't add up, something moved below the stop point (an edit-bumped re-order or a + # deletion) - promote this run to a full crawl once so nothing is missed. + if not full and clean and state is not None: + expected = (state["last_seen_count"] or 0) + new_rows + if self.user_uploads != expected: + self.callbacks.debug( + f"Upload count for {self.author} changed unexpectedly " + f"(ThePosterDB reports {self.user_uploads}, expected {expected}); re-crawling every page", + "ThePosterDBScraper/scrape") + full = True + new_rows, seen_ids, clean = self._crawl_user_pages(index, user_key, full=True) + + # Only reconcile/record over a crawl with no failed pages, so an invisible page error + # never mass-tombstones live assets. + if clean: + if full: + index.reconcile(user_key, seen_ids, crawl_started_at) + index.record_crawl(user_key, full, self.user_uploads) + + self._hydrate_from_cache(index, user_key) + + def _crawl_user_pages(self, index: AssetIndex, user_key: str, full: bool): + """Crawl the user's upload pages, recording each into the index. An incremental crawl + stops after the first page whose assets are all already indexed. Returns + (new_rows_recorded, ids_seen_this_crawl, all_pages_succeeded).""" + known = index.known_ids(user_key) + seen_ids = set() + new_rows = 0 + clean = True + collected = 0 + for user_page in range(self.user_pages): + self.callbacks.progress(user_page + 1, self.user_pages, f"Collecting assets from TPDb user {self.author} • {user_page + 1} of {self.user_pages} pages • {collected} assets collected of {self.user_uploads}") + page_catalog = [] + ok = self.scrape_user_page(user_page, catalog=page_catalog) + page_ids = {int(asset["id"]) for asset in page_catalog if str(asset.get("id", "")).isdigit()} + if ok: + new_rows += index.record(user_key, page_catalog) + seen_ids |= page_ids + collected += len(page_catalog) + else: + clean = False + self.callbacks.debug(f"Processed {user_page + 1} out of {self.user_pages} user pages. Collected {collected} assets so far", "ThePosterDBScraper/scrape") + if not full and ok and page_is_fully_known(page_ids, known): + self.callbacks.debug(f"Reached already-indexed uploads at page {user_page + 1}, stopping incremental crawl", "ThePosterDBScraper/scrape") + break + return new_rows, seen_ids, clean + + def _hydrate_from_cache(self, index: AssetIndex, user_key: str) -> None: + """Rebuild the artwork lists from the index, applying this run's filters/exclusions + exactly as get_posters does, so a cached run produces the same result as a full crawl.""" + self._reset_user_collections() + cache_buster = f"&_cb={int(time.time())}" + for i, row in enumerate(index.assets_for_user(user_key)): + media_type = row["media_type"] + poster_id = str(row["asset_id"]) + title = row["title"] + year = row["year"] + poster_url = f"{row['url']}{cache_buster}" + + if media_type in ("show_cover", "season_cover"): + file_type = media_type + season = "Cover" if media_type == "show_cover" else row["season"] + if (self.options.has_no_filters() and file_type in self.config.tpdb_filters) or self.options.has_filter(file_type): + if not self.options.is_excluded(poster_id, season if isinstance(season, int) else None, None): + self.callbacks.debug( + f"{i+1}. ✅ Including {file_type.replace('_', ' ')} for '{title} ({year})'" + + (f", Season {season}." if isinstance(season, int) else "."), "ThePosterDBScraper/get_posters") + self.tv_artwork.append({ + "title": title, + "author": self.author, + "tmdb_id": self.tmdb_id, + "url": poster_url, + "season": season, + "episode": None, + "year": year, + "source": ScraperSource.THEPOSTERDB.value, + "id": poster_id, + "type": file_type, + }) + else: + self.exclusions += 1 + self.callbacks.debug( + f"{i+1}. ⏩ Skipping {file_type.replace('_', ' ')} for '{title} ({year})'" + + (f", Season {season}." if isinstance(season, int) else "") + + f" based on exclusions.", "ThePosterDBScraper/get_posters") + else: + self.filtered += 1 + self.callbacks.debug( + f"{i+1}. ⏩ Skipping {file_type.replace('_', ' ')} for '{title} ({year})'" + + (f", Season {season}." if isinstance(season, int) else "") + + f" based on filters.", "ThePosterDBScraper/get_posters") + elif media_type == "movie_poster": + if (self.options.has_no_filters() and "movie_poster" in self.config.tpdb_filters) or self.options.has_filter("movie_poster"): + if not self.options.is_excluded(poster_id): + self.callbacks.debug(f"{i+1}. ✅ Including movie poster for '{title} ({year})'.", "ThePosterDBScraper/get_posters") + self.movie_artwork.append({ + "title": title, + "author": self.author, + "tmdb_id": self.tmdb_id, + "url": poster_url, + "year": year, + "source": ScraperSource.THEPOSTERDB.value, + "id": poster_id, + "type": "movie_poster", + }) + else: + self.exclusions += 1 + self.callbacks.debug(f"{i+1}. ⏩ Skipping movie poster for '{title} ({year})' based on exclusions.", "ThePosterDBScraper/get_posters") + else: + self.filtered += 1 + self.callbacks.debug(f"{i+1}. ⏩ Skipping movie poster for '{title} ({year})' based on filters.", "ThePosterDBScraper/get_posters") + elif media_type == "collection_poster": + if (self.options.has_no_filters() and "collection_poster" in self.config.tpdb_filters) or self.options.has_filter("collection_poster"): + if not self.options.is_excluded(poster_id): + self.callbacks.debug(f"{i+1}. ✅ Including collection poster for '{title}'.", "ThePosterDBScraper/get_posters") + self.collection_artwork.append({ + "title": title, + "author": self.author, + "url": poster_url, + "source": ScraperSource.THEPOSTERDB.value, + "id": poster_id, + "type": "collection_poster", + }) + else: + self.exclusions += 1 + self.callbacks.debug(f"{i+1}. ⏩ Skipping collection poster for '{title}' based on exclusions.", "ThePosterDBScraper/get_posters") + else: + self.filtered += 1 + self.callbacks.debug(f"{i+1}. ⏩ Skipping collection poster for '{title}' based on filters.", "ThePosterDBScraper/get_posters") + + self.skipped = self.exclusions + self.filtered + self.errored + self.total = len(self.movie_artwork) + len(self.tv_artwork) + len(self.collection_artwork) + self.skipped + self.callbacks.debug(f"---------> Total assets applied from cache: {len(self.movie_artwork) + len(self.tv_artwork) + len(self.collection_artwork)} of {self.user_uploads}", "ThePosterDBScraper/scrape") diff --git a/services/__init__.py b/services/__init__.py index 025c7b9..bf58f04 100644 --- a/services/__init__.py +++ b/services/__init__.py @@ -13,6 +13,7 @@ from .utility_service import UtilityService from .authentication_service import AuthenticationService from .notify_service import NotifyService +from .asset_index import AssetIndex __all__ = [ 'BulkFileService', @@ -23,5 +24,6 @@ #'UpdateService', 'UtilityService', 'AuthenticationService', - 'NotifyService' + 'NotifyService', + 'AssetIndex' ] diff --git a/services/asset_index.py b/services/asset_index.py new file mode 100644 index 0000000..0fbe2e1 --- /dev/null +++ b/services/asset_index.py @@ -0,0 +1,266 @@ +""" +Persistent index of ThePosterDB users' uploads. + +A small SQLite database (one file in the config directory) that mirrors the assets a +ThePosterDB user has uploaded. Repeat scrapes of that user then only need to fetch pages +until they reach uploads that are already indexed, and other features can look cached +artwork up by title. +""" + +import os +import re +import sqlite3 +import time +import unicodedata +from datetime import datetime, timezone +from typing import List, Optional, Set + +from core.constants import ASSET_INDEX_PATH + + +def normalize_title(title: str) -> str: + """Lowercase, strip accents and punctuation so titles compare equal regardless of styling, + e.g. 'Mission: Impossible' vs 'Mission - Impossible', 'Léon' vs 'Leon'.""" + title = unicodedata.normalize("NFKD", title).encode("ascii", "ignore").decode() + title = title.casefold().replace("&", " and ") + title = re.sub(r"[^\w\s]", " ", title) + return re.sub(r"\s+", " ", title).strip() + + +def _now() -> str: + """Current time as an ISO-8601 UTC string - sortable and comparable as plain text.""" + return datetime.now(timezone.utc).isoformat() + + +_CREATE_ASSETS = """ + CREATE TABLE IF NOT EXISTS user_assets ( + user_key TEXT NOT NULL, + asset_id INTEGER NOT NULL, + title TEXT, + title_key TEXT, + year INTEGER, + season INTEGER, + media_type TEXT, + author TEXT, + url TEXT, + first_seen TEXT, + last_seen TEXT, + missing_since TEXT, + PRIMARY KEY (user_key, asset_id) + ) +""" + +_CREATE_ASSETS_INDEX = """ + CREATE INDEX IF NOT EXISTS idx_user_assets_title + ON user_assets (title_key, media_type) +""" + +_CREATE_CRAWLS = """ + CREATE TABLE IF NOT EXISTS user_crawls ( + user_key TEXT PRIMARY KEY, + last_full_crawl TEXT, + last_crawl TEXT, + last_seen_count INTEGER + ) +""" + + +class AssetIndex: + """ + Persistent index of ThePosterDB users' uploads (SQLite, one file in the config + directory). Writers are the scrape thread; readers may run on other threads, so the + connection is short-lived per call and the database runs in WAL mode. + """ + + def __init__(self, path: str = ASSET_INDEX_PATH) -> None: + self.path = path + try: + self._ensure_schema() + except sqlite3.DatabaseError as e: + # debug_me lives in utils.notifications, which imports the services package - import + # it lazily here to avoid a start-up import cycle when the scraper pulls in the index. + from utils.notifications import debug_me + # The file exists but is not a valid SQLite database (corruption, a truncated + # write on a flaky filesystem, ...). Preserve it for inspection and start clean: + # the next crawl repopulates it, nothing is lost that a scrape can't rebuild. + corrupt = f"{self.path}.corrupt-{int(time.time())}" + try: + os.rename(self.path, corrupt) + except OSError: + debug_me(f"Asset index at '{self.path}' is unreadable ({e}) and could not be " + f"moved aside; giving up on the cache.", "AssetIndex") + raise + debug_me(f"Asset index at '{self.path}' was unreadable ({e}); moved it to " + f"'{corrupt}' and started a fresh index.", "AssetIndex") + self._ensure_schema() + + def _connect(self) -> sqlite3.Connection: + conn = sqlite3.connect(self.path, timeout=30) + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA busy_timeout=30000") + conn.row_factory = sqlite3.Row + return conn + + def _ensure_schema(self) -> None: + conn = self._connect() + try: + conn.execute("PRAGMA user_version = 1") + conn.execute(_CREATE_ASSETS) + conn.execute(_CREATE_ASSETS_INDEX) + conn.execute(_CREATE_CRAWLS) + conn.commit() + finally: + conn.close() + + def known_ids(self, user_key: str) -> Set[int]: + """Every asset id on record for the user, tombstoned ones included, so the stop rule + stays conservative (a re-listed deleted asset still counts as already known).""" + conn = self._connect() + try: + rows = conn.execute( + "SELECT asset_id FROM user_assets WHERE user_key = ?", (user_key,) + ).fetchall() + finally: + conn.close() + return {row["asset_id"] for row in rows} + + def record(self, user_key: str, assets: List[dict]) -> int: + """Upsert a batch of raw (pre-filter) assets for a user, returning how many were not + already indexed - the count the upload-count ledger checks against ThePosterDB.""" + if not assets: + return 0 + now = _now() + known = self.known_ids(user_key) + rows = [] + new_count = 0 + for asset in assets: + try: + asset_id = int(asset["id"]) + except (KeyError, TypeError, ValueError): + continue + if asset_id not in known: + new_count += 1 + known.add(asset_id) + title = asset.get("title") + season = asset.get("season") + rows.append(( + user_key, asset_id, title, normalize_title(title) if title else "", + asset.get("year"), season if isinstance(season, int) else None, + asset.get("media_type"), asset.get("author"), asset.get("url"), now, now, + )) + if not rows: + return 0 + conn = self._connect() + try: + conn.executemany( + """ + INSERT INTO user_assets + (user_key, asset_id, title, title_key, year, season, media_type, + author, url, first_seen, last_seen) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(user_key, asset_id) DO UPDATE SET + title=excluded.title, title_key=excluded.title_key, year=excluded.year, + season=excluded.season, media_type=excluded.media_type, + author=excluded.author, url=excluded.url, last_seen=excluded.last_seen, + missing_since=NULL + """, + rows, + ) + conn.commit() + finally: + conn.close() + return new_count + + def assets_for_user(self, user_key: str) -> List[sqlite3.Row]: + """Live (non-tombstoned, known-media-type) assets for the user, newest first - the + order a full crawl produces.""" + conn = self._connect() + try: + return conn.execute( + """ + SELECT * FROM user_assets + WHERE user_key = ? AND missing_since IS NULL AND media_type != 'unknown' + ORDER BY asset_id DESC + """, + (user_key,), + ).fetchall() + finally: + conn.close() + + def reconcile(self, user_key: str, seen_ids: Set[int], crawl_started_at: str) -> int: + """After a clean full crawl, tombstone assets that weren't seen (deleted from the + user's uploads), sparing any row inserted since the crawl began so an overlapping + scrape's fresh uploads are never tombstoned. Returns the number tombstoned.""" + conn = self._connect() + try: + conn.execute("CREATE TEMP TABLE IF NOT EXISTS seen (asset_id INTEGER PRIMARY KEY)") + conn.execute("DELETE FROM seen") + conn.executemany("INSERT OR IGNORE INTO seen (asset_id) VALUES (?)", + [(asset_id,) for asset_id in seen_ids]) + cursor = conn.execute( + """ + UPDATE user_assets SET missing_since = ? + WHERE user_key = ? AND missing_since IS NULL + AND asset_id NOT IN (SELECT asset_id FROM seen) + AND last_seen < ? + """, + (_now(), user_key, crawl_started_at), + ) + conn.commit() + return cursor.rowcount + finally: + conn.close() + + def crawl_state(self, user_key: str) -> Optional[sqlite3.Row]: + """The user's crawl bookkeeping row (last full/any crawl, last upload count), or None.""" + conn = self._connect() + try: + return conn.execute( + "SELECT * FROM user_crawls WHERE user_key = ?", (user_key,) + ).fetchone() + finally: + conn.close() + + def record_crawl(self, user_key: str, full: bool, seen_count: int) -> None: + """Update the crawl ledger. last_full_crawl only advances on a full crawl.""" + now = _now() + conn = self._connect() + try: + existing = conn.execute( + "SELECT last_full_crawl FROM user_crawls WHERE user_key = ?", (user_key,) + ).fetchone() + last_full = now if full else (existing["last_full_crawl"] if existing else None) + conn.execute( + """ + INSERT INTO user_crawls (user_key, last_full_crawl, last_crawl, last_seen_count) + VALUES (?, ?, ?, ?) + ON CONFLICT(user_key) DO UPDATE SET + last_full_crawl=excluded.last_full_crawl, + last_crawl=excluded.last_crawl, + last_seen_count=excluded.last_seen_count + """, + (user_key, last_full, now, seen_count), + ) + conn.commit() + finally: + conn.close() + + +def page_is_fully_known(page_ids: Set[int], known_ids: Set[int]) -> bool: + """Incremental stop rule: a page halts the crawl only if it had assets and every one was + already in the index. A page with any new asset (even mixed with known ones) keeps the + crawl going, which is why the rule is page- rather than item-granular - ThePosterDB + pages are newest-first by upload batch but not strictly by id within a batch.""" + return bool(page_ids) and page_ids <= known_ids + + +def full_crawl_due(last_full_crawl: Optional[str], refresh_days: int) -> bool: + """True when the user has never had a full crawl or the last one is at least refresh_days + old - the next scrape then re-crawls every page to catch edits and deletions.""" + if not last_full_crawl: + return True + try: + last = datetime.fromisoformat(last_full_crawl) + except (TypeError, ValueError): + return True + return (datetime.now(timezone.utc) - last).total_seconds() >= refresh_days * 86400 diff --git a/static/web_interface.js b/static/web_interface.js index 7ad2188..9d75186 100644 --- a/static/web_interface.js +++ b/static/web_interface.js @@ -631,6 +631,8 @@ function saveConfig() { // Checkbox for skipping artwork with locked fields in Plex save_config.skip_locked_artwork = document.getElementById("skip_locked_artwork").checked; toggleSkipLockedCheckbox(); + // Checkbox for caching ThePosterDB user scrapes + save_config.cache_user_scrapes = document.getElementById("cache_user_scrapes").checked; // Get selected mediux filters save_config.mediux_filters = Array.from(document.querySelectorAll('[id^="m_filter-"]:checked')) @@ -709,6 +711,7 @@ function loadConfig() { document.getElementById("auto_manage_bulk_files").checked = data.config.auto_manage_bulk_files; document.getElementById("reset_overlay").checked = data.config.reset_overlay; document.getElementById("skip_locked_artwork").checked = data.config.skip_locked_artwork; + document.getElementById("cache_user_scrapes").checked = data.config.cache_user_scrapes; document.getElementById("option-add-to-bulk").checked = data.config.auto_manage_bulk_files; document.getElementById("apprise_urls").value = data.config.apprise_urls.join(", "); diff --git a/templates/web_interface.html b/templates/web_interface.html index b8ca8ac..04c49ae 100644 --- a/templates/web_interface.html +++ b/templates/web_interface.html @@ -396,6 +396,10 @@
 Additional settings
+
+ + +
diff --git a/tests/test_asset_index.py b/tests/test_asset_index.py new file mode 100644 index 0000000..b9dba20 --- /dev/null +++ b/tests/test_asset_index.py @@ -0,0 +1,139 @@ +"""Unit tests for the persistent ThePosterDB user-uploads index (services/asset_index.py).""" + +import glob +from datetime import datetime, timedelta, timezone + +import pytest + +from services.asset_index import ( + AssetIndex, + normalize_title, + page_is_fully_known, + full_crawl_due, +) + +FAR_FUTURE = "2999-01-01T00:00:00+00:00" +FAR_PAST = "1999-01-01T00:00:00+00:00" + + +def _asset(asset_id, title="A Movie", year=2020, season=None, + media_type="movie_poster", author="artist", url=None): + return { + "id": asset_id, + "title": title, + "year": year, + "season": season, + "media_type": media_type, + "author": author, + "url": url or f"https://theposterdb.com/api/assets/{asset_id}", + } + + +@pytest.fixture +def index(tmp_path): + return AssetIndex(str(tmp_path / "asset_index.db")) + + +@pytest.mark.unit +def test_schema_creation_idempotent(tmp_path): + path = str(tmp_path / "asset_index.db") + AssetIndex(path) + AssetIndex(path).record("user", [_asset(1)]) # second init keeps the data, no error + assert AssetIndex(path).known_ids("user") == {1} + + +@pytest.mark.unit +def test_record_upsert_and_dedupe(index): + assert index.record("user", [_asset(1, title="First")]) == 1 + assert index.record("user", [_asset(1, title="First")]) == 0 # duplicate id collapses + assert index.known_ids("user") == {1} + # Tombstone then re-record: mutable fields update and the tombstone clears + index.reconcile("user", seen_ids=set(), crawl_started_at=FAR_FUTURE) + index.record("user", [_asset(1, title="Renamed", url="https://x/9")]) + rows = index.assets_for_user("user") + assert len(rows) == 1 + assert rows[0]["title"] == "Renamed" + assert rows[0]["url"] == "https://x/9" + assert rows[0]["missing_since"] is None + + +@pytest.mark.unit +def test_known_ids_includes_tombstoned(index): + index.record("user", [_asset(1), _asset(2)]) + index.reconcile("user", seen_ids={1}, crawl_started_at=FAR_FUTURE) + assert index.known_ids("user") == {1, 2} # 2 tombstoned but still "known" + + +@pytest.mark.unit +def test_assets_for_user_filters_and_order(index): + index.record("user", [ + _asset(1, media_type="movie_poster"), + _asset(3, media_type="movie_poster"), + _asset(2, media_type="unknown"), + ]) + index.record("user", [_asset(9, media_type="movie_poster")]) + index.reconcile("user", seen_ids={1, 3, 2}, crawl_started_at=FAR_FUTURE) + ids = [r["asset_id"] for r in index.assets_for_user("user")] + assert ids == [3, 1] # 9 tombstoned, 2 unknown-type; remainder newest-first + + +@pytest.mark.unit +def test_reconcile_tombstones_and_spares(index): + index.record("user", [_asset(1), _asset(2), _asset(3)]) + assert index.reconcile("user", seen_ids={1}, crawl_started_at=FAR_FUTURE) == 2 + assert {r["asset_id"] for r in index.assets_for_user("user")} == {1} + # A row inserted after the crawl started is spared even when unseen + index.record("user", [_asset(4)]) + assert index.reconcile("user", seen_ids=set(), crawl_started_at=FAR_PAST) == 0 + assert 4 in {r["asset_id"] for r in index.assets_for_user("user")} + + +@pytest.mark.unit +def test_crawl_state_roundtrip(index): + assert index.crawl_state("user") is None + index.record_crawl("user", full=True, seen_count=42) + state = index.crawl_state("user") + assert state["last_seen_count"] == 42 + full_ts = state["last_full_crawl"] + assert full_ts is not None + index.record_crawl("user", full=False, seen_count=50) # incremental keeps last_full_crawl + state = index.crawl_state("user") + assert state["last_seen_count"] == 50 + assert state["last_full_crawl"] == full_ts + + +@pytest.mark.unit +def test_page_is_fully_known(): + assert page_is_fully_known(set(), {1, 2}) is False # empty page never stops + assert page_is_fully_known({1, 9}, {1, 2}) is False # mixed page keeps crawling + assert page_is_fully_known({1, 2}, {1, 2, 3}) is True # fully known -> stop + + +@pytest.mark.unit +def test_full_crawl_due(): + now = datetime.now(timezone.utc) + assert full_crawl_due(None, 7) is True + assert full_crawl_due(now.isoformat(), 7) is False + stale = (now - timedelta(days=8)).isoformat() + assert full_crawl_due(stale, 7) is True + assert full_crawl_due(stale, 30) is False # 8 days old, 30-day window -> not due yet + assert full_crawl_due("not-a-date", 7) is True + + +@pytest.mark.unit +def test_normalize_title(): + assert normalize_title("Mission: Impossible") == normalize_title("Mission - Impossible") + assert normalize_title("Léon") == "leon" + assert normalize_title("Tom & Jerry") == "tom and jerry" + assert normalize_title(" Mad Max 2! ") == "mad max 2" + + +@pytest.mark.unit +def test_corrupt_file_self_heal(tmp_path): + path = str(tmp_path / "asset_index.db") + with open(path, "wb") as f: + f.write(b"this is not a sqlite database, just garbage bytes") + index = AssetIndex(path) # must self-heal, not raise + assert glob.glob(path + ".corrupt-*") + index.record("user", [_asset(1)]) + assert index.known_ids("user") == {1} diff --git a/utils/utils.py b/utils/utils.py index 81407bd..a4a266d 100644 --- a/utils/utils.py +++ b/utils/utils.py @@ -215,7 +215,7 @@ def parse_url_and_options(line): year == None options.year = year # If it's one of these flags it shouldn't have any arguments - elif flag in ["add-posters", "add-sets", "add-to-bulk", "force", "kometa", "stage", "temp", "skip-locked"]: + elif flag in ["add-posters", "add-sets", "add-to-bulk", "force", "kometa", "stage", "temp", "skip-locked", "no-cache"]: inv_flags.append(f"--{flag} has too many argumens") # If we ge to this point it's not a valid flag else: @@ -224,7 +224,7 @@ def parse_url_and_options(line): else: if flag in ["filters", "exclude", "year"]: inv_flags.append(f"--{flag} needs at least one argument") - elif flag not in ["add-posters", "add-sets" ,"add-to-bulk" ,"force", "kometa", "stage", "temp", "skip-locked"]: + elif flag not in ["add-posters", "add-sets" ,"add-to-bulk" ,"force", "kometa", "stage", "temp", "skip-locked", "no-cache"]: inv_flags.append(f"--{flag} is not a valid flag") else: setattr(options, flag.replace("-","_"), True) # Convert add-posters into add_posters as the Options class expects From a3bfcca7bb8c197c9973dabbb11ca554c4155cf8 Mon Sep 17 00:00:00 2001 From: Thoroslives Date: Wed, 15 Jul 2026 09:12:23 +1000 Subject: [PATCH 2/5] End a cached crawl at the last upload and report new cache entries When ThePosterDB's upload counter overshoots the pages it lists, the crawl used to run on into empty pages; the parse error on those marked the crawl unclean, which blocked both the crawl-state record and the tombstone pass, so an affected user did a full crawl every run and never reconciled. Stop the crawl at the first page that fetches cleanly but lists nothing, and don't treat that empty page as a failure. Guard the tombstone pass so a full crawl that was cut short (a markup change parsing to zero posters) never deletes what it didn't reach: only reconcile when the crawl covered most of the user's reported uploads, and otherwise leave the full-crawl timestamp unset so the next run retries. Also surface how many new assets a run added to the cache. --- artwork_uploader.py | 11 ++- core/constants.py | 4 + models/callbacks.py | 7 +- scrapers/theposterdb_scraper.py | 29 +++++- tests/test_user_cache_crawl.py | 151 ++++++++++++++++++++++++++++++++ 5 files changed, 194 insertions(+), 8 deletions(-) create mode 100644 tests/test_user_cache_crawl.py diff --git a/artwork_uploader.py b/artwork_uploader.py index d18e3db..148d2dc 100644 --- a/artwork_uploader.py +++ b/artwork_uploader.py @@ -248,6 +248,7 @@ def process_bulk_import_from_ui(instance: Instance, parsed_urls: list, filename: # Track successful poster uploads (those with ✅ or ♻️) success_counter = [0] assets_processed = [0] + cached_counter = [0] errors = 0 try: @@ -275,7 +276,8 @@ def process_bulk_import_from_ui(instance: Instance, parsed_urls: list, filename: notify_web(instance, "element_disable", {"element": ["bulk_button"], "mode": True}) try: - scrape_and_upload(instance, parsed_line.url, parsed_line.options, True, success_counter, assets_processed) + scrape_and_upload(instance, parsed_line.url, parsed_line.options, True, success_counter, assets_processed, cached_counter=cached_counter) + #time.sleep(1) except ScraperException as e: update_log(instance, f"❌ Error processing line: '{parsed_line.url}'") debug_me(f"ScraperException: Failed to scrape URL: {parsed_line.url} | {str(e)}") @@ -295,6 +297,7 @@ def process_bulk_import_from_ui(instance: Instance, parsed_urls: list, filename: + ("Scheduled b" if scheduled else "B") + f"ulk import of '{display_filename}' stopped by user • " + f"{assets_processed[0]} asset(s) processed • " + + (f"{cached_counter[0]} new in cache • " if cached_counter[0] else "") + f"{success_counter[0]} asset(s) updated" ) update_status(instance, message[2:], color=StatusColor.WARNING.value, sticky=False, spinner=False) @@ -306,6 +309,7 @@ def process_bulk_import_from_ui(instance: Instance, parsed_urls: list, filename: + f"ulk import of '{display_filename}' completed " + (f"successfully in {elapsed} • " if errors == 0 else f"with {errors} error(s) in {elapsed}, check logs for details • ") + f"{assets_processed[0]} asset(s) processed • " + + (f"{cached_counter[0]} new in cache • " if cached_counter[0] else "") + f"{success_counter[0]} asset(s) updated" ) update_status(instance, message[2:], color=StatusColor.SUCCESS.value if errors == 0 else StatusColor.WARNING.value, sticky=False, spinner=False) @@ -327,7 +331,7 @@ def process_bulk_import_from_ui(instance: Instance, parsed_urls: list, filename: notify_web(instance, "add_spinner", { "element": "bulk_button", "mode": False }) # Scraped the URL then uploads what it's scraped to Plex or download to Kometa asset directory -def scrape_and_upload(instance: Instance, url, options, bulk=False, success_counter=None, assets_processed=None): +def scrape_and_upload(instance: Instance, url, options, bulk=False, success_counter=None, assets_processed=None, cached_counter=None): """ Scrape artwork from a URL and upload to Plex. @@ -354,7 +358,8 @@ def progress_callback(current: int, total: int, title: str, bar_type:str = "main on_debug=debug_callback, on_progress_update=progress_callback, success_counter=success_counter, - assets_processed=assets_processed + assets_processed=assets_processed, + cached_counter=cached_counter ) # Use the service to do the actual work diff --git a/core/constants.py b/core/constants.py index 6c38996..35880ca 100644 --- a/core/constants.py +++ b/core/constants.py @@ -102,6 +102,10 @@ TPDB_API_ASSETS_URL = "https://theposterdb.com/api/assets" TPDB_RATE_LIMIT_DELAY = 6 # seconds between requests TPDB_USER_UPLOADS_PER_PAGE = 24 +# A full crawl only tombstones assets it didn't see if it reached at least this share of the +# user's reported uploads. ThePosterDB's counter runs a little high, so this is below 1.0, but a +# crawl cut short by a bad page must never be mistaken for a catalogue that shrank. +RECONCILE_MIN_COVERAGE = 0.9 # MediUX configuration MEDIUX_BASE_URL = "https://mediux.pro" diff --git a/models/callbacks.py b/models/callbacks.py index 2b5a104..3096f89 100644 --- a/models/callbacks.py +++ b/models/callbacks.py @@ -15,6 +15,7 @@ class ProcessingCallbacks: on_debug: Optional[Callable[[str, Optional[str]], None]] = None # (message, context) - for debug messages success_counter: Optional[list] = None # Mutable list to track successful uploads (contains count as single element) assets_processed: Optional[list] = None # Mutable list to track total assets processed (contains count as single element) + cached_counter: Optional[list] = None # Mutable list to track assets newly added to the user cache (contains count as single element) def status(self, message: str, color: str = "info", spinner: bool = False, sticky: bool = False): if self.on_status_update: @@ -38,4 +39,8 @@ def success(self, count: int): def assets(self, count: int): if self.assets_processed: - self.assets_processed[0] += count \ No newline at end of file + self.assets_processed[0] += count + + def cached(self, count: int): + if self.cached_counter: + self.cached_counter[0] += count \ No newline at end of file diff --git a/scrapers/theposterdb_scraper.py b/scrapers/theposterdb_scraper.py index ae7587f..a6f3247 100644 --- a/scrapers/theposterdb_scraper.py +++ b/scrapers/theposterdb_scraper.py @@ -10,7 +10,7 @@ from core.config import Config from core import globals from core.enums import MediaType, ScraperSource -from core.constants import TPDB_API_ASSETS_URL, TPDB_USER_UPLOADS_PER_PAGE, BOOTSTRAP_COLORS, ANSI_RESET, ANSI_BOLD +from core.constants import TPDB_API_ASSETS_URL, TPDB_USER_UPLOADS_PER_PAGE, RECONCILE_MIN_COVERAGE, BOOTSTRAP_COLORS, ANSI_RESET, ANSI_BOLD from models.artwork_types import MovieArtworkList, TVArtworkList, CollectionArtworkList import sqlite3 @@ -454,11 +454,26 @@ def _scrape_user_cached(self) -> None: new_rows, seen_ids, clean = self._crawl_user_pages(index, user_key, full=True) # Only reconcile/record over a crawl with no failed pages, so an invisible page error - # never mass-tombstones live assets. + # never mass-tombstones live assets. A full crawl now stops at the first empty page, which + # is normally the end of the uploads - but a TPDB markup change could make a mid-catalogue + # page parse to zero and cut the crawl short, and reconciling then would tombstone + # everything it didn't reach. So only count it as a full crawl (reconcile + advance the + # full-crawl timestamp) when it actually covered most of the user's reported uploads; + # otherwise leave the index alone so the next run retries the full crawl. if clean: - if full: + covered = len(seen_ids) >= self.user_uploads * RECONCILE_MIN_COVERAGE + if full and covered: index.reconcile(user_key, seen_ids, crawl_started_at) - index.record_crawl(user_key, full, self.user_uploads) + elif full: + self.callbacks.debug( + f"Full crawl for {self.author} saw only {len(seen_ids)} of {self.user_uploads} " + f"reported uploads; not tombstoning and retrying a full crawl next run", + "ThePosterDBScraper/scrape") + index.record_crawl(user_key, full and covered, self.user_uploads) + + if new_rows: + self.callbacks.cached(new_rows) + self.callbacks.log(f"🆕 TPDb user • {self.author} | {new_rows} new asset(s) added to the cache") self._hydrate_from_cache(index, user_key) @@ -483,6 +498,12 @@ def _crawl_user_pages(self, index: AssetIndex, user_key: str, full: bool): else: clean = False self.callbacks.debug(f"Processed {user_page + 1} out of {self.user_pages} user pages. Collected {collected} assets so far", "ThePosterDBScraper/scrape") + # The uploads counter can be higher than the number of assets actually listed, so the + # page count can overshoot. A page that fetched cleanly but held nothing is the end of + # the user's uploads - stop here rather than fetching the phantom pages after it. + if ok and not page_catalog: + self.callbacks.debug(f"No assets on page {user_page + 1}, the user's uploads end here", "ThePosterDBScraper/scrape") + break if not full and ok and page_is_fully_known(page_ids, known): self.callbacks.debug(f"Reached already-indexed uploads at page {user_page + 1}, stopping incremental crawl", "ThePosterDBScraper/scrape") break diff --git a/tests/test_user_cache_crawl.py b/tests/test_user_cache_crawl.py new file mode 100644 index 0000000..ea50ebb --- /dev/null +++ b/tests/test_user_cache_crawl.py @@ -0,0 +1,151 @@ +"""Tests for the cached user-crawl: stopping at the end of the uploads, not tombstoning a +catalogue from a crawl that was cut short, and reporting new assets added to the cache.""" + +import os + +import pytest + +from models.callbacks import ProcessingCallbacks +from models.options import Options +from services.asset_index import AssetIndex +import scrapers.theposterdb_scraper as tpdb +from scrapers.theposterdb_scraper import ThePosterDBScraper + + +@pytest.fixture(autouse=True) +def _isolate_cwd(tmp_path, monkeypatch): + os.makedirs(tmp_path / "config", exist_ok=True) + monkeypatch.chdir(tmp_path) + + +def _asset(asset_id): + return { + "id": asset_id, + "title": f"Film {asset_id}", + "year": 2020, + "season": None, + "media_type": "movie_poster", + "author": "someone", + "url": f"https://theposterdb.com/api/assets/{asset_id}", + } + + +def _scraper(counters=None): + scraper = ThePosterDBScraper("https://theposterdb.com/user/someone", + ProcessingCallbacks(**(counters or {}))) + scraper.set_options(Options()) + scraper.author = "someone" + return scraper + + +def _fill(pages): + """Return a fake scrape_user_page that fills the catalog from `pages` (dict page->id list).""" + fetched = [] + + def fake(page, catalog=None): + fetched.append(page) + for asset_id in pages.get(page, []): + catalog.append(_asset(asset_id)) + return True + + fake.fetched = fetched + return fake + + +# --- A2: stop at the end of the uploads, don't poison a clean crawl ---------------------------- + +def test_crawl_stops_at_empty_page_without_marking_it_unclean(tmp_path, monkeypatch): + scraper = _scraper() + scraper.user_uploads = 120 + scraper.user_pages = 5 # counter overshoots: only 2 pages hold assets + index = AssetIndex(str(tmp_path / "idx.db")) + fake = _fill({0: range(1000, 1024), 1: range(2000, 2024)}) + monkeypatch.setattr(scraper, "scrape_user_page", fake) + + new_rows, seen_ids, clean = scraper._crawl_user_pages(index, "someone", full=True) + + assert fake.fetched == [0, 1, 2] # stopped AT the first empty page + assert clean is True # a trailing empty page is not a failure + assert len(seen_ids) == 48 + + +def test_a_failed_page_still_marks_the_crawl_unclean(tmp_path, monkeypatch): + scraper = _scraper() + scraper.user_uploads = 72 + scraper.user_pages = 3 + index = AssetIndex(str(tmp_path / "idx.db")) + + def fake(page, catalog=None): + if page == 1: + return False # a genuine fetch failure, not an empty page + for asset_id in range(page * 1000, page * 1000 + 24): + catalog.append(_asset(asset_id)) + return True + + monkeypatch.setattr(scraper, "scrape_user_page", fake) + new_rows, seen_ids, clean = scraper._crawl_user_pages(index, "someone", full=True) + + assert clean is False # a real failure must still block reconcile + + +# --- reconcile guard: never tombstone from a crawl that was cut short -------------------------- + +def _seed(tmp_path, n): + index = AssetIndex(str(tmp_path / "idx.db")) + index.record("someone", [_asset(i) for i in range(n)]) + index.record_crawl("someone", True, n) + return index + + +def _run_full_crawl(scraper, tmp_path, monkeypatch, page_zero_ids): + scraper.options.no_cache = True # force a full crawl + monkeypatch.setattr(tpdb, "AssetIndex", lambda: AssetIndex(str(tmp_path / "idx.db"))) + monkeypatch.setattr(scraper, "_hydrate_from_cache", lambda *a, **k: None) + fake = _fill({0: page_zero_ids}) # page 1 is empty -> crawl stops after it + monkeypatch.setattr(scraper, "scrape_user_page", fake) + scraper._scrape_user_cached() + + +def test_short_full_crawl_does_not_tombstone(tmp_path, monkeypatch): + # Index has 100 assets. A full crawl that reaches only 10 of the reported 100 (a broken page, + # not a real shrink) must NOT tombstone the other 90. + index = _seed(tmp_path, 100) + scraper = _scraper() + scraper.user_uploads = 100 + scraper.user_pages = 50 + + _run_full_crawl(scraper, tmp_path, monkeypatch, page_zero_ids=range(0, 10)) + + live = index.assets_for_user("someone") + assert len(live) == 100 # coverage 10% < 90% -> nothing tombstoned + + +def test_full_crawl_with_good_coverage_tombstones_deleted(tmp_path, monkeypatch): + # Index has 100. The user really deleted 2, so the crawl sees 98 of a reported 98. Coverage is + # high, so the 2 that vanished are tombstoned as normal. + index = _seed(tmp_path, 100) + scraper = _scraper() + scraper.user_uploads = 98 + scraper.user_pages = 50 + + _run_full_crawl(scraper, tmp_path, monkeypatch, page_zero_ids=range(0, 98)) + + live = index.assets_for_user("someone") + assert len(live) == 98 # coverage 100% >= 90% -> 98,99 tombstoned + + +# --- the "new in cache" counter ---------------------------------------------------------------- + +def test_new_assets_are_reported_through_the_cached_counter(tmp_path, monkeypatch): + cached = [0] + scraper = _scraper({"cached_counter": cached}) + scraper.user_uploads = 48 + scraper.user_pages = 5 + monkeypatch.setattr(tpdb, "AssetIndex", lambda: AssetIndex(str(tmp_path / "idx.db"))) + monkeypatch.setattr(scraper, "_hydrate_from_cache", lambda *a, **k: None) + fake = _fill({0: range(1000, 1024), 1: range(2000, 2024)}) + monkeypatch.setattr(scraper, "scrape_user_page", fake) + + scraper._scrape_user_cached() + + assert cached[0] == 48 # all 48 were new to the cache From 1780b7f9899e9588431e5cdf20e832bca357a7fa Mon Sep 17 00:00:00 2001 From: Thoroslives Date: Fri, 3 Jul 2026 11:52:56 +1000 Subject: [PATCH 3/5] Add a Sonarr/Radarr webhook that applies cached artwork on import --- README.md | 19 +++ artwork_uploader.py | 2 + core/config.py | 24 +++- core/constants.py | 4 + core/globals.py | 3 +- services/__init__.py | 4 +- services/asset_index.py | 50 ++++++++ services/webhook_service.py | 220 ++++++++++++++++++++++++++++++++++ static/web_interface.js | 33 +++++ templates/web_interface.html | 29 +++++ tests/test_webhook_service.py | 128 ++++++++++++++++++++ web_routes.py | 42 +++++++ 12 files changed, 554 insertions(+), 4 deletions(-) create mode 100644 services/webhook_service.py create mode 100644 tests/test_webhook_service.py diff --git a/README.md b/README.md index 451383c..bbbe029 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,13 @@ Plus you can allow your bulk file to be auto-managed (cleaned and sorted for you ### Web UI Oh, last but not least, there's now a shiny new web UI so you can leave it running on your Plex Server and access it remotely! +### Automatic artwork for new imports (Sonarr/Radarr webhook) +With ```cache_user_scrapes``` enabled, the app already knows every poster your favourite ThePosterDB users have uploaded, so it can apply the right artwork within about a minute of Sonarr or Radarr importing something, instead of waiting for the next scheduled run. Turn on ```enable_webhooks```, set a ```webhook_token``` and list the ThePosterDB users to apply from (in order of preference) under Webhook settings in the web UI, then add a webhook connection in each app: + +- **Radarr / Sonarr:** Settings -> Connect -> + -> Webhook. URL ```http://:4567/webhook/radarr``` (or ```/webhook/sonarr```), Method POST. Tick only the "On File Import" trigger. Send the token as the connection's Password, or as a header: click the **Advanced** (cog) button and add a Header with Key ```X-Webhook-Token``` and Value set to the token. The Test button is acknowledged so you can save the connection. + +On an import the title is looked up in the cached index; if one of the configured users covers it, that single poster (plus season covers for the imported seasons, on TV) is applied through the same processing path as a normal scrape, so artwork labels, locked-artwork skips and Kometa asset mode all behave the same. Imports usually reach the webhook before Plex has scanned the new file, so the apply retries quietly for a few minutes, then gives up and leaves it to the next scheduled run. Ambiguous title matches (for example same-name remakes) are skipped rather than guessed, and nothing is applied when no configured user has the title. The endpoints return 404 while ```enable_webhooks``` is off. + ### Scheduler with Apprise notifications Basic scheduler, so that you can leave this running and update all your artwork every day. @@ -263,6 +270,18 @@ This is optional - if you don't do this, a new config.json will be created when ```"apprise_urls"``` - Provide a comma-separated list of Apprise service URLs to send notifications upong completion of scheduled bulk imports. Check [the Apprise service list](https://appriseit.com/services/) for details on the supported services and how to set them up and generate a notification URL for your favorite services. +```"enable_webhooks"``` +- Set to ```true``` to enable the Sonarr/Radarr import webhook endpoints (```/webhook/radarr``` and ```/webhook/sonarr```). ```false``` (default) leaves them disabled and returning 404, so the app is unchanged unless you turn this on. Requires ```cache_user_scrapes``` so there is an index to look artwork up in. + +```"webhook_token"``` +- The shared secret that webhook requests must provide, in an ```X-Webhook-Token``` header, as the HTTP Basic password, or a ```?token=``` query parameter. Required when ```enable_webhooks``` is ```true```. + +```"webhook_tpdb_users"``` +- A list of ThePosterDB user names to apply cached artwork from on import, in order of preference (first match wins). Only used when ```enable_webhooks``` is ```true```. + +```"webhook_apply_delay"``` +- Seconds to wait after an import before applying artwork (default ```30```). The *arr apps fire the webhook the moment they import a file, usually before Plex has scanned it, so this gives Plex a head start; if the item still is not in Plex the apply retries for a few minutes before giving up. + ### Filter options Both mediux_filters and tpdb_filters specify which artwork types to upload by including the flags below. Specify one or more in an array ["show_cover, "title_card"]. TPDb does not provide title cards, backgrounds or square art so these filters are not available in the web UI. - show_cover diff --git a/artwork_uploader.py b/artwork_uploader.py index 148d2dc..943240c 100644 --- a/artwork_uploader.py +++ b/artwork_uploader.py @@ -30,6 +30,7 @@ BulkFileService, ImageService, SchedulerService, + WebhookService, UtilityService ) from services.artwork_processor import ArtworkProcessor @@ -693,6 +694,7 @@ def update_scheduled_jobs(): # Create services globals.bulk_file_service = BulkFileService(get_exe_dir()) globals.scheduler_service = SchedulerService(check_interval=SCHEDULER_CHECK_INTERVAL) + globals.webhook_service = WebhookService() globals.update_service = UpdateService( github_repo=GITHUB_REPO, current_version=current_version, diff --git a/core/config.py b/core/config.py index ec0e669..93e3c3e 100644 --- a/core/config.py +++ b/core/config.py @@ -40,6 +40,10 @@ class Config: auth_username: Username for web server authentication auth_password_hash: Hashed password for web server authentication apprise_urls: List of Apprise notification URLs + enable_webhooks: Whether the Sonarr/Radarr import webhook endpoint is enabled + webhook_token: Shared secret required on webhook requests + webhook_tpdb_users: ThePosterDB users to apply cached artwork from on import, in priority order + webhook_apply_delay: Seconds to wait after an import before applying artwork (lets Plex scan first) """ def __init__(self, config_path: str = "config/config.json") -> None: @@ -66,6 +70,10 @@ def __init__(self, config_path: str = "config/config.json") -> None: self.auth_username: str = "" self.auth_password_hash: str = "" self.apprise_urls: List[str] = [] + self.enable_webhooks: bool = False + self.webhook_token: str = "" + self.webhook_tpdb_users: List[str] = [] + self.webhook_apply_delay: int = 30 def load(self) -> None: @@ -106,6 +114,10 @@ def load(self) -> None: self.auth_username = config.get("auth_username", "") self.auth_password_hash = config.get("auth_password_hash", "") self.apprise_urls = config.get("apprise_urls", []) + self.enable_webhooks = config.get("enable_webhooks", False) + self.webhook_token = config.get("webhook_token", "") + self.webhook_tpdb_users = config.get("webhook_tpdb_users", []) + self.webhook_apply_delay = config.get("webhook_apply_delay", 30) except Exception as e: raise ConfigLoadError(f"Error loading configuration from '{self.path}': {e}") from e @@ -131,7 +143,11 @@ def create(self) -> None: "auto_manage_bulk_files": True, "reset_overlay": True, "schedules": [], - "apprise_urls": [] + "apprise_urls": [], + "enable_webhooks": False, + "webhook_token": "", + "webhook_tpdb_users": [], + "webhook_apply_delay": 30 } if globals.docker: @@ -177,7 +193,11 @@ def save(self) -> None: "auth_enabled": self.auth_enabled, "auth_username": self.auth_username, "auth_password_hash": self.auth_password_hash, - "apprise_urls": self.apprise_urls + "apprise_urls": self.apprise_urls, + "enable_webhooks": self.enable_webhooks, + "webhook_token": self.webhook_token, + "webhook_tpdb_users": self.webhook_tpdb_users, + "webhook_apply_delay": self.webhook_apply_delay } try: diff --git a/core/constants.py b/core/constants.py index 35880ca..2d20c65 100644 --- a/core/constants.py +++ b/core/constants.py @@ -107,6 +107,10 @@ # crawl cut short by a bad page must never be mistaken for a catalogue that shrank. RECONCILE_MIN_COVERAGE = 0.9 +# Webhook (Sonarr/Radarr on-import trigger) +WEBHOOK_TOKEN_HEADER = "X-Webhook-Token" +WEBHOOK_RETRY_DELAYS = [20, 60, 180, 600] # seconds; retries while the imported item is not yet in Plex + # MediUX configuration MEDIUX_BASE_URL = "https://mediux.pro" MEDIUX_API_BASE_URL = "https://api.mediux.pro/assets/" diff --git a/core/globals.py b/core/globals.py index 9759521..051d6c8 100644 --- a/core/globals.py +++ b/core/globals.py @@ -10,7 +10,8 @@ bulk_file_service = None scheduler_service = None update_service = None +webhook_service = None # Scrape cancellation (user-initiated "Stop" from the web UI) cancel_scrape: bool = False # Set when the user asks to stop; long loops check it and stop cleanly -scrapes_running: int = 0 # How many scrapes are in flight; the flag clears when the last one ends \ No newline at end of file +scrapes_running: int = 0 # How many scrapes are in flight; the flag clears when the last one ends diff --git a/services/__init__.py b/services/__init__.py index bf58f04..40e4a7c 100644 --- a/services/__init__.py +++ b/services/__init__.py @@ -14,6 +14,7 @@ from .authentication_service import AuthenticationService from .notify_service import NotifyService from .asset_index import AssetIndex +from .webhook_service import WebhookService __all__ = [ 'BulkFileService', @@ -25,5 +26,6 @@ 'UtilityService', 'AuthenticationService', 'NotifyService', - 'AssetIndex' + 'AssetIndex', + 'WebhookService' ] diff --git a/services/asset_index.py b/services/asset_index.py index 0fbe2e1..20936ac 100644 --- a/services/asset_index.py +++ b/services/asset_index.py @@ -245,6 +245,41 @@ def record_crawl(self, user_key: str, full: bool, seen_count: int) -> None: finally: conn.close() + def lookup(self, user_keys: List[str], title: str, year: Optional[int], + media_types: List[str], season: Optional[int] = None) -> Optional[sqlite3.Row]: + """Find the best cached asset for an imported title, or None. + + user_keys is the preferred-first list of ThePosterDB users; the first user with an + acceptable match wins, and within a user the newest (highest asset_id) is taken. Year + is matched exactly then within one year; a missing year only matches when the + candidates share a single year, so same-name remakes are skipped rather than guessed.""" + title_keys = {normalize_title(title)} + stripped = re.sub(r"\s*\([^)]*\)\s*$", "", title).strip() + if stripped and stripped != title: + title_keys.add(normalize_title(stripped)) + title_keys = [key for key in title_keys if key] + if not title_keys or not user_keys or not media_types: + return None + conn = self._connect() + try: + sql = ( + "SELECT * FROM user_assets WHERE missing_since IS NULL " + f"AND media_type IN ({','.join('?' * len(media_types))}) " + f"AND title_key IN ({','.join('?' * len(title_keys))})" + ) + params = list(media_types) + title_keys + if season is not None: + sql += " AND season = ?" + params.append(season) + rows = conn.execute(sql, params).fetchall() + finally: + conn.close() + for user_key in user_keys: + chosen = _best_by_year([row for row in rows if row["user_key"] == user_key], year) + if chosen is not None: + return chosen + return None + def page_is_fully_known(page_ids: Set[int], known_ids: Set[int]) -> bool: """Incremental stop rule: a page halts the crawl only if it had assets and every one was @@ -264,3 +299,18 @@ def full_crawl_due(last_full_crawl: Optional[str], refresh_days: int) -> bool: except (TypeError, ValueError): return True return (datetime.now(timezone.utc) - last).total_seconds() >= refresh_days * 86400 + + +def _best_by_year(candidates: List[sqlite3.Row], year: Optional[int]) -> Optional[sqlite3.Row]: + """Pick the newest candidate whose year matches (exact, then within one year). With no year + to match on, only pick when every candidate shares one year - otherwise it is ambiguous + (a same-name remake) and nothing is returned.""" + if not candidates: + return None + if year is not None: + pool = [row for row in candidates if row["year"] == year] or \ + [row for row in candidates if row["year"] is not None and abs(row["year"] - year) <= 1] + return max(pool, key=lambda row: row["asset_id"]) if pool else None + if len({row["year"] for row in candidates}) > 1: + return None + return max(candidates, key=lambda row: row["asset_id"]) diff --git a/services/webhook_service.py b/services/webhook_service.py new file mode 100644 index 0000000..6d4ba21 --- /dev/null +++ b/services/webhook_service.py @@ -0,0 +1,220 @@ +""" +In-tool Sonarr/Radarr import webhook. + +On a Radarr/Sonarr "Download" (import) event, look the imported title up in the persistent +asset index and apply the cached poster through the normal upload path, retrying for a few +minutes while Plex finishes scanning the new file. Everything downstream of the dispatch (the +authoritative TMDb resolution, artwork-ID skips, locked-artwork skips, Kometa mode) is the same +code a normal scrape uses, so the webhook inherits all of it without re-implementing any of it. +""" + +import threading +import time +from dataclasses import dataclass +from typing import FrozenSet, List, Optional, Union + +from core import globals +from core.constants import WEBHOOK_RETRY_DELAYS +from core.enums import ScraperSource +from core.exceptions import MovieNotFound, ShowNotFound +from models.instance import Instance +from models.options import Options +from services.asset_index import AssetIndex, normalize_title + + +def _log(text: str) -> None: + # utils.notifications imports the services package, so import it lazily to avoid an import + # cycle when the services package pulls this module in at start-up. + from utils.notifications import update_log + update_log(Instance(broadcast=True), text) + + +def _debug(message: str) -> None: + from utils.notifications import debug_me + debug_me(message, "WebhookService") + + +@dataclass(frozen=True) +class WebhookEvent: + """A parsed Sonarr/Radarr import event.""" + kind: str # "movie" or "tv" + title: str + year: Optional[int] + tmdb_id: Optional[int] + tvdb_id: Optional[int] + seasons: FrozenSet[int] + source: str # "radarr" or "sonarr" + + def label(self) -> str: + return f"{self.title} ({self.year})" if self.year else self.title + + +def _int_or_none(value) -> Optional[int]: + """Map absent numeric fields to None. Radarr/Sonarr serialise absent ints and years as 0 + (C# defaults), so 0 is treated as absent. NOT used for season numbers, where 0 (Specials) + is a real value.""" + try: + number = int(value) + except (TypeError, ValueError): + return None + return number or None + + +def parse_event(payload: dict) -> Union[WebhookEvent, str, None]: + """Parse a Radarr/Sonarr webhook payload. + + Returns a WebhookEvent for an import ("Download") event, the string "test" for the + connection Test button, or None for any other event type or malformed payload. Never + raises, so an unexpected payload is acknowledged and ignored rather than erroring.""" + if not isinstance(payload, dict): + return None + event_type = payload.get("eventType") + if event_type == "Test": + return "test" + if event_type != "Download": + return None + movie = payload.get("movie") + if isinstance(movie, dict) and movie.get("title"): + return WebhookEvent( + kind="movie", title=movie["title"], year=_int_or_none(movie.get("year")), + tmdb_id=_int_or_none(movie.get("tmdbId")), tvdb_id=None, + seasons=frozenset(), source="radarr", + ) + series = payload.get("series") + if isinstance(series, dict) and series.get("title"): + episodes = payload.get("episodes") or [] + seasons = frozenset( + episode["seasonNumber"] for episode in episodes + if isinstance(episode, dict) and isinstance(episode.get("seasonNumber"), int) + ) + return WebhookEvent( + kind="tv", title=series["title"], year=_int_or_none(series.get("year")), + tmdb_id=_int_or_none(series.get("tmdbId")), tvdb_id=_int_or_none(series.get("tvdbId")), + seasons=seasons, source="sonarr", + ) + return None + + +class WebhookService: + """Applies cached artwork for imported titles off the request thread, with a retry ladder + for the window where an item has been imported but Plex has not scanned it in yet.""" + + def __init__(self) -> None: + self._inflight: set = set() + self._lock = threading.Lock() + + def enqueue(self, event: WebhookEvent) -> None: + """Queue an import event for application. Returns immediately; work happens on a thread.""" + key = self._dedupe_key(event) + with self._lock: + if key in self._inflight: + _debug(f"Import for {event.label()} is already pending, ignoring the duplicate") + return + self._inflight.add(key) + # Wait a configurable delay before the first attempt: the *arr apps fire the webhook the + # moment they import a file, usually before Plex has scanned it in, so give Plex a head + # start. If it is still not ready by then, the retry ladder takes over. + delay = max(0, globals.config.webhook_apply_delay or 0) + timer = threading.Timer(delay, self._attempt, args=(event, key)) + timer.daemon = True + timer.start() + + @staticmethod + def _dedupe_key(event: WebhookEvent): + identity = event.tmdb_id or event.tvdb_id or normalize_title(event.title) + return (event.kind, identity, event.seasons) + + def _release(self, key) -> None: + with self._lock: + self._inflight.discard(key) + + def _attempt(self, event: WebhookEvent, key, attempt: int = 0, + artwork: Optional[List[dict]] = None) -> None: + # UploadProcessor pulls in the processors -> plex chain; import it here so the services + # package does not drag that in at start-up (mirrors the app's own lazy-import pattern). + from processors.upload_processor import UploadProcessor + try: + if artwork is None: + artwork = self._collect_artwork(event) + if not artwork: + _log(f"📥 Webhook | No cached artwork for '{event.label()}' from the configured users") + self._release(key) + return + _log(f"📥 Webhook | {event.source.title()} import: {event.label()}") + globals.plex.connect() + processor = UploadProcessor(globals.plex) + processor.set_options(Options()) + pending = [] + for item in artwork: + try: + if event.kind == "movie": + results = processor.process_movie_artwork(item) + else: + results = processor.process_tv_artwork(item) + results = results or [] + # A season/episode whose show is in Plex but which Plex has not scanned in yet + # comes back as a "not available" result rather than an exception; treat that + # the same as not-found so it retries once Plex catches up. + if any("not available" in str(result).lower() for result in results): + pending.append(item) + else: + for result in results: + _log(result) + except (MovieNotFound, ShowNotFound): + pending.append(item) # not in Plex yet, retry it + except Exception as error: + _log(f"❌ Webhook | {event.label()}: {error}") + if pending and attempt < len(WEBHOOK_RETRY_DELAYS): + delay = WEBHOOK_RETRY_DELAYS[attempt] + _debug(f"{event.label()} not in Plex yet, retrying in {delay}s") + timer = threading.Timer(delay, self._attempt, args=(event, key, attempt + 1, pending)) + timer.daemon = True + timer.start() + return # keep the in-flight key until the chain ends + if pending: + _log(f"⚠️ Webhook | '{event.label()}' has not appeared in Plex, leaving it for the next scheduled run") + except Exception as error: + _debug(f"Webhook apply failed for {event.label()}: {error}") + self._release(key) + + def _collect_artwork(self, event: WebhookEvent) -> List[dict]: + """Build the artwork dicts to apply by looking the title up in the index for each + configured user, in preference order. Empty when nobody covers it.""" + user_keys = [user.strip().casefold() for user in (globals.config.webhook_tpdb_users or []) if user.strip()] + if not user_keys: + return [] + index = AssetIndex() + cache_buster = f"&_cb={int(time.time())}" + artwork: List[dict] = [] + if event.kind == "movie": + row = index.lookup(user_keys, event.title, event.year, ["movie_poster"]) + if row: + artwork.append(self._artwork_dict(row, cache_buster, "movie_poster")) + else: + row = index.lookup(user_keys, event.title, event.year, ["show_cover"]) + if row: + artwork.append(self._artwork_dict(row, cache_buster, "show_cover")) + for season in sorted(event.seasons): + season_row = index.lookup(user_keys, event.title, event.year, ["season_cover"], season=season) + if season_row: + artwork.append(self._artwork_dict(season_row, cache_buster, "season_cover", season)) + return artwork + + @staticmethod + def _artwork_dict(row, cache_buster: str, artwork_type: str, season=None) -> dict: + """Build a get_posters-shaped artwork dict from an index row. tmdb_id stays None so the + processor resolves identity authoritatively from the poster page at write time.""" + artwork = { + "title": row["title"], + "author": row["author"], + "tmdb_id": None, + "url": f"{row['url']}{cache_buster}", + "year": row["year"], + "source": ScraperSource.THEPOSTERDB.value, + "id": str(row["asset_id"]), + "type": artwork_type, + } + if artwork_type in ("show_cover", "season_cover"): + artwork["season"] = "Cover" if artwork_type == "show_cover" else season + artwork["episode"] = None + return artwork diff --git a/static/web_interface.js b/static/web_interface.js index 9d75186..c09b341 100644 --- a/static/web_interface.js +++ b/static/web_interface.js @@ -650,6 +650,12 @@ function saveConfig() { save_config.auth_username = document.getElementById("auth_username").value.trim(); save_config.auth_password = document.getElementById("auth_password").value; + // Webhook settings + save_config.enable_webhooks = document.getElementById("enable_webhooks").checked; + save_config.webhook_token = document.getElementById("webhook_token").value.trim(); + save_config.webhook_tpdb_users = document.getElementById("webhook_tpdb_users").value.split(",").map(u => u.trim()).filter(u => u); + save_config.webhook_apply_delay = parseInt(document.getElementById("webhook_apply_delay").value, 10) || 0; + // Process every possible condition of auth enable/disable and same/different username and password/no password provided // If auth is enabled and a password is NOT provided if (save_config.auth_enabled && !save_config.auth_password) { @@ -718,6 +724,12 @@ function loadConfig() { // Load authentication settings document.getElementById("auth_enabled").checked = data.config.auth_enabled || false; document.getElementById("auth_username").value = data.config.auth_username || ""; + + // Load webhook settings + document.getElementById("enable_webhooks").checked = data.config.enable_webhooks || false; + document.getElementById("webhook_token").value = data.config.webhook_token || ""; + document.getElementById("webhook_tpdb_users").value = (data.config.webhook_tpdb_users || []).join(", "); + document.getElementById("webhook_apply_delay").value = data.config.webhook_apply_delay ?? 30; // Toggle Kometa settings visibility toggleKometaSettings(); @@ -727,6 +739,9 @@ function loadConfig() { // Toggle auth settings visibility toggleAuthSettings(); + + // Toggle webhook settings visibility + toggleWebhookSettings(); // Make sure Plex options visibility is set correctly on load togglePlexOptions(); @@ -1782,6 +1797,24 @@ function toggleAuthSettings() { // Add event listener for auth_enabled checkbox document.getElementById("auth_enabled").addEventListener("change", toggleAuthSettings); +// ================================================== +// Webhook Settings Toggle +// ================================================== + +function toggleWebhookSettings() { + const enabled = document.getElementById("enable_webhooks").checked; + document.getElementById("webhook_settings").style.display = enabled ? "block" : "none"; + // When enabling with an empty token, pre-fill a random one. getRandomValues works in + // insecure (plain http) contexts, unlike crypto.randomUUID. + const tokenField = document.getElementById("webhook_token"); + if (enabled && !tokenField.value) { + tokenField.value = Array.from(crypto.getRandomValues(new Uint8Array(16))) + .map(b => b.toString(16).padStart(2, "0")).join(""); + } +} + +document.getElementById("enable_webhooks").addEventListener("change", toggleWebhookSettings); + // ================================================== // Kometa Settings Toggle // ================================================== diff --git a/templates/web_interface.html b/templates/web_interface.html index 04c49ae..6246e4a 100644 --- a/templates/web_interface.html +++ b/templates/web_interface.html @@ -276,6 +276,35 @@
 Authentication settings + +
+
 Webhook settings
+
+ + +
+ +
diff --git a/tests/test_webhook_service.py b/tests/test_webhook_service.py new file mode 100644 index 0000000..83bc4ec --- /dev/null +++ b/tests/test_webhook_service.py @@ -0,0 +1,128 @@ +"""Unit tests for the Sonarr/Radarr import webhook (services/webhook_service.py).""" + +import pytest + +from services.asset_index import AssetIndex +from services.webhook_service import WebhookEvent, WebhookService, parse_event + +FAR_FUTURE = "2999-01-01T00:00:00+00:00" + + +@pytest.fixture +def index(tmp_path): + return AssetIndex(str(tmp_path / "asset_index.db")) + + +def _rec(index, user, asset_id, title, year, media_type, season=None): + index.record(user, [{ + "id": asset_id, "title": title, "year": year, "season": season, + "media_type": media_type, "author": user, + "url": f"https://theposterdb.com/api/assets/{asset_id}", + }]) + + +# ------------------------------- parse_event ------------------------------- + +@pytest.mark.unit +def test_parse_radarr_download(): + event = parse_event({"eventType": "Download", + "movie": {"title": "Dune", "year": 2021, "tmdbId": 438631}}) + assert isinstance(event, WebhookEvent) + assert (event.kind, event.title, event.year, event.tmdb_id, event.source) == \ + ("movie", "Dune", 2021, 438631, "radarr") + assert event.seasons == frozenset() + + +@pytest.mark.unit +def test_parse_sonarr_download_multi_episode(): + event = parse_event({"eventType": "Download", + "series": {"title": "Severance", "year": 2022, "tvdbId": 371980, "tmdbId": 95396}, + "episodes": [{"seasonNumber": 1}, {"seasonNumber": 2}, {"seasonNumber": 0}]}) + assert event.kind == "tv" + assert event.tvdb_id == 371980 and event.tmdb_id == 95396 + assert event.seasons == frozenset({0, 1, 2}) # season 0 (Specials) is kept + + +@pytest.mark.unit +def test_parse_csharp_zero_defaults_are_none(): + event = parse_event({"eventType": "Download", + "movie": {"title": "Nope", "year": 0, "tmdbId": 0}}) + assert event.year is None and event.tmdb_id is None + + +@pytest.mark.unit +def test_parse_test_and_ignored_events(): + assert parse_event({"eventType": "Test"}) == "test" + for event_type in ("Grab", "MovieAdded", "Rename", "Health", "SeriesAdd"): + assert parse_event({"eventType": event_type, "movie": {"title": "X"}}) is None + assert parse_event({}) is None # missing eventType + assert parse_event({"eventType": "Download"}) is None # no movie/series + assert parse_event({"eventType": "Download", "movie": {}}) is None # no title + assert parse_event("not a dict") is None # never raises + + +# --------------------------------- lookup ---------------------------------- + +@pytest.mark.unit +def test_lookup_preferred_user_wins(index): + _rec(index, "filler", 999, "Heat", 1995, "movie_poster") # newer id, less preferred + _rec(index, "preferred", 1, "Heat", 1995, "movie_poster") + row = index.lookup(["preferred", "filler"], "Heat", 1995, ["movie_poster"]) + assert row["user_key"] == "preferred" + + +@pytest.mark.unit +def test_lookup_newest_within_user(index): + _rec(index, "user", 1, "Heat", 1995, "movie_poster") + _rec(index, "user", 5, "Heat", 1995, "movie_poster") + assert index.lookup(["user"], "Heat", 1995, ["movie_poster"])["asset_id"] == 5 + + +@pytest.mark.unit +def test_lookup_year_exact_then_nearby(index): + _rec(index, "user", 1, "Heat", 1995, "movie_poster") + assert index.lookup(["user"], "Heat", 1995, ["movie_poster"]) is not None + assert index.lookup(["user"], "Heat", 1996, ["movie_poster"]) is not None # +/-1 + assert index.lookup(["user"], "Heat", 2010, ["movie_poster"]) is None # too far + + +@pytest.mark.unit +def test_lookup_year_none_ambiguity(index): + _rec(index, "user", 1, "Solo Title", 2020, "movie_poster") + assert index.lookup(["user"], "Solo Title", None, ["movie_poster"]) is not None + _rec(index, "user", 2, "Solo Title", 1990, "movie_poster") # now two distinct years + assert index.lookup(["user"], "Solo Title", None, ["movie_poster"]) is None + + +@pytest.mark.unit +def test_lookup_season_and_show_cover(index): + _rec(index, "user", 10, "Severance", 2022, "show_cover") + _rec(index, "user", 11, "Severance", 2022, "season_cover", season=0) + _rec(index, "user", 12, "Severance", 2022, "season_cover", season=1) + assert index.lookup(["user"], "Severance", 2022, ["season_cover"], season=0)["asset_id"] == 11 + assert index.lookup(["user"], "Severance", 2022, ["season_cover"], season=1)["asset_id"] == 12 + assert index.lookup(["user"], "Severance", 2022, ["show_cover"])["asset_id"] == 10 + + +@pytest.mark.unit +def test_lookup_tombstoned_and_parenthetical(index): + _rec(index, "user", 1, "The Office", 2005, "show_cover") # index stores the parsed (stripped) title + row = index.lookup(["user"], "The Office (US)", 2005, ["show_cover"]) # query strips the suffix + assert row is not None and row["asset_id"] == 1 + index.reconcile("user", seen_ids=set(), crawl_started_at=FAR_FUTURE) # tombstone everything + assert index.lookup(["user"], "The Office (US)", 2005, ["show_cover"]) is None + + +# ------------------------------- dedupe key -------------------------------- + +@pytest.mark.unit +def test_dedupe_key(): + def series(seasons): + return WebhookEvent(kind="tv", title="Show", year=2020, tmdb_id=42, tvdb_id=7, + seasons=frozenset(seasons), source="sonarr") + assert WebhookService._dedupe_key(series([1])) == WebhookService._dedupe_key(series([1])) + assert WebhookService._dedupe_key(series([1])) != WebhookService._dedupe_key(series([2])) + # falls back to normalized title when there is no id + no_id = WebhookEvent(kind="movie", title="Léon", year=1994, tmdb_id=None, tvdb_id=None, + seasons=frozenset(), source="radarr") + assert WebhookService._dedupe_key(no_id)[1] == "leon" diff --git a/web_routes.py b/web_routes.py index cb66aac..3945010 100644 --- a/web_routes.py +++ b/web_routes.py @@ -16,6 +16,7 @@ import sys import re import base64 +import hmac import tempfile import zipfile import subprocess @@ -36,6 +37,8 @@ from processors.media_metadata import parse_title from utils.notifications import update_log, update_status, notify_web, debug_me from services import UtilityService, AuthenticationService +from services.webhook_service import parse_event +from core.constants import WEBHOOK_TOKEN_HEADER def login_required(f): @@ -122,6 +125,45 @@ def uploaded_file(filename): uploads_path = os.path.join(UtilityService.get_exe_dir(), 'uploads') return send_from_directory(uploads_path, filename) + def webhook_token_ok(): + """A webhook request is authorised when it carries the configured token, sent as the + X-Webhook-Token header, the HTTP Basic password, or a ?token= query parameter.""" + token = globals.config.webhook_token + if not token: + return False + provided = (request.headers.get(WEBHOOK_TOKEN_HEADER) + or (request.authorization.password if request.authorization else None) + or request.args.get("token") or "") + return hmac.compare_digest(provided, token) + + @web_app.route("/webhook/radarr", methods=["POST"]) + @web_app.route("/webhook/sonarr", methods=["POST"]) + def arr_webhook(): + """Receive a Sonarr/Radarr import event and apply the cached artwork for it. Disabled + by default (404), so the endpoint is indistinguishable from upstream when off.""" + if not globals.config.enable_webhooks: + return {"status": "not found"}, 404 + if not webhook_token_ok(): + update_log(Instance(broadcast=True), "⛔ Webhook request rejected (missing or invalid token)") + return {"status": "unauthorized"}, 401 + payload = request.get_json(silent=True) + if payload is None: + return {"status": "invalid json"}, 400 + event = parse_event(payload) + if event == "test": + update_log(Instance(broadcast=True), "🧪 Webhook test received") + return {"status": "test ok"}, 200 + if event is None: + return {"status": "ignored"}, 200 + if globals.plex is None or not (globals.plex.movie_libraries or globals.plex.tv_libraries): + update_log(Instance(broadcast=True), "⛔ Webhook received but Plex libraries are not configured") + return {"status": "plex not configured"}, 503 + if not globals.config.webhook_tpdb_users: + update_log(Instance(broadcast=True), "📥 Webhook received but no ThePosterDB users are configured (webhook_tpdb_users)") + return {"status": "no users configured"}, 200 + globals.webhook_service.enqueue(event) + return {"status": "queued"}, 200 + def setup_socket_handlers( config: Config, From 7e6fda785b67f0b1e6a69604c2a1dda5de804827 Mon Sep 17 00:00:00 2001 From: Thoroslives Date: Tue, 21 Jul 2026 15:58:08 +1000 Subject: [PATCH 4/5] Key cached-crawl artwork as file_type so the uploader can resolve its label _hydrate_from_cache built the artwork dicts that feed the upload processor with the key "type", but get_posters and the processor use "file_type". ARTWORK_ID_MAP.get(None) then returns None and PlexUploader.set_artwork does None + md5(url), raising "unsupported operand type(s) for +: NoneType and str". Latent until a cached crawl meets an unlocked in-library item. Regression test drives _hydrate_from_cache for real (the existing cache tests stub it out) and asserts every produced dict carries a file_type that ARTWORK_ID_MAP resolves. Co-Authored-By: Claude Opus 4.8 (1M context) --- scrapers/theposterdb_scraper.py | 6 +++--- tests/test_user_cache_crawl.py | 34 +++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/scrapers/theposterdb_scraper.py b/scrapers/theposterdb_scraper.py index a6f3247..37eec1c 100644 --- a/scrapers/theposterdb_scraper.py +++ b/scrapers/theposterdb_scraper.py @@ -539,7 +539,7 @@ def _hydrate_from_cache(self, index: AssetIndex, user_key: str) -> None: "year": year, "source": ScraperSource.THEPOSTERDB.value, "id": poster_id, - "type": file_type, + "file_type": file_type, }) else: self.exclusions += 1 @@ -565,7 +565,7 @@ def _hydrate_from_cache(self, index: AssetIndex, user_key: str) -> None: "year": year, "source": ScraperSource.THEPOSTERDB.value, "id": poster_id, - "type": "movie_poster", + "file_type": "movie_poster", }) else: self.exclusions += 1 @@ -583,7 +583,7 @@ def _hydrate_from_cache(self, index: AssetIndex, user_key: str) -> None: "url": poster_url, "source": ScraperSource.THEPOSTERDB.value, "id": poster_id, - "type": "collection_poster", + "file_type": "collection_poster", }) else: self.exclusions += 1 diff --git a/tests/test_user_cache_crawl.py b/tests/test_user_cache_crawl.py index ea50ebb..cb29ee6 100644 --- a/tests/test_user_cache_crawl.py +++ b/tests/test_user_cache_crawl.py @@ -149,3 +149,37 @@ def test_new_assets_are_reported_through_the_cached_counter(tmp_path, monkeypatc scraper._scrape_user_cached() assert cached[0] == 48 # all 48 were new to the cache + + +# --- regression: hydrate must key artwork as file_type, exactly as get_posters does ------------ + +def test_hydrate_from_cache_uses_the_processor_file_type_key(tmp_path): + """_hydrate_from_cache rebuilds the artwork lists that feed the upload processor, so its dicts + must carry 'file_type' (what get_posters writes and the processor reads), not 'type'. With + 'type', ARTWORK_ID_MAP.get(...) is None and the uploader crashes on None + md5 - the same + fault that broke the import webhook, latent here until a cached crawl meets an unlocked + in-library item.""" + from core.constants import ARTWORK_ID_MAP + + index = AssetIndex(str(tmp_path / "idx.db")) + index.record("someone", [ + {"id": 1, "title": "Dune", "year": 2021, "season": None, + "media_type": "movie_poster", "author": "someone", + "url": "https://theposterdb.com/api/assets/1"}, + {"id": 2, "title": "Severance", "year": 2022, "season": None, + "media_type": "show_cover", "author": "someone", + "url": "https://theposterdb.com/api/assets/2"}, + {"id": 3, "title": "Severance", "year": 2022, "season": 1, + "media_type": "season_cover", "author": "someone", + "url": "https://theposterdb.com/api/assets/3"}, + ]) + + scraper = _scraper() + scraper.config.tpdb_filters = ["movie_poster", "show_cover", "season_cover", "collection_poster"] + scraper._hydrate_from_cache(index, "someone") + + produced = scraper.movie_artwork + scraper.tv_artwork + assert produced, "hydrate produced no artwork - fixture/filter setup is wrong" + for artwork in produced: + assert "file_type" in artwork, f"hydrated artwork missing file_type: {artwork}" + assert ARTWORK_ID_MAP.get(artwork["file_type"]) is not None From 9820423c0be79625c7108c6d2e63e9d62424a996 Mon Sep 17 00:00:00 2001 From: Thoroslives Date: Tue, 21 Jul 2026 15:58:08 +1000 Subject: [PATCH 5/5] Key webhook artwork as file_type so imports don't crash on upload The import webhook's _artwork_dict keyed the artwork type as "type", but the upload processor reads "file_type" to resolve the Plex label prefix. It came out None and PlexUploader.set_artwork did None + md5(url), so every Radarr/ Sonarr import with cached artwork died with "unsupported operand type(s) for +: NoneType and str" (seen in production on a Disclosure Day (2026) import). Regression test builds the webhook artwork dict and runs it through the same label path the processor uses, asserting it no longer crashes. Co-Authored-By: Claude Opus 4.8 (1M context) --- services/webhook_service.py | 7 +++++-- tests/test_webhook_service.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/services/webhook_service.py b/services/webhook_service.py index 6d4ba21..328e9f4 100644 --- a/services/webhook_service.py +++ b/services/webhook_service.py @@ -203,7 +203,10 @@ def _collect_artwork(self, event: WebhookEvent) -> List[dict]: @staticmethod def _artwork_dict(row, cache_buster: str, artwork_type: str, season=None) -> dict: """Build a get_posters-shaped artwork dict from an index row. tmdb_id stays None so the - processor resolves identity authoritatively from the poster page at write time.""" + processor resolves identity authoritatively from the poster page at write time. The + artwork type MUST be keyed as 'file_type' - the key get_posters writes and the upload + processor reads to resolve the Plex label prefix; 'type' leaves it unresolved and the + uploader crashes on None + md5.""" artwork = { "title": row["title"], "author": row["author"], @@ -212,7 +215,7 @@ def _artwork_dict(row, cache_buster: str, artwork_type: str, season=None) -> dic "year": row["year"], "source": ScraperSource.THEPOSTERDB.value, "id": str(row["asset_id"]), - "type": artwork_type, + "file_type": artwork_type, } if artwork_type in ("show_cover", "season_cover"): artwork["season"] = "Cover" if artwork_type == "show_cover" else season diff --git a/tests/test_webhook_service.py b/tests/test_webhook_service.py index 83bc4ec..c54deaf 100644 --- a/tests/test_webhook_service.py +++ b/tests/test_webhook_service.py @@ -1,7 +1,11 @@ """Unit tests for the Sonarr/Radarr import webhook (services/webhook_service.py).""" +from unittest.mock import MagicMock + import pytest +from core.constants import ARTWORK_ID_MAP, ARTWORK_TYPE_MAP +from plex.plex_uploader import PlexUploader from services.asset_index import AssetIndex from services.webhook_service import WebhookEvent, WebhookService, parse_event @@ -126,3 +130,33 @@ def series(seasons): no_id = WebhookEvent(kind="movie", title="Léon", year=1994, tmdb_id=None, tvdb_id=None, seasons=frozenset(), source="radarr") assert WebhookService._dedupe_key(no_id)[1] == "leon" + + +# ------------------- regression: the file_type contract (the live crash) ------------------- + +@pytest.mark.unit +@pytest.mark.parametrize("artwork_type,season", [ + ("movie_poster", None), + ("show_cover", None), + ("season_cover", 2), +]) +def test_artwork_dict_carries_the_file_type_key_the_processor_reads(artwork_type, season): + """The webhook must key the artwork type as 'file_type' - the key get_posters writes and the + upload processor reads. Keyed as 'type' (the old bug), ARTWORK_ID_MAP.get(...) returns None + and PlexUploader.set_artwork does None + md5, the exact 'unsupported operand type(s) for +: + NoneType and str' that broke a real Radarr import in production.""" + row = {"title": "Dune", "author": "someuser", "year": 2021, "asset_id": 42, + "url": "https://theposterdb.com/api/assets/42"} + artwork = WebhookService._artwork_dict(row, "&_cb=123", artwork_type, season) + + assert artwork["file_type"] == artwork_type + assert ARTWORK_ID_MAP.get(artwork["file_type"]) is not None + + # Reproduce the runtime path: the processor derives artwork_id from file_type and hands it to + # the uploader, whose set_artwork builds label = artwork_id + md5(url). A None artwork_id here + # is the crash. + artwork_id = ARTWORK_ID_MAP.get(artwork.get("file_type")) + artwork_type_str = ARTWORK_TYPE_MAP.get(artwork.get("file_type")) + uploader = PlexUploader(MagicMock(), artwork_type_str, artwork_id) + uploader.set_artwork(artwork) + assert isinstance(uploader.label, str) and uploader.label