Match scraped artwork against the local libraries to speed up big scrapes

README.md

@@ -162,6 +162,7 @@ The basic version is available now on the bulk imports page, click on the clock
 It's there for when we have API access (and works for scrapers in the meantime) but is limited to running once a day which should be fine.
 
 If you enable ```skip_locked_artwork```, a scheduled user or bulk scrape will only fill items that are still on their default artwork and leave anything you've set by hand untouched, so it's safe to leave running against a curated library.
+With ```local_library_matching``` enabled (the default), a user-catalog scrape skips everything that isn't in your libraries without any web requests, so full-catalog runs take minutes rather than hours.
 
 Additionally, you can configure one or more push notification services so you get a notification every time a scheduled bulk import job completes. Notifications are provided by Apprise. Configure the list of notification services by providing a comma-separated list of Apprise notification URLs through the web UI or via the ```apprise_urls``` variable in the ```config.json```file. Check [the Apprise service list](https://appriseit.com/services/) for details on the supported services and how to set them up and generate a notification URL for your favorite services.
 
@@ -233,6 +234,9 @@ This is optional - if you don't do this, a new config.json will be created when
 - Setting this to ```true``` will skip any artwork whose target field (poster, background or square art) is locked in Plex, unless ```--force``` is used.  Plex locks a field whenever artwork is deliberately set - manually or by an upload - so this makes scheduled bulk imports and user scrapes fill items still on default artwork while leaving anything you've already set alone.
 - Setting to ```false``` (the default) keeps the existing behaviour where artwork is applied regardless of locks.
 
+```"local_library_matching"```
+- Setting this to ```true``` (the default) matches scraped artwork against your Plex libraries locally before fetching each poster's page, so big user scrapes skip everything you don't own without a web request and run dramatically faster.  The poster page is still checked right before anything is uploaded, so nothing less accurate ever gets written.
+- Setting to ```false``` restores the previous per-poster lookups.  Use this if items you own start logging "not available on Plex" because their Plex titles differ from the titles on ThePosterDB.
 ```"auto_manage_bulk_files"```
 - Setting this to ```true``` will automatically add, label and sort URLs from the scrape tab into the currently loaded bulk import file.  At the moment it won't auto-save, but I might add that later.
 - Setting to ```false``` will leave the organisation of your bulk files up to you.

core/config.py

@@ -31,6 +31,7 @@ class Config:
         stage_assets: Whether to download assets for seasons and episodes that are not in Plex yet (except Specials)
         track_artwork_ids: Whether to track artwork IDs using Plex labels
         skip_locked_artwork: Whether to skip artwork whose target field is locked in Plex (already set)
+        local_library_matching: Whether to match scraped artwork against the local libraries before fetching poster pages
         auto_manage_bulk_files: Whether to auto-organize bulk files
         reset_overlay: Whether to reset Kometa overlay labels on upload
         schedules: List of scheduled bulk import jobs
@@ -55,6 +56,7 @@ def __init__(self, config_path: str = "config/config.json") -> None:
         self.stage_assets: bool = False
         self.track_artwork_ids: bool = True
         self.skip_locked_artwork: bool = False
+        self.local_library_matching: bool = True
         self.auto_manage_bulk_files: bool = True
         self.reset_overlay: bool = False
         self.schedules: List[Dict[str, Any]] = []
@@ -93,6 +95,7 @@ def load(self) -> None:
             self.bulk_txt = config.get("bulk_txt", "bulk_import.txt")
             self.track_artwork_ids = config.get("track_artwork_ids", True)
             self.skip_locked_artwork = config.get("skip_locked_artwork", False)
+            self.local_library_matching = config.get("local_library_matching", True)
             self.auto_manage_bulk_files = config.get("auto_manage_bulk_files", True)
             self.reset_overlay = config.get("reset_overlay", False)
             self.schedules = config.get("schedules", [])
@@ -120,6 +123,7 @@ def create(self) -> None:
             "stage_assets": False,
             "track_artwork_ids": True,
             "skip_locked_artwork": False,
+            "local_library_matching": True,
             "auto_manage_bulk_files": True,
             "reset_overlay": True,
             "schedules": [],
@@ -161,6 +165,7 @@ def save(self) -> None:
             "bulk_txt": self.bulk_txt,
             "track_artwork_ids": self.track_artwork_ids,
             "skip_locked_artwork": self.skip_locked_artwork,
+            "local_library_matching": self.local_library_matching,
             "auto_manage_bulk_files": self.auto_manage_bulk_files,
             "reset_overlay": self.reset_overlay,
             "schedules": self.schedules,

kometa/kometa_saver.py

@@ -27,6 +27,7 @@ def __init__(
         self.type: Optional[str] = None
         self.artwork: Optional[AnyArtwork] = None
         self.options: Options = Options()
+        self.confirm_match = None
 
     def set_artwork(self, artwork: AnyArtwork) -> None:
         self.artwork = artwork
@@ -75,6 +76,9 @@ def save_to_kometa(self) -> str:
         except OSError as e:
             return f"❌ {self.description} | Error checking existing {self.artwork_type.lower()} asset: {e}"
 
+        if self.confirm_match is not None and not self.confirm_match():
+            return f"⚠️ {self.description} | {self.artwork_type} skipped for {self.library} - artwork is for a different title"
+
         if self.type == "file":
             # Save from local file path
             source_file = self.artwork['path']

plex/library_index.py

@@ -0,0 +1,93 @@
+import re
+import time
+import unicodedata
+from typing import List, Optional, Tuple
+
+from utils.notifications import debug_me
+
+
+def normalize_title(title: str) -> str:
+    """Lowercase, strip accents and punctuation so titles compare equal regardless of styling,
+       e.g. 'Mission: Impossible' vs 'Mission - Impossible', 'Léon' vs 'Leon', 'Mad Max 2' vs 'Mad Max 2!'"""
+    title = unicodedata.normalize("NFKD", title).encode("ascii", "ignore").decode()
+    title = title.casefold().replace("&", " and ")
+    title = re.sub(r"[^\w\s]", " ", title)
+    return re.sub(r"\s+", " ", title).strip()
+
+
+class PlexLibraryIndex:
+    """
+    In-memory index of the configured Plex libraries, so scraped artwork can be matched to
+    the library by title and year without a web request per asset.
+
+    Built once per processing run from a single request per library (library.all() includes
+    guids, titles and years in one response - see the plexapi getGuid docs, which recommend
+    exactly this kind of lookup dictionary for performance).
+    """
+
+    def __init__(self, movie_libraries: List, tv_libraries: List) -> None:
+        self.movie_index: dict = {}
+        self.tv_index: dict = {}
+        start_time = time.time()
+        for library in movie_libraries:
+            self._add_library(self.movie_index, library)
+        for library in tv_libraries:
+            self._add_library(self.tv_index, library)
+        debug_me(f"Indexed {sum(len(v) for v in self.movie_index.values())} movie and "
+                 f"{sum(len(v) for v in self.tv_index.values())} TV index entries "
+                 f"in {time.time() - start_time:.1f}s", "PlexLibraryIndex/__init__")
+
+    def _add_library(self, index: dict, library) -> None:
+        for item in library.all():
+            # Index values come from the listing response only - without this, reading an attribute
+            # the item doesn't have (e.g. originalTitle on most items) makes plexapi reload the
+            # item, which would be one extra request per library item
+            item._autoReload = False
+            tmdb_id: Optional[int] = None
+            for guid in item.guids:
+                if "tmdb://" in guid.id:
+                    try:
+                        tmdb_id = int(guid.id.split("tmdb://", 1)[-1])
+                    except ValueError:
+                        pass
+                    break
+            entry = (item.year, tmdb_id)
+            for key in self._title_keys(item.title, getattr(item, "originalTitle", None)):
+                index.setdefault(key, []).append(entry)
+
+    def _title_keys(self, title: str, original_title: Optional[str]) -> set:
+        """All the normalized keys an item should be findable under."""
+        keys = {normalize_title(title)}
+        if original_title:
+            keys.add(normalize_title(original_title))
+        # Also index without a trailing parenthetical, so "The Office (US)" is findable as "The Office"
+        stripped = re.sub(r"\s*\([^)]*\)\s*$", "", title)
+        if stripped and stripped != title:
+            keys.add(normalize_title(stripped))
+        keys.discard("")
+        return keys
+
+    def lookup(self, kind: str, title: str, year: Optional[int]) -> Tuple[str, Optional[int]]:
+        """
+        Look up a title/year in the index.
+
+        Returns a tuple of:
+        - status (str): "matched", "ambiguous" or "not_found"
+        - tmdb_id (int | None): the TMDb ID when status is "matched"
+        """
+        index = self.movie_index if kind == "movie" else self.tv_index
+        candidates = index.get(normalize_title(title), [])
+        if candidates and year is not None:
+            # Same retry pattern as PlexConnector.movie_or_show: exact year, then -1, then +1
+            for candidate_year in (year, int(year) - 1, int(year) + 1):
+                matched = [c for c in candidates if c[0] == candidate_year]
+                if matched:
+                    break
+        else:
+            matched = candidates
+        tmdb_ids = {tmdb_id for _, tmdb_id in matched if tmdb_id is not None}
+        if len(tmdb_ids) == 1:
+            return "matched", tmdb_ids.pop()
+        if len(tmdb_ids) > 1:
+            return "ambiguous", None
+        return "not_found", None

plex/plex_uploader.py

@@ -29,6 +29,8 @@ def __init__(
         self.track_artwork_ids: bool = True
         self.reset_overlay: bool = False
         self.skip_locked: bool = False
+        self.confirm_match = None
+        self.stale_labels: list = []
 
     def set_artwork(self, artwork: AnyArtwork) -> None:
         self.artwork = artwork
@@ -59,6 +61,9 @@ def upload_to_plex(self) -> str:
                 return f'🔒 {self.description} | {self.artwork_type} locked, skipped in {self.upload_target.librarySectionTitle}'
             if self.artwork_exists_on_plex() is False or self.options.force:
 
+                if self.confirm_match is not None and not self.confirm_match():
+                    return f'⚠️ {self.description} | {self.artwork_type} skipped in {self.upload_target.librarySectionTitle} - artwork is for a different title'
+                self.remove_stale_labels()
                 self.process_overlay_label()
 
                 if self.artwork_id == ArtworkIDPrefix.BACKGROUND.value:
@@ -94,6 +99,7 @@ def upload_to_plex(self) -> str:
 
     def artwork_exists_on_plex(self) -> bool:
         existing_artwork = False
+        self.stale_labels = []
 
         for label in self.upload_target.labels:
             existing_label = str(label)  # Convert the label object to a string if it's not already
@@ -104,8 +110,7 @@ def artwork_exists_on_plex(self) -> bool:
                         self.upload_target.removeLabel(existing_label, False)  # Remove the existing label as we're no longer tracking the artwork IDs
                         self.upload_target.reload()
                 else:
-                    self.upload_target.removeLabel(existing_label, False)  # Remove the existing label as we're replacing the artwork
-                    self.upload_target.reload()
+                    self.stale_labels.append(existing_label)  # Defer removal until we're about to replace the artwork (see remove_stale_labels)
 
         return existing_artwork
 
@@ -117,3 +122,11 @@ def artwork_field_is_locked(self) -> bool:
                 return True
         return False
 
+    def remove_stale_labels(self) -> None:
+        # Remove same-type labels for artwork we're replacing, just before the upload, so a
+        # declined confirmation or a failed upload doesn't leave the item without its label
+        for existing_label in self.stale_labels:
+            self.upload_target.removeLabel(existing_label, False)
+            self.upload_target.reload()
+        self.stale_labels = []
+