diff --git a/README.md b/README.md
index c2b0912..c14c82b 100644
--- a/README.md
+++ b/README.md
@@ -162,6 +162,7 @@ The basic version is available now on the bulk imports page, click on the clock
It's there for when we have API access (and works for scrapers in the meantime) but is limited to running once a day which should be fine.
If you enable ```skip_locked_artwork```, a scheduled user or bulk scrape will only fill items that are still on their default artwork and leave anything you've set by hand untouched, so it's safe to leave running against a curated library.
+With ```local_library_matching``` enabled (the default), a user-catalog scrape skips everything that isn't in your libraries without any web requests, so full-catalog runs take minutes rather than hours.
Additionally, you can configure one or more push notification services so you get a notification every time a scheduled bulk import job completes. Notifications are provided by Apprise. Configure the list of notification services by providing a comma-separated list of Apprise notification URLs through the web UI or via the ```apprise_urls``` variable in the ```config.json```file. Check [the Apprise service list](https://appriseit.com/services/) for details on the supported services and how to set them up and generate a notification URL for your favorite services.
@@ -233,6 +234,9 @@ This is optional - if you don't do this, a new config.json will be created when
- Setting this to ```true``` will skip any artwork whose target field (poster, background or square art) is locked in Plex, unless ```--force``` is used. Plex locks a field whenever artwork is deliberately set - manually or by an upload - so this makes scheduled bulk imports and user scrapes fill items still on default artwork while leaving anything you've already set alone.
- Setting to ```false``` (the default) keeps the existing behaviour where artwork is applied regardless of locks.
+```"local_library_matching"```
+- Setting this to ```true``` (the default) matches scraped artwork against your Plex libraries locally before fetching each poster's page, so big user scrapes skip everything you don't own without a web request and run dramatically faster. The poster page is still checked right before anything is uploaded, so nothing less accurate ever gets written.
+- Setting to ```false``` restores the previous per-poster lookups. Use this if items you own start logging "not available on Plex" because their Plex titles differ from the titles on ThePosterDB.
```"auto_manage_bulk_files"```
- Setting this to ```true``` will automatically add, label and sort URLs from the scrape tab into the currently loaded bulk import file. At the moment it won't auto-save, but I might add that later.
- Setting to ```false``` will leave the organisation of your bulk files up to you.
diff --git a/core/config.py b/core/config.py
index f593424..f5a47a4 100644
--- a/core/config.py
+++ b/core/config.py
@@ -31,6 +31,7 @@ class Config:
stage_assets: Whether to download assets for seasons and episodes that are not in Plex yet (except Specials)
track_artwork_ids: Whether to track artwork IDs using Plex labels
skip_locked_artwork: Whether to skip artwork whose target field is locked in Plex (already set)
+ local_library_matching: Whether to match scraped artwork against the local libraries before fetching poster pages
auto_manage_bulk_files: Whether to auto-organize bulk files
reset_overlay: Whether to reset Kometa overlay labels on upload
schedules: List of scheduled bulk import jobs
@@ -55,6 +56,7 @@ def __init__(self, config_path: str = "config/config.json") -> None:
self.stage_assets: bool = False
self.track_artwork_ids: bool = True
self.skip_locked_artwork: bool = False
+ self.local_library_matching: bool = True
self.auto_manage_bulk_files: bool = True
self.reset_overlay: bool = False
self.schedules: List[Dict[str, Any]] = []
@@ -93,6 +95,7 @@ def load(self) -> None:
self.bulk_txt = config.get("bulk_txt", "bulk_import.txt")
self.track_artwork_ids = config.get("track_artwork_ids", True)
self.skip_locked_artwork = config.get("skip_locked_artwork", False)
+ self.local_library_matching = config.get("local_library_matching", True)
self.auto_manage_bulk_files = config.get("auto_manage_bulk_files", True)
self.reset_overlay = config.get("reset_overlay", False)
self.schedules = config.get("schedules", [])
@@ -120,6 +123,7 @@ def create(self) -> None:
"stage_assets": False,
"track_artwork_ids": True,
"skip_locked_artwork": False,
+ "local_library_matching": True,
"auto_manage_bulk_files": True,
"reset_overlay": True,
"schedules": [],
@@ -161,6 +165,7 @@ def save(self) -> None:
"bulk_txt": self.bulk_txt,
"track_artwork_ids": self.track_artwork_ids,
"skip_locked_artwork": self.skip_locked_artwork,
+ "local_library_matching": self.local_library_matching,
"auto_manage_bulk_files": self.auto_manage_bulk_files,
"reset_overlay": self.reset_overlay,
"schedules": self.schedules,
diff --git a/kometa/kometa_saver.py b/kometa/kometa_saver.py
index 7e30f08..98767b9 100644
--- a/kometa/kometa_saver.py
+++ b/kometa/kometa_saver.py
@@ -27,6 +27,7 @@ def __init__(
self.type: Optional[str] = None
self.artwork: Optional[AnyArtwork] = None
self.options: Options = Options()
+ self.confirm_match = None
def set_artwork(self, artwork: AnyArtwork) -> None:
self.artwork = artwork
@@ -75,6 +76,9 @@ def save_to_kometa(self) -> str:
except OSError as e:
return f"❌ {self.description} | Error checking existing {self.artwork_type.lower()} asset: {e}"
+ if self.confirm_match is not None and not self.confirm_match():
+ return f"⚠️ {self.description} | {self.artwork_type} skipped for {self.library} - artwork is for a different title"
+
if self.type == "file":
# Save from local file path
source_file = self.artwork['path']
diff --git a/plex/library_index.py b/plex/library_index.py
new file mode 100644
index 0000000..17c066a
--- /dev/null
+++ b/plex/library_index.py
@@ -0,0 +1,93 @@
+import re
+import time
+import unicodedata
+from typing import List, Optional, Tuple
+
+from utils.notifications import debug_me
+
+
+def normalize_title(title: str) -> str:
+ """Lowercase, strip accents and punctuation so titles compare equal regardless of styling,
+ e.g. 'Mission: Impossible' vs 'Mission - Impossible', 'Léon' vs 'Leon', 'Mad Max 2' vs 'Mad Max 2!'"""
+ title = unicodedata.normalize("NFKD", title).encode("ascii", "ignore").decode()
+ title = title.casefold().replace("&", " and ")
+ title = re.sub(r"[^\w\s]", " ", title)
+ return re.sub(r"\s+", " ", title).strip()
+
+
+class PlexLibraryIndex:
+ """
+ In-memory index of the configured Plex libraries, so scraped artwork can be matched to
+ the library by title and year without a web request per asset.
+
+ Built once per processing run from a single request per library (library.all() includes
+ guids, titles and years in one response - see the plexapi getGuid docs, which recommend
+ exactly this kind of lookup dictionary for performance).
+ """
+
+ def __init__(self, movie_libraries: List, tv_libraries: List) -> None:
+ self.movie_index: dict = {}
+ self.tv_index: dict = {}
+ start_time = time.time()
+ for library in movie_libraries:
+ self._add_library(self.movie_index, library)
+ for library in tv_libraries:
+ self._add_library(self.tv_index, library)
+ debug_me(f"Indexed {sum(len(v) for v in self.movie_index.values())} movie and "
+ f"{sum(len(v) for v in self.tv_index.values())} TV index entries "
+ f"in {time.time() - start_time:.1f}s", "PlexLibraryIndex/__init__")
+
+ def _add_library(self, index: dict, library) -> None:
+ for item in library.all():
+ # Index values come from the listing response only - without this, reading an attribute
+ # the item doesn't have (e.g. originalTitle on most items) makes plexapi reload the
+ # item, which would be one extra request per library item
+ item._autoReload = False
+ tmdb_id: Optional[int] = None
+ for guid in item.guids:
+ if "tmdb://" in guid.id:
+ try:
+ tmdb_id = int(guid.id.split("tmdb://", 1)[-1])
+ except ValueError:
+ pass
+ break
+ entry = (item.year, tmdb_id)
+ for key in self._title_keys(item.title, getattr(item, "originalTitle", None)):
+ index.setdefault(key, []).append(entry)
+
+ def _title_keys(self, title: str, original_title: Optional[str]) -> set:
+ """All the normalized keys an item should be findable under."""
+ keys = {normalize_title(title)}
+ if original_title:
+ keys.add(normalize_title(original_title))
+ # Also index without a trailing parenthetical, so "The Office (US)" is findable as "The Office"
+ stripped = re.sub(r"\s*\([^)]*\)\s*$", "", title)
+ if stripped and stripped != title:
+ keys.add(normalize_title(stripped))
+ keys.discard("")
+ return keys
+
+ def lookup(self, kind: str, title: str, year: Optional[int]) -> Tuple[str, Optional[int]]:
+ """
+ Look up a title/year in the index.
+
+ Returns a tuple of:
+ - status (str): "matched", "ambiguous" or "not_found"
+ - tmdb_id (int | None): the TMDb ID when status is "matched"
+ """
+ index = self.movie_index if kind == "movie" else self.tv_index
+ candidates = index.get(normalize_title(title), [])
+ if candidates and year is not None:
+ # Same retry pattern as PlexConnector.movie_or_show: exact year, then -1, then +1
+ for candidate_year in (year, int(year) - 1, int(year) + 1):
+ matched = [c for c in candidates if c[0] == candidate_year]
+ if matched:
+ break
+ else:
+ matched = candidates
+ tmdb_ids = {tmdb_id for _, tmdb_id in matched if tmdb_id is not None}
+ if len(tmdb_ids) == 1:
+ return "matched", tmdb_ids.pop()
+ if len(tmdb_ids) > 1:
+ return "ambiguous", None
+ return "not_found", None
diff --git a/plex/plex_uploader.py b/plex/plex_uploader.py
index 15b2469..91e00c5 100644
--- a/plex/plex_uploader.py
+++ b/plex/plex_uploader.py
@@ -29,6 +29,8 @@ def __init__(
self.track_artwork_ids: bool = True
self.reset_overlay: bool = False
self.skip_locked: bool = False
+ self.confirm_match = None
+ self.stale_labels: list = []
def set_artwork(self, artwork: AnyArtwork) -> None:
self.artwork = artwork
@@ -59,6 +61,9 @@ def upload_to_plex(self) -> str:
return f'🔒 {self.description} | {self.artwork_type} locked, skipped in {self.upload_target.librarySectionTitle}'
if self.artwork_exists_on_plex() is False or self.options.force:
+ if self.confirm_match is not None and not self.confirm_match():
+ return f'⚠️ {self.description} | {self.artwork_type} skipped in {self.upload_target.librarySectionTitle} - artwork is for a different title'
+ self.remove_stale_labels()
self.process_overlay_label()
if self.artwork_id == ArtworkIDPrefix.BACKGROUND.value:
@@ -94,6 +99,7 @@ def upload_to_plex(self) -> str:
def artwork_exists_on_plex(self) -> bool:
existing_artwork = False
+ self.stale_labels = []
for label in self.upload_target.labels:
existing_label = str(label) # Convert the label object to a string if it's not already
@@ -104,8 +110,7 @@ def artwork_exists_on_plex(self) -> bool:
self.upload_target.removeLabel(existing_label, False) # Remove the existing label as we're no longer tracking the artwork IDs
self.upload_target.reload()
else:
- self.upload_target.removeLabel(existing_label, False) # Remove the existing label as we're replacing the artwork
- self.upload_target.reload()
+ self.stale_labels.append(existing_label) # Defer removal until we're about to replace the artwork (see remove_stale_labels)
return existing_artwork
@@ -117,3 +122,11 @@ def artwork_field_is_locked(self) -> bool:
return True
return False
+ def remove_stale_labels(self) -> None:
+ # Remove same-type labels for artwork we're replacing, just before the upload, so a
+ # declined confirmation or a failed upload doesn't leave the item without its label
+ for existing_label in self.stale_labels:
+ self.upload_target.removeLabel(existing_label, False)
+ self.upload_target.reload()
+ self.stale_labels = []
+
diff --git a/processors/upload_processor.py b/processors/upload_processor.py
index ad65bff..33d2108 100644
--- a/processors/upload_processor.py
+++ b/processors/upload_processor.py
@@ -8,6 +8,7 @@
from models.options import Options
from plex.plex_connector import PlexConnector
from plex.plex_uploader import PlexUploader
+from plex.library_index import PlexLibraryIndex
from plexapi.exceptions import NotFound
from kometa.kometa_saver import KometaSaver
from utils import soup_utils
@@ -24,6 +25,8 @@ def __init__(self, plex: PlexConnector) -> None:
self.options: Options = Options()
self.config: Config = Config()
self.config.load()
+ self._media_index: Optional[PlexLibraryIndex] = None
+ self._match_confirm_cache: dict = {}
def set_options(self, options: Options) -> None:
@@ -31,6 +34,82 @@ def set_options(self, options: Options) -> None:
self.kometa: bool = self.options.kometa or globals.config.save_to_kometa
self.skip_locked: bool = self.options.skip_locked or globals.config.skip_locked_artwork
+ def _resolve_tmdb_id(self, artwork, description: str, kind: str) -> bool:
+ """
+ Resolve the TMDb ID for a TPDb artwork item before matching it to the library.
+
+ With local_library_matching enabled (the default), the title and year from the scrape are
+ matched against an in-memory index of the Plex libraries first, so items that aren't in the
+ library are skipped without any web request, and items that are get their TMDb ID from
+ Plex's own guids. Only an ambiguous local match falls back to fetching the poster page.
+
+ Returns True when the artwork was matched locally - the caller then attaches a
+ confirm_match hook so the poster page is still checked before anything is written.
+ """
+ if artwork.get("tmdb_id") or artwork.get("source") != ScraperSource.THEPOSTERDB.value or artwork.get("id") == "Upload":
+ return False
+
+ if self.config.local_library_matching and artwork.get("title"):
+ if self._media_index is None:
+ self._media_index = PlexLibraryIndex(self.plex.movie_libraries, self.plex.tv_libraries)
+ status, tmdb_id = self._media_index.lookup(kind, artwork.get("title"), artwork.get("year"))
+ if status == "not_found":
+ if kind == "movie":
+ raise MovieNotFound(f'{description} | Movie not available on Plex')
+ raise ShowNotFound(f'{description} | Show not available on Plex')
+ if status == "matched":
+ debug_me(f"Matched '{artwork.get('title')} ({artwork.get('year')})' locally as TMDb ID '{tmdb_id}'", "UploadProcessor/_resolve_tmdb_id")
+ artwork["tmdb_id"] = tmdb_id
+ return True
+ debug_me(f"'{artwork.get('title')} ({artwork.get('year')})' is ambiguous locally, fetching the poster page", "UploadProcessor/_resolve_tmdb_id")
+
+ self._fetch_tmdb_id_from_tpdb(artwork, description)
+ return False
+
+ def _fetch_tmdb_id_from_tpdb(self, artwork, description: str) -> None:
+ """Fetch the poster page from ThePosterDB to read its TMDb ID (data-media-id), falling back
+ to a local Plex title/year search if the page doesn't expose one."""
+ poster_id = artwork.get("id", None)
+ poster_page_url = f"https://theposterdb.com/poster/{poster_id}"
+ debug_me(f"Fetching TMDb ID from '{poster_page_url}'", "UploadProcessor/_fetch_tmdb_id_from_tpdb")
+ try:
+ poster_page_soup = soup_utils.cook_soup(poster_page_url)
+ except ScraperException as e:
+ debug_me(f"Unable to fetch TMDb ID due to error: {str(e)}", "UploadProcessor/_fetch_tmdb_id_from_tpdb")
+ raise ScraperException(f"{description} | {str(e)}") from None
+ try:
+ artwork["tmdb_id"] = int(poster_page_soup.find('div', {"data-media-id": True})['data-media-id'])
+ except (KeyError, TypeError, ValueError) as e:
+ debug_me(f"Failed to extract TMDb ID from poster page, trying another way. Error was: {e}", "UploadProcessor/_fetch_tmdb_id_from_tpdb")
+ _, artwork["tmdb_id"], _, _ = self.plex.movie_or_show(artwork.get("title"), artwork.get("year"))
+ debug_me(f"Found TMDb ID '{artwork['tmdb_id']}' for '{artwork.get('title')}' using Plex search.", "UploadProcessor/_fetch_tmdb_id_from_tpdb")
+
+ def _artwork_matches_item(self, artwork, plex_item, kind: str) -> bool:
+ """
+ Called by the uploader/saver just before artwork is actually written, and only for
+ locally-matched items: fetches the poster page once (cached per title) and checks the
+ TPDb media id against the matched Plex item's guids, so a title-and-year match can never
+ write another title's artwork. Skips, locked items and items not in the library never
+ trigger this request.
+ """
+ cache_key = (kind, artwork.get("title"), artwork.get("year"), artwork.get("tmdb_id"))
+ cached = self._match_confirm_cache.get(cache_key)
+ if cached is not None:
+ return cached
+ poster_page_url = f"https://theposterdb.com/poster/{artwork.get('id')}"
+ debug_me(f"Confirming local match for '{artwork.get('title')}' from '{poster_page_url}'", "UploadProcessor/_artwork_matches_item")
+ try:
+ poster_page_soup = soup_utils.cook_soup(poster_page_url)
+ except ScraperException:
+ raise
+ try:
+ tpdb_tmdb_id = int(poster_page_soup.find('div', {"data-media-id": True})['data-media-id'])
+ matches = any(guid.id == f"tmdb://{tpdb_tmdb_id}" for guid in plex_item.guids)
+ except (KeyError, TypeError, ValueError):
+ matches = True # No media id on the page - trust the local title and year match, same as the existing Plex-search fallback
+ self._match_confirm_cache[cache_key] = matches
+ return matches
+
def process_collection_artwork(self, artwork: CollectionArtwork) -> Optional[str]:
try:
@@ -88,22 +167,8 @@ def process_movie_artwork(self, artwork: MovieArtwork) -> Optional[str]:
artwork_type = ARTWORK_TYPE_MAP.get(artwork.get('file_type'))
artwork_id = ARTWORK_ID_MAP.get(artwork.get('file_type'))
- # Since the TBDb scraper doesn't fetch the TMDb ID up front for each poster, we need to get it here
- if not artwork.get('tmdb_id') and artwork.get("source") == ScraperSource.THEPOSTERDB.value and artwork.get('id') != ScraperSource.UPLOAD.value:
- poster_id=artwork.get('id', None)
- poster_page_url = f"https://theposterdb.com/poster/{poster_id}"
- debug_me(f"Fetching TMDb ID from '{poster_page_url}'")
- try:
- poster_page_soup = soup_utils.cook_soup(poster_page_url)
- except ScraperException as e:
- debug_me(f"Unable to fetch TMDb ID due to error: {str(e)}", "UploadProcesser/process_tv_artwork")
- raise ScraperException(f"{description} | {str(e)}") from None
- try:
- artwork['tmdb_id'] = int(poster_page_soup.find('div', {"data-media-id": True})['data-media-id'])
- except (KeyError, TypeError, ValueError) as e:
- debug_me(f"Failed to extract TMDb ID from poster page, trying another way. Error was: {e}")
- _, artwork['tmdb_id'], _, _= self.plex.movie_or_show(artwork.get('title'), artwork.get('year'))
- debug_me(f"Found TMDb ID '{artwork['tmdb_id']}' for '{artwork.get('title')}' using Plex search.")
+ # Since the TPDb scraper doesn't fetch the TMDb ID up front for each poster, we resolve it here
+ locally_matched = self._resolve_tmdb_id(artwork, description, "movie")
try:
movie_items, libraries = self.plex.find_in_library("movie", artwork)
@@ -131,6 +196,8 @@ def process_movie_artwork(self, artwork: MovieArtwork) -> Optional[str]:
saver.dest_file_ext = ".jpg"
saver.set_description(desc)
saver.set_options(self.options)
+ if locally_matched:
+ saver.confirm_match = lambda a=artwork, item=movie_item: self._artwork_matches_item(a, item, "movie")
result = saver.save_to_kometa()
results.append(result)
else:
@@ -139,6 +206,8 @@ def process_movie_artwork(self, artwork: MovieArtwork) -> Optional[str]:
uploader.track_artwork_ids = self.config.track_artwork_ids
uploader.reset_overlay = self.config.reset_overlay
uploader.skip_locked = self.skip_locked
+ if locally_matched:
+ uploader.confirm_match = lambda a=artwork, item=movie_item: self._artwork_matches_item(a, item, "movie")
uploader.set_description(desc)
uploader.set_options(self.options)
result = uploader.upload_to_plex()
@@ -176,23 +245,8 @@ def process_tv_artwork(self, artwork: TVArtwork) -> Optional[str]:
artwork['year'] = self.options.year if self.options.year else artwork['year']
artwork_type = ARTWORK_TYPE_MAP.get(artwork.get('file_type'))
- # Since the TBDb scraper doesn't fetch the TMDb ID up front for each poster, we need to get it here
- if not artwork.get('tmdb_id') and artwork.get('source') == ScraperSource.THEPOSTERDB.value and artwork.get('id') != ScraperSource.UPLOAD.value:
- poster_id=artwork.get('id', None)
- poster_page_url = f"https://theposterdb.com/poster/{poster_id}"
- debug_me(f"Fetching TMDb ID from '{poster_page_url}'")
- try:
- poster_page_soup = soup_utils.cook_soup(poster_page_url)
- except ScraperException as e:
- debug_me(f"Unable to fetch TMDb ID due to error: {str(e)}", "UploadProcesser/process_tv_artwork")
- raise ScraperException(f"{description} | {str(e)}") from None
- try:
- artwork['tmdb_id'] = int(poster_page_soup.find('div', {"data-media-id": True})['data-media-id'])
- except (KeyError, TypeError, ValueError) as e:
- debug_me(f"Failed to extract TMDb ID from poster page, trying another way. Error was: {e}")
- _, artwork['tmdb_id'], _, _= self.plex.movie_or_show(artwork.get('title'), artwork.get('year'))
- debug_me(f"Found TMDb ID '{artwork['tmdb_id']}' for '{artwork.get('title')}' using Plex search.")
-
+ # Since the TPDb scraper doesn't fetch the TMDb ID up front for each poster, we resolve it here
+ locally_matched = self._resolve_tmdb_id(artwork, description, "tv")
try:
tv_show_items, libraries = self.plex.find_in_library("tv", artwork)
@@ -277,6 +331,8 @@ def process_tv_artwork(self, artwork: TVArtwork) -> Optional[str]:
saver.dest_file_ext = ".jpg"
saver.set_description(desc)
saver.set_options(self.options)
+ if locally_matched:
+ saver.confirm_match = lambda a=artwork, item=tv_show: self._artwork_matches_item(a, item, "tv")
result = saver.save_to_kometa()
results.append(result)
elif upload_target:
@@ -286,6 +342,8 @@ def process_tv_artwork(self, artwork: TVArtwork) -> Optional[str]:
uploader.track_artwork_ids = self.config.track_artwork_ids
uploader.reset_overlay = self.config.reset_overlay
uploader.skip_locked = self.skip_locked
+ if locally_matched:
+ uploader.confirm_match = lambda a=artwork, item=tv_show: self._artwork_matches_item(a, item, "tv")
uploader.set_description(desc)
uploader.set_options(self.options)
result = uploader.upload_to_plex()
diff --git a/static/web_interface.js b/static/web_interface.js
index 7ad2188..8315411 100644
--- a/static/web_interface.js
+++ b/static/web_interface.js
@@ -631,6 +631,8 @@ function saveConfig() {
// Checkbox for skipping artwork with locked fields in Plex
save_config.skip_locked_artwork = document.getElementById("skip_locked_artwork").checked;
toggleSkipLockedCheckbox();
+ // Checkbox for matching artwork against the local libraries before fetching poster pages
+ save_config.local_library_matching = document.getElementById("local_library_matching").checked;
// Get selected mediux filters
save_config.mediux_filters = Array.from(document.querySelectorAll('[id^="m_filter-"]:checked'))
@@ -709,6 +711,7 @@ function loadConfig() {
document.getElementById("auto_manage_bulk_files").checked = data.config.auto_manage_bulk_files;
document.getElementById("reset_overlay").checked = data.config.reset_overlay;
document.getElementById("skip_locked_artwork").checked = data.config.skip_locked_artwork;
+ document.getElementById("local_library_matching").checked = data.config.local_library_matching;
document.getElementById("option-add-to-bulk").checked = data.config.auto_manage_bulk_files;
document.getElementById("apprise_urls").value = data.config.apprise_urls.join(", ");
diff --git a/templates/web_interface.html b/templates/web_interface.html
index b8ca8ac..60a5917 100644
--- a/templates/web_interface.html
+++ b/templates/web_interface.html
@@ -396,6 +396,10 @@