lastgenre: Genre spelling normalization (aliases)

beetsplug/lastgenre/__init__.py

@@ -22,7 +22,7 @@
 from beets import config, library, plugins, ui
 from beets.library import Album, Item
 from beets.util import plurality, unique_list
-from beetsplug.lastgenre.utils import drop_ignored_genres, is_ignored
+from beetsplug.lastgenre.utils import is_ignored, normalize_genre
 
 from .client import LastFmClient
 
@@ -33,7 +33,7 @@
     from beets.importer import ImportSession, ImportTask
     from beets.library import LibModel
 
-    from .utils import GenreIgnorePatterns
+    from .utils import AliasPatternWithReplacement, IgnorePatternsByArtist
 
     Whitelist = set[str]
     """Set of valid genre names (lowercase). Empty set means all genres allowed."""
@@ -98,6 +98,7 @@ def sort_by_depth(tags: list[str], branches: CanonTree) -> list[str]:
 
 WHITELIST = os.path.join(os.path.dirname(__file__), "genres.txt")
 C14N_TREE = os.path.join(os.path.dirname(__file__), "genres-tree.yaml")
+ALIASES_FILE = os.path.join(os.path.dirname(__file__), "aliases.yaml")
 
 
 class LastGenrePlugin(plugins.BeetsPlugin):
@@ -120,6 +121,7 @@ def __init__(self) -> None:
                 "title_case": True,
                 "pretend": False,
                 "ignorelist": {},
+                "aliases": True,
             }
         )
         self.setup()
@@ -132,9 +134,15 @@ def setup(self) -> None:
         self.whitelist: Whitelist = self._load_whitelist()
         self.c14n_branches: CanonTree
         self.c14n_branches, self.canonicalize = self._load_c14n_tree()
-        self.ignore_patterns: GenreIgnorePatterns = self._load_ignorelist()
+        self.ignore_patterns: IgnorePatternsByArtist = self._load_ignorelist()
+        self.alias_patterns: list[AliasPatternWithReplacement] = (
+            self._load_aliases()
+        )
         self.client = LastFmClient(
-            self._log, self.config["min_weight"].get(int), self.ignore_patterns
+            self._log,
+            self.config["min_weight"].get(int),
+            self.ignore_patterns,
+            self.alias_patterns,
         )
 
     def _load_whitelist(self) -> Whitelist:
@@ -178,7 +186,7 @@ def _load_c14n_tree(self) -> tuple[CanonTree, bool]:
             flatten_tree(genres_tree, [], c14n_branches)
         return c14n_branches, canonicalize
 
-    def _load_ignorelist(self) -> GenreIgnorePatterns:
+    def _load_ignorelist(self) -> IgnorePatternsByArtist:
         r"""Load patterns from configuration and compile them.
 
         Mapping of artist names to regex or literal patterns. Use the
@@ -209,7 +217,7 @@ def _load_ignorelist(self) -> GenreIgnorePatterns:
             confuse.MappingValues(confuse.Sequence(str))
         )
 
-        compiled_ignorelist: GenreIgnorePatterns = defaultdict(list)
+        compiled_ignorelist: IgnorePatternsByArtist = defaultdict(list)
         for artist, patterns in raw_ignorelist.items():
             artist_patterns = []
             for pattern in patterns:
@@ -225,10 +233,62 @@ def _load_ignorelist(self) -> GenreIgnorePatterns:
                 [p.pattern for p in artist_patterns],
             )
 
-            compiled_ignorelist[artist] = artist_patterns
+            compiled_ignorelist[artist.lower()] = artist_patterns
 
         return compiled_ignorelist
 
+    def _load_aliases(self) -> list[AliasPatternWithReplacement]:
+        """Load the genre alias table from the beets config.
+
+        ``lastgenre.aliases`` is a tri-state option:
+
+        - ``yes`` (default): load the built-in aliases.
+        - ``no``: disable alias normalization entirely.
+        - mapping: an inline dict of canonical genre names to lists of regex
+          patterns.
+
+        The key (genre name) is used as a ``re.Match.expand()`` template,
+        so ``\\1`` / ``\\g<N>`` back-references to capture groups are supported.
+
+        Raises:
+            confuse.ConfigTypeError: when the config value is not a bool or
+            mapping, or when a mapping value is not a list.
+            re.error: when a pattern is not valid regex syntax.
+        """
+        aliases_config = self.config["aliases"].get()
+        if aliases_config is False:
+            return []
+
+        # Define view with either built-in or user-configured
+        aliases_view = confuse.Configuration(
+            self.config["aliases"].name, read=False
+        )
+        if aliases_config in (True, "", None):
+            self._log.debug("Loading built-in aliases")
+            with Path(ALIASES_FILE).open(encoding="utf-8") as f:
+                aliases_view.set(yaml.safe_load(f))
+        elif not isinstance(aliases_config, dict):
+            raise confuse.ConfigTypeError(
+                f"{self.config['aliases'].name} must be a dict or bool."
+            )
+        else:
+            aliases_view.set(aliases_config)
+
+        # Parse and compile. Raise for invalid regex!
+        raw_aliases = aliases_view.get(
+            confuse.MappingValues(confuse.Sequence(str))
+        )
+        compiled_aliases: list[AliasPatternWithReplacement] = []
+        for canonical, patterns in raw_aliases.items():
+            lower_canonical = canonical.lower()
+            compiled_aliases.extend(
+                (re.compile(p, re.IGNORECASE), lower_canonical)
+                for p in patterns
+            )
+
+        self._log.debug("Loaded {} alias entries", len(compiled_aliases))
+        return compiled_aliases
+
     @property
     def sources(self) -> tuple[str, ...]:
         """A tuple of allowed genre sources. May contain 'track',
@@ -250,6 +310,8 @@ def _resolve_genres(
         """Canonicalize, sort and filter a list of genres.
 
         - Returns an empty list if the input tags list is empty.
+        - If aliases are configured, variant spellings are normalised first
+          (e.g. 'hip-hop' → 'hip hop', 'dnb' → 'drum and bass').
         - If canonicalization is enabled, it extends the list by incorporating
           parent genres from the canonicalization tree. When a whitelist is set,
           only parent tags that pass the whitelist filter are included;
@@ -269,6 +331,13 @@ def _resolve_genres(
         if not tags:
             return []
 
+        # Normalize variant spellings before any other processing.
+        if self.alias_patterns:
+            tags = [
+                normalize_genre(self._log, self.alias_patterns, tag)
+                for tag in tags
+            ]
+
         count = self.config["count"].get(int)
 
         # Canonicalization (if enabled)
@@ -325,24 +394,18 @@ def _filter_valid(
     ) -> list[str]:
         """Filter genres through whitelist and ignorelist.
 
-        Drops empty/whitespace-only strings, then applies whitelist and
-        ignorelist checks. Returns all genres if neither is configured.
-        Whitelist is checked first for performance reasons (ignorelist regex
-        matching is more expensive and for some call sites ignored genres were
-        already filtered).
+        Strips leading/trailing whitespace and drops empty strings, then
+        applies whitelist and ignorelist checks. Whitelist is checked first
+        for performance reasons (ignorelist regex matching is more expensive
+        and for some call sites ignored genres were already filtered).
         """
-        cleaned = [g for g in genres if g and g.strip()]
-        if not self.whitelist and not self.ignore_patterns:
-            return cleaned
-
-        whitelisted = [
+        non_blank = [s for g in genres if (s := g.strip())]
+        return [
             g
-            for g in cleaned
-            if not self.whitelist or g.lower() in self.whitelist
+            for g in non_blank
+            if (not self.whitelist or g.lower() in self.whitelist)
+            and not is_ignored(self._log, self.ignore_patterns, g, artist)
         ]
-        return drop_ignored_genres(
-            self._log, self.ignore_patterns, whitelisted, artist
-        )
 
     # Genre resolution pipeline.
 

beetsplug/lastgenre/aliases.yaml

@@ -0,0 +1,72 @@
+# Ampersands / Delimiters
+drum and bass:
+    - d(rum)?[ &n/]*b(ass)?
+r&b:
+    - r(hythm)?[ &n/]*b(lues)?
+rock and roll:
+    - rock[- ‐'&n/]*roll
+
+# Hyphenation
+\1-\2:
+    - ([ckj])[- ]*(pop)
+    - (j)[- ]*(rock)
+    - (post)[- ]*(\w+)
+    - (g?lo)[- ]*(fi)
+    - ([pg])[- ]*(funk)
+    - (synth)[- ]*(pop)
+avant-garde:
+    - avant[- ]*(garde?)?
+
+# Adding space
+\1 \2:
+    - (nu)[ -]*(disco|jazz|metal|soul)
+    - (dark)[ -]*(wave)
+    - (blues|folk|indie|gothic|progressive|alternative|punk)[ -]*(rock)
+    - (ska)[ -]*(jazz)
+    - (synth)[- ]*(funk)
+    - (glitch|hip|trip)y?(?:[ -]*hip)?[ -]*(hop)
+
+# Removing delimiters
+\1\2:
+    - (synth)[- ]*(wave)
+    - (psy)[ -]*(trance)
+    - (chill)[ -]*(out)
+
+# Old School spellings: old-school death metal, oldschool hip-hop, old-skool-jungle/-hardcore
+old school death metal:
+    - old[ -]*(?:school|skool)[ -]+death[ -]*metal
+old school hip hop:
+    - old[ -]*(?:school|skool)[ -]+hip[ -]*hop
+old school \1:
+    - old[ -]*(?:school|skool)[ -]+(\w+)
+
+# Terminology / Synonyms
+electronic:
+    - electronic music
+world music:
+    - world
+downtempo:
+    - down[ -]*beat
+shoegaze:
+    - shoegaz(e?r?|ing)
+footwork:
+    - (chicago )?footwork( music)?
+juke:
+    - (chicago )?juke( music| house)?
+
+# Compound genres
+alternative rock:
+    - alt[ -]*rock
+# negative lookahead avoids matching 'gothic metal'
+gothic rock:
+    - goth(?!ic)([ -]*rock)?
+# avoids matching 'progressive metal'
+progressive rock:
+    - prog([ -]*rock)?
+punk rock:
+    - punk
+# psy folk, psych-pop, psychedelic-rock, psychedelic soul, but not psychedelic (standalone)
+psychedelic \1:
+    - psy(?:ch(?:edelic)?)?[ -]*(folk|pop|rock|soul)
+dixieland:
+    - dixieland[ -]*jazz

beetsplug/lastgenre/client.py

@@ -9,15 +9,15 @@
 
 from beets import plugins
 
-from .utils import drop_ignored_genres
+from .utils import is_ignored, normalize_genre
 
 if TYPE_CHECKING:
     from collections.abc import Callable
 
     from beets.library import LibModel
     from beets.logging import BeetsLogger
 
-    from .utils import GenreIgnorePatterns
+    from .utils import AliasPatternWithReplacement, IgnorePatternsByArtist
 
     GenreCache = dict[str, list[str]]
     """Cache mapping entity keys to their genre lists.
@@ -52,17 +52,19 @@ def __init__(
         self,
         log: BeetsLogger,
         min_weight: int,
-        ignore_patterns: GenreIgnorePatterns,
+        ignore_patterns: IgnorePatternsByArtist,
+        alias_patterns: list[AliasPatternWithReplacement],
     ):
         """Initialize the client.
 
         The min_weight parameter filters tags by their minimum weight.
         The ignorelist filters forbidden genres directly after Last.fm lookup.
         """
-        self._log = log
-        self._min_weight = min_weight
-        self._ignore_patterns: GenreIgnorePatterns = ignore_patterns
-        self._genre_cache: GenreCache = {}
+        self.log = log
+        self.min_weight = min_weight
+        self.ignore_patterns: IgnorePatternsByArtist = ignore_patterns
+        self.alias_patterns: list[AliasPatternWithReplacement] = alias_patterns
+        self.genre_cache: GenreCache = {}
 
     def fetch_genres(
         self, obj: pylast.Album | pylast.Artist | pylast.Track
@@ -71,16 +73,16 @@ def fetch_genres(
         try:
             res = obj.get_top_tags()
         except PYLAST_EXCEPTIONS as exc:
-            self._log.debug("last.fm error: {}", exc)
+            self.log.debug("last.fm error: {}", exc)
             return []
         except Exception as exc:
             # Isolate bugs in pylast.
-            self._log.debug("{}", traceback.format_exc())
-            self._log.error("error in pylast library: {}", exc)
+            self.log.debug("{}", traceback.format_exc())
+            self.log.error("error in pylast library: {}", exc)
             return []
 
         # Filter by weight (optionally).
-        if min_weight := self._min_weight:
+        if min_weight := self.min_weight:
             res = [el for el in res if (int(el.weight or 0)) >= min_weight]
 
         # Get strings from tags.
@@ -103,19 +105,21 @@ def _last_lookup(
 
         args_replaced = [a.replace("\u2010", "-") for a in args]
         key = f"{entity}.{'-'.join(str(a) for a in args_replaced)}"
-        if key not in self._genre_cache:
-            self._genre_cache[key] = self.fetch_genres(method(*args_replaced))
+        if key not in self.genre_cache:
+            self.genre_cache[key] = self.fetch_genres(method(*args_replaced))
 
-        genres = self._genre_cache[key]
-        self._log.extra_debug(
-            "last.fm (unfiltered) {} tags: {}", entity, genres
-        )
+        genres = self.genre_cache[key]
+        self.log.extra_debug("last.fm (unfiltered) {} tags: {}", entity, genres)
 
+        # Apply aliases and log each change.
         # Filter forbidden genres on every call so ignorelist hits are logged.
         # Artist is always the first element in args (album, artist, track lookups).
-        return drop_ignored_genres(
-            self._log, self._ignore_patterns, genres, args[0]
-        )
+        return [
+            normal
+            for g in genres
+            if (normal := normalize_genre(self.log, self.alias_patterns, g))
+            and not is_ignored(self.log, self.ignore_patterns, normal, args[0])
+        ]
 
     def fetch(self, kind: str, obj: LibModel, *args: str) -> list[str]:
         """Fetch Last.fm genres for the specified kind and entity.