Add 42 languages with Pygments fallback and CLI updates

src/PyReprism/__init__.py

@@ -60,6 +60,14 @@ def get_language(lang: LanguageLike) -> Type:
                or LanguageRegistry.get(name.upper()))
         if cls:
             return cls
+        # Long-tail fallback: no built-in module matched, so try a Pygments-backed
+        # language (500+ lexers). Only a language name or an extension is eligible,
+        # and only when Pygments is installed; otherwise fall through to the error.
+        from .languages.dynamic import pygments_language_for
+        spec = name if (name.startswith('.') or '.' not in name) else os.path.splitext(name)[1]
+        cls = pygments_language_for(spec)
+        if cls:
+            return cls
     raise ValueError(f"Unknown language: {lang!r}")
 
 

src/PyReprism/cli.py

@@ -9,6 +9,7 @@
     pyreprism preprocess --steps comments,strings,whitespace file.java
     pyreprism tokenize --json file.py
     pyreprism languages
+    pyreprism languages --pygments
 """
 import argparse
 import glob
@@ -332,22 +333,71 @@ def _cmd_clones(args) -> int:
     return 0
 
 
+# Pygments lexers that are generic fallbacks, not real languages.
+_PYGMENTS_SKIP = {'text', 'raw', 'output', 'pseudo', ' specials'}
+
+
+def _pygments_fallback_rows(builtin_exts):
+    """Rows ``(alias, ext)`` for Pygments lexers not covered by a built-in module.
+
+    Returns ``None`` when Pygments is not installed. Deduped by primary extension
+    so the count reflects genuinely *additional* coverage.
+    """
+    try:
+        from pygments.lexers import get_all_lexers
+    except ImportError:
+        return None
+    rows, seen_ext = [], set()
+    for name, aliases, filenames, _mime in get_all_lexers():
+        alias = aliases[0] if aliases else name
+        if alias.lower() in _PYGMENTS_SKIP or name == 'Text only':
+            continue
+        ext = next((f[1:] for f in filenames if f.startswith('*.')), '')
+        if not ext or ext in builtin_exts or ext in seen_ext:
+            continue
+        seen_ext.add(ext)
+        rows.append((alias, ext))
+    return sorted(rows)
+
+
 def _cmd_languages(args) -> int:
     from .languages import _load_all_languages
     from .languages.registry import LanguageRegistry
 
     _load_all_languages()
     rows = []
+    builtin_exts = set()
     for name, cls in sorted(LanguageRegistry.all().items()):
         try:
             ext = cls.file_extension()
         except Exception:
             ext = '?'
         rows.append((name, ext))
+        builtin_exts.add(ext)
+
+    if getattr(args, 'pygments', False):
+        extra = _pygments_fallback_rows(builtin_exts)
+        if extra is None:
+            print("Pygments is not installed; install it with "
+                  "`pip install pyreprism[accurate]` to enable the fallback tier.",
+                  file=sys.stderr)
+            return 1
+        width = max((len(n) for n, _ in extra), default=0)
+        for name, ext in extra:
+            print(f"{name.ljust(width)}  {ext}")
+        print(f"\n{len(extra)} additional languages via the Pygments fallback "
+              f"(resolved on demand, not validated)", file=sys.stderr)
+        return 0
+
     width = max((len(n) for n, _ in rows), default=0)
     for name, ext in rows:
         print(f"{name.ljust(width)}  {ext}")
-    print(f"\n{len(rows)} languages", file=sys.stderr)
+    footer = f"\n{len(rows)} built-in languages (zero-dependency)"
+    extra = _pygments_fallback_rows(builtin_exts)
+    if extra:
+        footer += (f"; +{len(extra)} more available via the optional Pygments "
+                   f"backend (see `pyreprism languages --pygments`)")
+    print(footer, file=sys.stderr)
     return 0
 
 
@@ -378,7 +428,8 @@ def build_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(
         prog='pyreprism',
         description='Preprocess source code: strip/extract/count comments, strings, '
-                    'numbers, operators, keywords and identifiers across 145+ languages.',
+                    'numbers, operators, keywords and identifiers across 160+ '
+                    'built-in languages (500+ with the optional Pygments backend).',
     )
     parser.add_argument('--version', action='version', version=f'pyreprism {__version__}')
     sub = parser.add_subparsers(dest='command', required=True)
@@ -522,6 +573,9 @@ def add_fingerprint_opts(p):
     p_clones.set_defaults(func=_cmd_clones)
 
     p_langs = sub.add_parser('languages', help='list supported languages and extensions')
+    p_langs.add_argument('--pygments', action='store_true',
+                         help='instead list the additional languages available via '
+                              'the optional Pygments fallback backend')
     p_langs.set_defaults(func=_cmd_languages)
 
     return parser

src/PyReprism/languages/__init__.py

@@ -71,6 +71,9 @@ def _load_all_languages() -> None:
     '.sql': 'SQL',
     '.vb': 'Vbnet',
     '.js': 'JavaScript',
+    # '.v' is shared by Verilog and V (vlang); Verilog is far more common in
+    # existing corpora, so it wins extension lookup. Resolve V by name (`v`).
+    '.v': 'Verilog',
 }
 
 

src/PyReprism/languages/_scanners.py

@@ -0,0 +1,137 @@
+"""Span-driven language base for comment syntaxes a regex cannot express.
+
+Most languages declare a ``comment_regex`` and inherit the whole API from
+:class:`~PyReprism.languages.base.BaseLanguage`. A few do not fit a regular
+expression:
+
+* **Nested block comments** (Odin, V, Jai): ``/* /* */ */`` requires counting
+  nesting depth, which a regex cannot do.
+* **Column-sensitive comments** (COBOL): a ``*`` in the indicator column marks the
+  whole line as a comment.
+
+:class:`ScannedLanguage` handles these by delegating to a single classmethod,
+:meth:`ScannedLanguage._comment_spans`, that returns the ``(start, end)`` offsets
+of every comment. All comment operations---``match``/``extract``/``count``/%
+``remove``---and a consistent :meth:`tokenize` are derived from those spans, so a
+scanned language behaves exactly like a regex one (losslessly) without a
+``comment_regex``. Code between comments is sub-classified by the inherited
+:meth:`~PyReprism.languages.base.BaseLanguage._tokenize_code`.
+
+The module name is underscore-prefixed so it is *not* auto-imported as a language
+by ``_load_all_languages``; concrete languages import it explicitly.
+"""
+from typing import List, Tuple
+
+from ..tokens import Token, TokenType
+from .base import BaseLanguage, _line_at, _newline_offsets
+
+
+class ScannedLanguage(BaseLanguage):
+    """A language whose comments are found by a scanner, not a regex."""
+
+    @classmethod
+    def _comment_spans(cls, source: str) -> List[Tuple[int, int]]:
+        """Return sorted, non-overlapping ``(start, end)`` offsets of comments."""
+        raise NotImplementedError
+
+    @classmethod
+    def comment_regex(cls):
+        raise NotImplementedError(
+            f"{cls.__name__} is scanned, not regex-based; use the comment methods."
+        )
+
+    @classmethod
+    def match_comments(cls, source: str) -> List[Token]:
+        newlines = _newline_offsets(source)
+        return [Token(TokenType.COMMENT, source[s:e], s, e, _line_at(newlines, s))
+                for s, e in cls._comment_spans(source) if e > s]
+
+    @classmethod
+    def extract_comments(cls, source: str) -> List[str]:
+        return [t.value for t in cls.match_comments(source)]
+
+    @classmethod
+    def count_comments(cls, source: str) -> int:
+        return len(cls.match_comments(source))
+
+    @classmethod
+    def remove_comments(cls, source_code: str, isList: bool = False):
+        segments, pos = [], 0
+        for start, end in cls._comment_spans(source_code):
+            if start > pos:
+                segments.append(source_code[pos:start])
+            pos = max(pos, end)
+        if pos < len(source_code):
+            segments.append(source_code[pos:])
+        return segments if isList else ''.join(segments)
+
+    @classmethod
+    def tokenize(cls, source: str) -> List[Token]:
+        newlines = _newline_offsets(source)
+        tokens: List[Token] = []
+        pos = 0
+        for start, end in cls._comment_spans(source):
+            if start > pos:
+                cls._tokenize_code(source[pos:start], pos, newlines, tokens)
+            if end > start:
+                tokens.append(Token(TokenType.COMMENT, source[start:end], start, end,
+                                    _line_at(newlines, start)))
+            pos = max(pos, end)
+        if pos < len(source):
+            cls._tokenize_code(source[pos:], pos, newlines, tokens)
+        return tokens
+
+
+# --------------------------------------------------------------- nested C-style
+def nested_cstyle_spans(source: str) -> List[Tuple[int, int]]:
+    """Comment spans for ``//`` line comments and *nested* ``/* */`` blocks,
+    skipping string/char/raw-string literals so a delimiter inside a string is not
+    mistaken for a comment."""
+    spans: List[Tuple[int, int]] = []
+    i, n = 0, len(source)
+    while i < n:
+        c = source[i]
+        if c in '"\'`':
+            quote = c
+            i += 1
+            while i < n:
+                if source[i] == '\\' and quote != '`':
+                    i += 2
+                    continue
+                if source[i] == quote:
+                    i += 1
+                    break
+                i += 1
+            continue
+        if c == '/' and i + 1 < n and source[i + 1] == '/':
+            start = i
+            i += 2
+            while i < n and source[i] != '\n':
+                i += 1
+            spans.append((start, i))
+            continue
+        if c == '/' and i + 1 < n and source[i + 1] == '*':
+            start = i
+            depth = 1
+            i += 2
+            while i < n and depth > 0:
+                if source[i] == '/' and i + 1 < n and source[i + 1] == '*':
+                    depth += 1
+                    i += 2
+                elif source[i] == '*' and i + 1 < n and source[i + 1] == '/':
+                    depth -= 1
+                    i += 2
+                else:
+                    i += 1
+            spans.append((start, i))
+            continue
+        i += 1
+    return spans
+
+
+class NestedCStyleLanguage(ScannedLanguage):
+    """C-style ``//`` and *nested* ``/* */`` comments (Odin, V, Jai, ...)."""
+
+    @classmethod
+    def _comment_spans(cls, source: str) -> List[Tuple[int, int]]:
+        return nested_cstyle_spans(source)

src/PyReprism/languages/cobol.py

@@ -0,0 +1,69 @@
+"""COBOL: a dedicated, column-sensitive comment scanner.
+
+COBOL comments do not fit any regex family, so this is hand-written on top of
+:class:`~PyReprism.languages._scanners.ScannedLanguage`:
+
+* **Fixed format:** a ``*`` (or ``/`` for a page eject) in the *indicator area*,
+  column 7 (0-based index 6), marks the entire line as a comment.
+* **Free format (COBOL 2002+):** ``*>`` begins an inline comment that runs to the
+  end of the line; ``'...'`` / ``"..."`` string literals are skipped so a ``*>``
+  inside a string is not treated as a comment.
+"""
+from typing import List, Optional, Tuple
+
+from ._scanners import ScannedLanguage
+from .registry import LanguageRegistry
+
+_INDICATOR_COL = 6  # column 7, 0-based
+
+
+def _inline_comment_start(text: str) -> Optional[int]:
+    """Index of a free-format ``*>`` comment on ``text`` (a single line without its
+    newline), skipping string literals, or ``None``."""
+    i, n = 0, len(text)
+    while i < n:
+        ch = text[i]
+        if ch in '"\'':
+            quote = ch
+            i += 1
+            while i < n and text[i] != quote:
+                i += 1
+            i += 1  # consume the closing quote (or run off the end)
+            continue
+        if ch == '*' and i + 1 < n and text[i + 1] == '>':
+            return i
+        i += 1
+    return None
+
+
+@LanguageRegistry.register
+class Cobol(ScannedLanguage):
+    @classmethod
+    def file_extension(cls) -> str:
+        return '.cob'
+
+    @classmethod
+    def keywords(cls) -> list:
+        return ('IDENTIFICATION|DIVISION|PROGRAM-ID|ENVIRONMENT|CONFIGURATION|'
+                'DATA|WORKING-STORAGE|LINKAGE|PROCEDURE|SECTION|PARAGRAPH|PIC|'
+                'PICTURE|VALUE|OCCURS|REDEFINES|MOVE|TO|FROM|ADD|SUBTRACT|'
+                'MULTIPLY|DIVIDE|COMPUTE|IF|ELSE|END-IF|EVALUATE|WHEN|PERFORM|'
+                'UNTIL|VARYING|CALL|USING|DISPLAY|ACCEPT|OPEN|CLOSE|READ|WRITE|'
+                'STOP|RUN|GOBACK|EXIT|GO|INITIALIZE|STRING|UNSTRING|INSPECT').split('|')
+
+    @classmethod
+    def _comment_spans(cls, source: str) -> List[Tuple[int, int]]:
+        spans: List[Tuple[int, int]] = []
+        pos = 0
+        for line in source.splitlines(keepends=True):
+            content = line.rstrip('\n\r')
+            content_end = pos + len(content)
+            if len(content) > _INDICATOR_COL and content[_INDICATOR_COL] in '*/':
+                # Whole line is a comment (leave the newline as code -> blank line).
+                spans.append((pos, content_end))
+            else:
+                idx = _inline_comment_start(content)
+                if idx is not None:
+                    spans.append((pos + idx, content_end))
+            pos += len(line)
+        return spans