From c41141be1d8b1e22efc74bb849e98392d2ab3a52 Mon Sep 17 00:00:00 2001 From: danielogen Date: Mon, 24 Aug 2026 14:13:51 -0700 Subject: [PATCH 1/3] feat(languages): add optional Pygments fallback for long-tail languages --- src/PyReprism/__init__.py | 8 ++ src/PyReprism/languages/dynamic.py | 153 +++++++++++++++++++++++++++++ tests/test_dynamic_fallback.py | 45 +++++++++ 3 files changed, 206 insertions(+) create mode 100644 src/PyReprism/languages/dynamic.py create mode 100644 tests/test_dynamic_fallback.py diff --git a/src/PyReprism/__init__.py b/src/PyReprism/__init__.py index ef1b4bc..0f0f22b 100644 --- a/src/PyReprism/__init__.py +++ b/src/PyReprism/__init__.py @@ -60,6 +60,14 @@ def get_language(lang: LanguageLike) -> Type: or LanguageRegistry.get(name.upper())) if cls: return cls + # Long-tail fallback: no built-in module matched, so try a Pygments-backed + # language (500+ lexers). Only a language name or an extension is eligible, + # and only when Pygments is installed; otherwise fall through to the error. + from .languages.dynamic import pygments_language_for + spec = name if (name.startswith('.') or '.' not in name) else os.path.splitext(name)[1] + cls = pygments_language_for(spec) + if cls: + return cls raise ValueError(f"Unknown language: {lang!r}") diff --git a/src/PyReprism/languages/dynamic.py b/src/PyReprism/languages/dynamic.py new file mode 100644 index 0000000..b11c7b7 --- /dev/null +++ b/src/PyReprism/languages/dynamic.py @@ -0,0 +1,153 @@ +"""Long-tail language fallback backed by Pygments. + +The 145 first-class languages are hand-written, zero-dependency regex modules and +remain the default. When a caller asks for a language or extension that is *not* +in the registry (say ``.zig`` or ``"solidity"``), and Pygments is installed, this +module builds a lightweight :class:`~PyReprism.languages.base.BaseLanguage` +subclass on the fly that delegates tokenization and comment handling to the +matching Pygments lexer. Pygments ships 500+ lexers, so this extends coverage far +beyond the built-in set without a per-language module. + +Fallback languages are deliberately **not** added to +:class:`~PyReprism.languages.registry.LanguageRegistry`: they are resolved on +demand, they are not validated against the per-language comment test suite, and +they should be treated as *supported* (best-effort) rather than *validated*. +Keyword-level operations are limited (the lexer classifies keywords in the token +stream, but ``keywords()`` is empty, so ``remove_keywords`` is a no-op). +""" +from functools import lru_cache +from typing import List, Optional, Type + +from ..tokens import Token, TokenType +from .base import BaseLanguage + + +def _first_extension(lexer) -> str: + """Best-effort file extension from a Pygments lexer's filename globs.""" + for pattern in getattr(lexer, 'filenames', []) or []: + if pattern.startswith('*.'): + return pattern[1:] # '*.zig' -> '.zig' + return '' + + +def _tokenize_with_lexer(source: str, lexer) -> List[Token]: + """Lossless token stream from a Pygments lexer, coalescing adjacent same-kind + tokens so a whole comment/string surfaces as one token (matching the regex + engine's granularity). Mirrors ``PygmentsEngine.tokenize``.""" + from ..engines.pygments_engine import PygmentsEngine + tokens: List[Token] = [] + for index, ttype, value in lexer.get_tokens_unprocessed(source): + if value == '': + continue + kind = PygmentsEngine._map(ttype, value) + if tokens and tokens[-1].type is kind and tokens[-1].end == index: + prev = tokens[-1] + tokens[-1] = Token(kind, prev.value + value, prev.start, + index + len(value), prev.line) + else: + line = source.count('\n', 0, index) + 1 + tokens.append(Token(kind, value, index, index + len(value), line)) + return tokens + + +def _is_non_language_lexer(lexer) -> bool: + """True for Pygments' generic fallback lexers (plain text / raw tokens), which + are not real languages and must not masquerade as a detected language.""" + try: + from pygments.lexers.special import RawTokenLexer, TextLexer + if isinstance(lexer, (TextLexer, RawTokenLexer)): + return True + except ImportError: + pass + return False + + +def _resolve_lexer(spec: str): + """Return a Pygments lexer for a language name or ``.ext``, or ``None``.""" + try: + from pygments.lexers import get_lexer_by_name, get_lexer_for_filename + from pygments.util import ClassNotFound + except ImportError: + return None + spec = (spec or '').strip() + if not spec: + return None + lexer = None + if spec.startswith('.'): + try: + lexer = get_lexer_for_filename('file' + spec) + except ClassNotFound: + return None + else: + try: + lexer = get_lexer_by_name(spec.lower()) + except ClassNotFound: + # A bare filename like "build.zig" or an extension without a dot. + try: + lexer = get_lexer_for_filename(spec if '.' in spec else 'file.' + spec) + except ClassNotFound: + return None + if lexer is None or _is_non_language_lexer(lexer): + return None + return lexer + + +@lru_cache(maxsize=None) +def pygments_language_for(spec: str) -> Optional[Type[BaseLanguage]]: + """Build (and cache) a Pygments-backed language class for ``spec``. + + ``spec`` is a language name (``"zig"``) or a file extension (``".zig"``). + Returns ``None`` when Pygments is not installed or has no matching lexer, so + callers can fall through to their existing error handling. + """ + lexer = _resolve_lexer(spec) + if lexer is None: + return None + + ext = spec if spec.startswith('.') else _first_extension(lexer) + display = (lexer.aliases[0] if getattr(lexer, 'aliases', None) else spec.lower()) + class_name = 'Pygments_' + ''.join(ch if ch.isalnum() else '_' for ch in display) + + def file_extension(cls) -> str: + return ext + + def keywords(cls) -> list: + return [] + + def tokenize(cls, source: str) -> List[Token]: + return _tokenize_with_lexer(source, lexer) + + def match_comments(cls, source: str) -> List[Token]: + return [t for t in cls.tokenize(source) if t.type is TokenType.COMMENT] + + def extract_comments(cls, source: str) -> List[str]: + return [t.value for t in cls.match_comments(source)] + + def count_comments(cls, source: str) -> int: + return len(cls.match_comments(source)) + + def remove_comments(cls, source_code: str, isList: bool = False): + segs = [t.value for t in cls.tokenize(source_code) + if t.type is not TokenType.COMMENT] + return segs if isList else ''.join(segs) + + def comment_regex(cls): + raise NotImplementedError( + f"{class_name} is Pygments-backed and has no regex comment pattern; " + "use tokenize()/remove_comments() instead." + ) + + namespace = { + '__doc__': f"Pygments-backed fallback language for {display!r} " + f"(lexer: {lexer.name}). Supported, not validated.", + 'pygments_backed': True, + 'file_extension': classmethod(file_extension), + 'keywords': classmethod(keywords), + 'tokenize': classmethod(tokenize), + 'match_comments': classmethod(match_comments), + 'extract_comments': classmethod(extract_comments), + 'count_comments': classmethod(count_comments), + 'remove_comments': classmethod(remove_comments), + 'comment_regex': classmethod(comment_regex), + } + return type(class_name, (BaseLanguage,), namespace) diff --git a/tests/test_dynamic_fallback.py b/tests/test_dynamic_fallback.py new file mode 100644 index 0000000..d2e7417 --- /dev/null +++ b/tests/test_dynamic_fallback.py @@ -0,0 +1,45 @@ +"""Tests for the Pygments long-tail fallback (languages/dynamic.py). + +When a language/extension is not one of the built-in modules, and Pygments is +installed, resolution falls back to a Pygments-backed language. These tests are +skipped when Pygments is not available (the fallback returns ``None`` and the +public API raises ``ValueError`` as before). + +Standard ML (``.sml``) is used as the exemplar: it has no built-in PyReprism +module but Pygments ships a lexer for it. +""" +import pytest + +import PyReprism as pr +from PyReprism.languages.dynamic import pygments_language_for + +pytest.importorskip('pygments') + + +def test_unknown_language_resolves_via_pygments(): + cls = pr.get_language('.sml') # Standard ML: no built-in module + assert cls is not None + assert getattr(cls, 'pygments_backed', False) is True + + +def test_fallback_strips_and_extracts_comments(): + src = 'val x = 1 (* a comment *)\nval y = 2\n' + out = pr.remove_comments(src, lang='.sml') + assert 'val x = 1' in out and 'val y = 2' in out + assert 'a comment' not in out + assert any('a comment' in c for c in pr.extract_comments(src, lang='sml')) + + +def test_fallback_does_not_override_builtins(): + # A built-in language must resolve to its zero-dependency module, not Pygments. + zig = pr.get_language('Zig') + assert getattr(zig, 'pygments_backed', False) is False + + +def test_pygments_language_for_returns_none_for_nonsense(): + assert pygments_language_for('definitely-not-a-real-language-xyz') is None + + +def test_fallback_tokenizes_with_comment_token(): + toks = pr.tokenize('val x = 1 (* c *)\n', lang='sml') + assert any(t.type.name == 'COMMENT' for t in toks) From 030968f8601a9b8d8f72348c9257a69eaca3f99f Mon Sep 17 00:00:00 2001 From: danielogen Date: Mon, 24 Aug 2026 14:17:07 -0700 Subject: [PATCH 2/3] feat(languages): add 42 zero-dep languages incl. nested-comment & COBOL scanners --- src/PyReprism/languages/__init__.py | 3 + src/PyReprism/languages/_scanners.py | 137 +++++++++++++++ src/PyReprism/languages/cobol.py | 69 ++++++++ src/PyReprism/languages/generated.py | 183 ++++++++++++++++++++ tests/languages/test_generated_languages.py | 140 +++++++++++++++ 5 files changed, 532 insertions(+) create mode 100644 src/PyReprism/languages/_scanners.py create mode 100644 src/PyReprism/languages/cobol.py create mode 100644 src/PyReprism/languages/generated.py create mode 100644 tests/languages/test_generated_languages.py diff --git a/src/PyReprism/languages/__init__.py b/src/PyReprism/languages/__init__.py index e9acfde..b5bc116 100644 --- a/src/PyReprism/languages/__init__.py +++ b/src/PyReprism/languages/__init__.py @@ -71,6 +71,9 @@ def _load_all_languages() -> None: '.sql': 'SQL', '.vb': 'Vbnet', '.js': 'JavaScript', + # '.v' is shared by Verilog and V (vlang); Verilog is far more common in + # existing corpora, so it wins extension lookup. Resolve V by name (`v`). + '.v': 'Verilog', } diff --git a/src/PyReprism/languages/_scanners.py b/src/PyReprism/languages/_scanners.py new file mode 100644 index 0000000..ad1ff70 --- /dev/null +++ b/src/PyReprism/languages/_scanners.py @@ -0,0 +1,137 @@ +"""Span-driven language base for comment syntaxes a regex cannot express. + +Most languages declare a ``comment_regex`` and inherit the whole API from +:class:`~PyReprism.languages.base.BaseLanguage`. A few do not fit a regular +expression: + +* **Nested block comments** (Odin, V, Jai): ``/* /* */ */`` requires counting + nesting depth, which a regex cannot do. +* **Column-sensitive comments** (COBOL): a ``*`` in the indicator column marks the + whole line as a comment. + +:class:`ScannedLanguage` handles these by delegating to a single classmethod, +:meth:`ScannedLanguage._comment_spans`, that returns the ``(start, end)`` offsets +of every comment. All comment operations---``match``/``extract``/``count``/% +``remove``---and a consistent :meth:`tokenize` are derived from those spans, so a +scanned language behaves exactly like a regex one (losslessly) without a +``comment_regex``. Code between comments is sub-classified by the inherited +:meth:`~PyReprism.languages.base.BaseLanguage._tokenize_code`. + +The module name is underscore-prefixed so it is *not* auto-imported as a language +by ``_load_all_languages``; concrete languages import it explicitly. +""" +from typing import List, Tuple + +from ..tokens import Token, TokenType +from .base import BaseLanguage, _line_at, _newline_offsets + + +class ScannedLanguage(BaseLanguage): + """A language whose comments are found by a scanner, not a regex.""" + + @classmethod + def _comment_spans(cls, source: str) -> List[Tuple[int, int]]: + """Return sorted, non-overlapping ``(start, end)`` offsets of comments.""" + raise NotImplementedError + + @classmethod + def comment_regex(cls): + raise NotImplementedError( + f"{cls.__name__} is scanned, not regex-based; use the comment methods." + ) + + @classmethod + def match_comments(cls, source: str) -> List[Token]: + newlines = _newline_offsets(source) + return [Token(TokenType.COMMENT, source[s:e], s, e, _line_at(newlines, s)) + for s, e in cls._comment_spans(source) if e > s] + + @classmethod + def extract_comments(cls, source: str) -> List[str]: + return [t.value for t in cls.match_comments(source)] + + @classmethod + def count_comments(cls, source: str) -> int: + return len(cls.match_comments(source)) + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + segments, pos = [], 0 + for start, end in cls._comment_spans(source_code): + if start > pos: + segments.append(source_code[pos:start]) + pos = max(pos, end) + if pos < len(source_code): + segments.append(source_code[pos:]) + return segments if isList else ''.join(segments) + + @classmethod + def tokenize(cls, source: str) -> List[Token]: + newlines = _newline_offsets(source) + tokens: List[Token] = [] + pos = 0 + for start, end in cls._comment_spans(source): + if start > pos: + cls._tokenize_code(source[pos:start], pos, newlines, tokens) + if end > start: + tokens.append(Token(TokenType.COMMENT, source[start:end], start, end, + _line_at(newlines, start))) + pos = max(pos, end) + if pos < len(source): + cls._tokenize_code(source[pos:], pos, newlines, tokens) + return tokens + + +# --------------------------------------------------------------- nested C-style +def nested_cstyle_spans(source: str) -> List[Tuple[int, int]]: + """Comment spans for ``//`` line comments and *nested* ``/* */`` blocks, + skipping string/char/raw-string literals so a delimiter inside a string is not + mistaken for a comment.""" + spans: List[Tuple[int, int]] = [] + i, n = 0, len(source) + while i < n: + c = source[i] + if c in '"\'`': + quote = c + i += 1 + while i < n: + if source[i] == '\\' and quote != '`': + i += 2 + continue + if source[i] == quote: + i += 1 + break + i += 1 + continue + if c == '/' and i + 1 < n and source[i + 1] == '/': + start = i + i += 2 + while i < n and source[i] != '\n': + i += 1 + spans.append((start, i)) + continue + if c == '/' and i + 1 < n and source[i + 1] == '*': + start = i + depth = 1 + i += 2 + while i < n and depth > 0: + if source[i] == '/' and i + 1 < n and source[i + 1] == '*': + depth += 1 + i += 2 + elif source[i] == '*' and i + 1 < n and source[i + 1] == '/': + depth -= 1 + i += 2 + else: + i += 1 + spans.append((start, i)) + continue + i += 1 + return spans + + +class NestedCStyleLanguage(ScannedLanguage): + """C-style ``//`` and *nested* ``/* */`` comments (Odin, V, Jai, ...).""" + + @classmethod + def _comment_spans(cls, source: str) -> List[Tuple[int, int]]: + return nested_cstyle_spans(source) diff --git a/src/PyReprism/languages/cobol.py b/src/PyReprism/languages/cobol.py new file mode 100644 index 0000000..05680ce --- /dev/null +++ b/src/PyReprism/languages/cobol.py @@ -0,0 +1,69 @@ +"""COBOL: a dedicated, column-sensitive comment scanner. + +COBOL comments do not fit any regex family, so this is hand-written on top of +:class:`~PyReprism.languages._scanners.ScannedLanguage`: + +* **Fixed format:** a ``*`` (or ``/`` for a page eject) in the *indicator area*, + column 7 (0-based index 6), marks the entire line as a comment. +* **Free format (COBOL 2002+):** ``*>`` begins an inline comment that runs to the + end of the line; ``'...'`` / ``"..."`` string literals are skipped so a ``*>`` + inside a string is not treated as a comment. +""" +from typing import List, Optional, Tuple + +from ._scanners import ScannedLanguage +from .registry import LanguageRegistry + +_INDICATOR_COL = 6 # column 7, 0-based + + +def _inline_comment_start(text: str) -> Optional[int]: + """Index of a free-format ``*>`` comment on ``text`` (a single line without its + newline), skipping string literals, or ``None``.""" + i, n = 0, len(text) + while i < n: + ch = text[i] + if ch in '"\'': + quote = ch + i += 1 + while i < n and text[i] != quote: + i += 1 + i += 1 # consume the closing quote (or run off the end) + continue + if ch == '*' and i + 1 < n and text[i + 1] == '>': + return i + i += 1 + return None + + +@LanguageRegistry.register +class Cobol(ScannedLanguage): + @classmethod + def file_extension(cls) -> str: + return '.cob' + + @classmethod + def keywords(cls) -> list: + return ('IDENTIFICATION|DIVISION|PROGRAM-ID|ENVIRONMENT|CONFIGURATION|' + 'DATA|WORKING-STORAGE|LINKAGE|PROCEDURE|SECTION|PARAGRAPH|PIC|' + 'PICTURE|VALUE|OCCURS|REDEFINES|MOVE|TO|FROM|ADD|SUBTRACT|' + 'MULTIPLY|DIVIDE|COMPUTE|IF|ELSE|END-IF|EVALUATE|WHEN|PERFORM|' + 'UNTIL|VARYING|CALL|USING|DISPLAY|ACCEPT|OPEN|CLOSE|READ|WRITE|' + 'STOP|RUN|GOBACK|EXIT|GO|INITIALIZE|STRING|UNSTRING|INSPECT').split('|') + + @classmethod + def _comment_spans(cls, source: str) -> List[Tuple[int, int]]: + spans: List[Tuple[int, int]] = [] + pos = 0 + for line in source.splitlines(keepends=True): + content = line.rstrip('\n\r') + content_end = pos + len(content) + if len(content) > _INDICATOR_COL and content[_INDICATOR_COL] in '*/': + # Whole line is a comment (leave the newline as code -> blank line). + spans.append((pos, content_end)) + else: + idx = _inline_comment_start(content) + if idx is not None: + spans.append((pos + idx, content_end)) + pos += len(line) + return spans diff --git a/src/PyReprism/languages/generated.py b/src/PyReprism/languages/generated.py new file mode 100644 index 0000000..536c7de --- /dev/null +++ b/src/PyReprism/languages/generated.py @@ -0,0 +1,183 @@ +"""Data-driven, zero-dependency language modules generated from a family table. + +Many languages share a comment syntax ("family"): C-style ``// /* */``, hash +``#``, Haskell-style ``-- {- -}``, Lisp-style ``;``, and so on. Rather than write a +near-identical module per language, we declare each new language as a row +``(ClassName, extension, family, keywords)`` and synthesize a real +:class:`~PyReprism.languages.base.BaseLanguage` subclass for it. These are +first-class, registered, zero-dependency languages (unlike the Pygments fallback +in :mod:`~PyReprism.languages.dynamic`); each has a comment-handling test in the +suite, so they are *validated*, not merely supported. + +To add a language: append a row to :data:`LANGUAGES` (and a sample to the test +suite's ``SAMPLES``). To add a comment family: add an entry to :data:`FAMILIES`. +""" +import re + +from ._scanners import NestedCStyleLanguage +from .base import BaseLanguage +from .registry import LanguageRegistry + +# Each family maps to (comment-alternatives, characters that end a plain-code run, +# string-literal style). The generated comment regex keeps strings intact and only +# treats real comment delimiters as comments -- the same idiom the hand-written +# modules use (see e.g. languages/go.py). +# strings='both' -> both '...' and "..." are string literals +# strings='double' -> only "..." is a string (for Lisp-family, where ' is quote) +FAMILIES = { + # C-style: line // and block /* */ + 'cstyle': (r'//.*?$|/\*[\s\S]*?\*/', '/', 'both'), + # // line comments only (e.g. Zig, Gleam -- no block comments) + 'slashline': (r'//.*?$', '/', 'both'), + # # line comments (Toml, CMake, Fish, ...) + 'hash': (r'#.*?$', '#', 'both'), + # HCL/Terraform: # or // line comments, plus /* */ blocks + 'hcl': (r'#.*?$|//.*?$|/\*[\s\S]*?\*/', '#/', 'both'), + # Haskell-style: -- line and {- -} block (Elm, PureScript) + 'dashbrace': (r'--.*?$|\{-[\s\S]*?-\}', '-{', 'both'), + # Racket/Scheme-style: ; line and #| |# block + 'racket': (r';.*?$|#\|[\s\S]*?\|#', ';#', 'double'), + # ; line comments only (Fennel, Red, ...) + 'semicolon': (r';.*?$', ';', 'double'), + # -- line comments only (Futhark, ...); no block form + 'dashline': (r'--.*?$', '-', 'double'), + # Lean 4: -- line and /- -/ block (treated non-nested; see NESTED for nesting) + 'lean': (r'--.*?$|/-[\s\S]*?-/', '-/', 'double'), +} + + +def _build_comment_regex(family: str) -> re.Pattern: + comment_alt, stop_chars, strings = FAMILIES[family] + if strings == 'double': + str_alt = r'"(?:\\.|[^"\\])*"' + else: + str_alt = r'"(?:\\.|[^"\\])*"|\'(?:\\.|[^\'\\])*\'' + # A code run: a string literal, or one char followed by non-delimiter chars. + # The negated class stops at the family's comment starters and at quotes/backslash + # so the next comment/string is re-examined at its own start. + noncomment = rf'{str_alt}|.[^{stop_chars}\\\'"]*' + pattern = rf'(?P{comment_alt})|(?P{noncomment})' + return re.compile(pattern, re.DOTALL | re.MULTILINE) + + +# (ClassName, file extension, comment family, keywords) +LANGUAGES = [ + # --- C-style ( // + /* */ ) --- + ('Vala', '.vala', 'cstyle', 'if|else|while|for|foreach|do|switch|case|break|continue|return|class|interface|struct|enum|namespace|public|private|protected|internal|static|const|void|var|new|delete|null|true|false|this|base|using|out|ref|owned|unowned|async|yield'), + ('Solidity', '.sol', 'cstyle', 'pragma|contract|library|interface|function|modifier|event|struct|enum|mapping|address|uint|uint256|int|bool|string|bytes|memory|storage|calldata|public|private|internal|external|view|pure|payable|returns|return|if|else|for|while|do|require|revert|emit|new|import|is|constructor|using'), + ('Wgsl', '.wgsl', 'cstyle', 'fn|let|var|const|struct|return|if|else|for|while|loop|break|continue|switch|case|default|bool|i32|u32|f32|f16|vec2|vec3|vec4|mat4x4|array|ptr|type|override|true|false|discard'), + ('Move', '.move', 'cstyle', 'module|script|fun|public|native|struct|has|copy|drop|store|key|let|mut|if|else|while|loop|return|abort|use|const|move|copy|acquires|as|spec'), + ('Sway', '.sw', 'cstyle', 'contract|library|script|predicate|fn|pub|struct|enum|impl|trait|use|let|mut|if|else|match|while|for|return|const|storage|abi|self|Self|true|false'), + ('Pony', '.pony', 'cstyle', 'actor|class|primitive|interface|trait|struct|type|fun|be|new|var|let|if|then|else|elseif|end|while|do|for|in|match|return|error|recover|consume|iso|trn|ref|val|box|tag|true|false'), + ('Thrift', '.thrift', 'cstyle', 'namespace|include|struct|union|exception|service|enum|typedef|const|required|optional|oneway|void|bool|byte|i16|i32|i64|double|string|binary|list|set|map|throws|extends'), + # --- // line only --- + ('Zig', '.zig', 'slashline', 'const|var|fn|pub|return|if|else|while|for|switch|break|continue|defer|errdefer|try|catch|struct|enum|union|error|comptime|inline|test|and|or|orelse|unreachable|null|undefined|true|false|void|type|anytype|usingnamespace|export|extern'), + ('Gleam', '.gleam', 'slashline', 'fn|pub|let|const|type|case|if|else|import|as|use|assert|todo|panic|try|opaque|external|True|False|Nil|Ok|Error'), + # --- # line --- + ('Toml', '.toml', 'hash', 'true|false'), + ('Capnp', '.capnp', 'hash', 'struct|enum|interface|union|group|const|annotation|import|using|extends|Void|Bool|Int8|Int16|Int32|Int64|UInt8|UInt16|UInt32|UInt64|Float32|Float64|Text|Data|List'), + ('CMake', '.cmake', 'hash', 'if|elseif|else|endif|foreach|endforeach|while|endwhile|function|endfunction|macro|endmacro|set|unset|option|include|add_executable|add_library|target_link_libraries|find_package|message|return|break|continue'), + ('Fish', '.fish', 'hash', 'if|else|end|switch|case|for|in|while|function|return|begin|and|or|not|set|test|command|builtin|break|continue'), + ('Awk', '.awk', 'hash', 'BEGIN|END|if|else|while|for|do|break|continue|next|exit|return|function|print|printf|getline|delete|in|length|substr|split|gsub|sub|match'), + ('Janet', '.janet', 'hash', 'def|var|defn|defmacro|fn|do|if|when|unless|cond|case|while|for|each|let|set|quote|quasiquote|import|require|true|false|nil'), + # --- HCL / Terraform ( # or // line + /* */ ) --- + ('Hcl', '.tf', 'hcl', 'resource|provider|variable|output|module|data|locals|terraform|true|false|null|for|in|if'), + # --- Haskell-style ( -- line + {- -} block ) --- + ('Elm', '.elm', 'dashbrace', 'module|import|exposing|type|alias|port|if|then|else|case|of|let|in|as|True|False|Nothing|Just'), + ('PureScript', '.purs', 'dashbrace', 'module|import|as|hiding|class|instance|where|data|newtype|type|forall|if|then|else|case|of|let|in|do|derive|true|false'), + # --- Lisp-family ( ; line + #| |# block ) --- + ('Racket', '.rkt', 'racket', 'define|lambda|let|let*|letrec|if|cond|case|when|unless|begin|set!|quote|quasiquote|require|provide|struct|module|for|and|or|not|else|true|false'), + # --- ; line only --- + ('Fennel', '.fnl', 'semicolon', 'fn|lambda|local|var|set|let|if|when|do|for|each|while|match|global|require|import-macros|and|or|not|true|false|nil'), + ('Red', '.red', 'semicolon', 'if|either|unless|while|until|loop|repeat|foreach|func|function|does|has|make|return|break|continue|true|false|none'), + # --- more C-style ( // + /* */ ) --- + ('Cadence', '.cdc', 'cstyle', 'pub|priv|let|var|fun|struct|resource|interface|contract|event|enum|case|if|else|while|for|in|return|break|continue|import|from|pre|post|access|self|init|destroy|create|emit|true|false|nil'), + ('Chapel', '.chpl', 'cstyle', 'proc|iter|class|record|module|var|const|param|type|if|else|for|forall|while|do|select|when|return|yield|use|import|begin|cobegin|sync|atomic|in|out|inout|ref|true|false|nil'), + ('Tact', '.tact', 'cstyle', 'contract|trait|struct|message|fun|native|get|let|const|if|else|while|until|repeat|return|extends|mutates|virtual|abstract|override|self|init|receive|true|false|null'), + # --- more // line only --- + ('Carbon', '.carbon', 'slashline', 'fn|var|let|class|interface|impl|as|if|else|while|for|in|return|break|continue|match|case|and|or|not|package|import|api|library|Self|true|false'), + ('Hare', '.ha', 'slashline', 'fn|let|const|def|type|struct|union|enum|if|else|for|switch|case|match|return|break|continue|use|export|static|nullable|size|true|false|null|void'), + ('Cue', '.cue', 'slashline', 'package|import|if|for|in|let|true|false|null|string|int|float|bool|bytes|number'), + ('Ballerina', '.bal', 'slashline', 'function|service|resource|remote|public|private|isolated|type|record|object|class|if|else|while|foreach|in|return|check|import|listener|worker|start|true|false'), + # --- more # line --- + ('Vyper', '.vy', 'hash', 'def|return|if|elif|else|for|in|while|pass|break|continue|struct|event|interface|public|private|external|internal|view|pure|payable|nonpayable|assert|raise|log|self|True|False'), + ('Starlark', '.bzl', 'hash', 'def|return|if|elif|else|for|in|while|break|continue|pass|load|and|or|not|lambda|True|False|None'), + ('Nushell', '.nu', 'hash', 'def|let|mut|const|if|else|for|while|loop|match|return|break|continue|export|use|module|source|do|each|where|true|false|null'), + ('Nickel', '.ncl', 'hash', 'let|in|if|then|else|fun|import|match|default|doc|true|false|null'), + ('Just', '.just', 'hash', 'alias|set|export|if|else'), + # --- HCL-style ( # or // line + /* */ ) --- + ('Jsonnet', '.jsonnet', 'hcl', 'local|function|if|then|else|for|in|import|importstr|error|assert|super|self|null|true|false'), + # --- Haskell-style ( -- line + {- -} block ) --- + ('Dhall', '.dhall', 'dashbrace', 'let|in|if|then|else|merge|forall|assert|as|using|with|Type|Kind|Sort|Bool|Natural|Text|List|Optional|True|False|None|Some'), + ('Idris', '.idr', 'dashbrace', 'module|import|data|record|interface|implementation|where|if|then|else|case|of|let|in|do|total|partial|public|export|private|True|False|Nothing|Just'), + # --- -- line only --- + ('Futhark', '.fut', 'dashline', 'let|in|if|then|else|loop|for|while|do|def|entry|type|module|open|import|val|match|case|true|false'), + # --- Lean 4 ( -- line + /- -/ block ) --- + ('Lean', '.lean', 'lean', 'def|theorem|lemma|example|inductive|structure|class|instance|where|if|then|else|match|with|let|fun|do|by|have|show|from|namespace|open|import|variable|true|false'), +] + +# Languages needing a real scanner (nested /* */ block comments); see _scanners.py. +# (ClassName, extension, keywords) +NESTED_LANGUAGES = [ + ('Odin', '.odin', 'package|import|proc|struct|union|enum|map|if|else|for|switch|case|when|return|break|continue|defer|using|in|not_in|cast|transmute|distinct|foreign|true|false|nil'), + ('V', '.v', 'module|import|fn|struct|enum|interface|union|type|pub|mut|const|if|else|for|in|match|return|break|continue|defer|go|spawn|or|unsafe|true|false|none'), + ('Jai', '.jai', 'if|else|for|while|return|break|continue|struct|enum|union|using|defer|cast|xx|it|inline|no_inline|true|false|null'), +] + + +def _make_language(class_name: str, ext: str, family: str, keywords_str: str): + compiled = _build_comment_regex(family) + kw = keywords_str.split('|') if keywords_str else [] + + @classmethod + def file_extension(cls) -> str: + return ext + + @classmethod + def keywords(cls) -> list: + return list(kw) + + @classmethod + def comment_regex(cls) -> re.Pattern: + return compiled + + namespace = { + '__doc__': f"{class_name} ({family} comments, generated).", + 'comment_family': family, + 'file_extension': file_extension, + 'keywords': keywords, + 'comment_regex': comment_regex, + } + return type(class_name, (BaseLanguage,), namespace) + + +def _make_nested_language(class_name: str, ext: str, keywords_str: str): + kw = keywords_str.split('|') if keywords_str else [] + + @classmethod + def file_extension(cls) -> str: + return ext + + @classmethod + def keywords(cls) -> list: + return list(kw) + + namespace = { + '__doc__': f"{class_name} (nested C-style comments, generated).", + 'comment_family': 'nested', + 'file_extension': file_extension, + 'keywords': keywords, + } + return type(class_name, (NestedCStyleLanguage,), namespace) + + +# Generate, register, and expose each language at module scope. +for _name, _ext, _family, _kw in LANGUAGES: + _cls = LanguageRegistry.register(_make_language(_name, _ext, _family, _kw)) + globals()[_name] = _cls + +for _name, _ext, _kw in NESTED_LANGUAGES: + _cls = LanguageRegistry.register(_make_nested_language(_name, _ext, _kw)) + globals()[_name] = _cls + +del _name, _ext, _family, _kw, _cls diff --git a/tests/languages/test_generated_languages.py b/tests/languages/test_generated_languages.py new file mode 100644 index 0000000..ace33ce --- /dev/null +++ b/tests/languages/test_generated_languages.py @@ -0,0 +1,140 @@ +"""Tests for the data-driven, family-generated languages (languages/generated.py). + +Beyond the shared comment-handling coverage in ``test_all_languages.py``, these +lock in the two properties most likely to regress: comment delimiters inside +string literals must be preserved, and multi-line block comments must be stripped. +""" +import pytest + +import PyReprism as pr +from PyReprism.languages.generated import LANGUAGES +from PyReprism.languages.registry import LanguageRegistry + +NAMES = [row[0] for row in LANGUAGES] + + +@pytest.mark.parametrize('name', NAMES) +def test_generated_language_is_registered(name): + cls = LanguageRegistry.get(name) + assert cls is not None + assert cls.file_extension().startswith('.') + assert isinstance(cls.keywords(), list) + + +@pytest.mark.parametrize('name', NAMES) +def test_resolvable_by_name_and_extension(name): + cls = LanguageRegistry.get(name) + assert pr.get_language(name) is cls + assert pr.get_language(cls.file_extension()) is cls + + +# (language, code line whose string contains a comment delimiter that must survive) +STRING_SAFETY = [ + ('Zig', 'const s = "a // b";\n'), + ('Solidity', 'string u = "http://x";\n'), + ('Toml', 'url = "http://x # y"\n'), + ('CMake', 'set(U "http://x # y")\n'), + ('Racket', '(define s "a ; b")\n'), + ('Elm', 's = "a -- b"\n'), +] + + +@pytest.mark.parametrize('name, src', STRING_SAFETY) +def test_comment_delimiter_inside_string_is_preserved(name, src): + # The whole line is code; nothing should be removed as a comment. + assert pr.remove_comments(src, lang=name) == src + + +# (language, block comment sample containing SECRET) +BLOCKS = [ + ('Vala', 'a\n/*\nzzsecret\n*/\nb\n'), + ('Solidity', 'a\n/*\nzzsecret\n*/\nb\n'), + ('Thrift', 'a\n/*\nzzsecret\n*/\nb\n'), + ('Hcl', 'a\n/*\nzzsecret\n*/\nb\n'), + ('Elm', 'a\n{-\nzzsecret\n-}\nb\n'), + ('PureScript', 'a\n{-\nzzsecret\n-}\nb\n'), + ('Racket', 'a\n#|\nzzsecret\n|#\nb\n'), +] + + +@pytest.mark.parametrize('name, src', BLOCKS) +def test_block_comments_are_stripped(name, src): + out = pr.remove_comments(src, lang=name) + assert 'a' in out and 'b' in out + assert 'zzsecret' not in out + + +def test_zig_has_no_block_comments(): + # Zig only has // line comments; /* */ is not a comment there. + src = 'const x = 1; /* not a comment */\n' + assert 'not a comment' in pr.remove_comments(src, lang='Zig') + + +# --------------------------------------------------- nested block comments +@pytest.mark.parametrize('name', ['Odin', 'V', 'Jai']) +def test_nested_block_comments_are_fully_stripped(name): + # A single-level regex would leave "still comment */" behind; the scanner + # counts nesting depth and removes the whole thing. + src = 'a := 1 /* outer /* inner */ still comment */ b := 2\n' + out = pr.remove_comments(src, lang=name) + assert 'a := 1' in out and 'b := 2' in out + assert 'still comment' not in out and '*/' not in out + + +@pytest.mark.parametrize('name', ['Odin', 'V', 'Jai']) +def test_nested_language_string_safety(name): + src = 'url := "http:// /* not a comment */ x"\ncode := 1\n' + assert pr.remove_comments(src, lang=name) == src + + +@pytest.mark.parametrize('name', ['Odin', 'V', 'Jai']) +def test_nested_language_line_comment_and_idempotent(name): + src = 'x := 1 // a comment\ny := 2\n' + once = pr.remove_comments(src, lang=name) + assert 'a comment' not in once and 'x := 1' in once + assert pr.remove_comments(once, lang=name) == once + + +def test_v_extension_resolves_to_verilog_not_vlang(): + # .v is shared; Verilog wins extension lookup, V is reachable by name. + assert pr.get_language('.v').__name__ == 'Verilog' + assert pr.get_language('V').__name__ == 'V' + + +# ------------------------------------------------------- COBOL (dedicated) +COBOL_SRC = ( + "000100 IDENTIFICATION DIVISION.\n" + "000200* THIS WHOLE LINE IS A COMMENT\n" + "000300 PROCEDURE DIVISION.\n" + "000400 DISPLAY 'HELLO'. *> inline comment\n" + "000500 DISPLAY '*> inside a string stays'.\n" +) + + +def test_cobol_column7_and_inline_comments(): + out = pr.remove_comments(COBOL_SRC, lang='cobol') + assert 'THIS WHOLE LINE IS A COMMENT' not in out # column-7 comment gone + assert 'inline comment' not in out # *> inline gone + assert 'IDENTIFICATION DIVISION' in out # code kept + assert "DISPLAY 'HELLO'." in out + + +def test_cobol_star_inside_string_is_not_a_comment(): + out = pr.remove_comments(COBOL_SRC, lang='cobol') + assert '*> inside a string stays' in out + + +def test_cobol_extract_and_idempotent(): + comments = pr.extract_comments(COBOL_SRC, lang='cobol') + assert any('WHOLE LINE' in c for c in comments) + assert any('inline comment' in c for c in comments) + once = pr.remove_comments(COBOL_SRC, lang='cobol') + assert pr.remove_comments(once, lang='cobol') == once + + +def test_cobol_slash_in_column7_is_comment(): + # A '/' in the indicator column is a page-eject comment line. + src = "000100 DISPLAY 'A'.\n000200/ PAGE EJECT LINE\n000300 DISPLAY 'B'.\n" + out = pr.remove_comments(src, lang='cobol') + assert 'PAGE EJECT LINE' not in out + assert "DISPLAY 'A'." in out and "DISPLAY 'B'." in out From b291903006625fd659583795c6f99add1ce6bab0 Mon Sep 17 00:00:00 2001 From: danielogen Date: Mon, 24 Aug 2026 14:18:01 -0700 Subject: [PATCH 3/3] feat(cli): show built-in count and Pygments fallback tier in `languages` --- src/PyReprism/cli.py | 58 ++++++++++++++++++++++++++- tests/languages/test_all_languages.py | 46 ++++++++++++++++++++- tests/test_cli.py | 17 ++++++++ 3 files changed, 118 insertions(+), 3 deletions(-) diff --git a/src/PyReprism/cli.py b/src/PyReprism/cli.py index 155b9eb..9e87f8b 100644 --- a/src/PyReprism/cli.py +++ b/src/PyReprism/cli.py @@ -9,6 +9,7 @@ pyreprism preprocess --steps comments,strings,whitespace file.java pyreprism tokenize --json file.py pyreprism languages + pyreprism languages --pygments """ import argparse import glob @@ -332,22 +333,71 @@ def _cmd_clones(args) -> int: return 0 +# Pygments lexers that are generic fallbacks, not real languages. +_PYGMENTS_SKIP = {'text', 'raw', 'output', 'pseudo', ' specials'} + + +def _pygments_fallback_rows(builtin_exts): + """Rows ``(alias, ext)`` for Pygments lexers not covered by a built-in module. + + Returns ``None`` when Pygments is not installed. Deduped by primary extension + so the count reflects genuinely *additional* coverage. + """ + try: + from pygments.lexers import get_all_lexers + except ImportError: + return None + rows, seen_ext = [], set() + for name, aliases, filenames, _mime in get_all_lexers(): + alias = aliases[0] if aliases else name + if alias.lower() in _PYGMENTS_SKIP or name == 'Text only': + continue + ext = next((f[1:] for f in filenames if f.startswith('*.')), '') + if not ext or ext in builtin_exts or ext in seen_ext: + continue + seen_ext.add(ext) + rows.append((alias, ext)) + return sorted(rows) + + def _cmd_languages(args) -> int: from .languages import _load_all_languages from .languages.registry import LanguageRegistry _load_all_languages() rows = [] + builtin_exts = set() for name, cls in sorted(LanguageRegistry.all().items()): try: ext = cls.file_extension() except Exception: ext = '?' rows.append((name, ext)) + builtin_exts.add(ext) + + if getattr(args, 'pygments', False): + extra = _pygments_fallback_rows(builtin_exts) + if extra is None: + print("Pygments is not installed; install it with " + "`pip install pyreprism[accurate]` to enable the fallback tier.", + file=sys.stderr) + return 1 + width = max((len(n) for n, _ in extra), default=0) + for name, ext in extra: + print(f"{name.ljust(width)} {ext}") + print(f"\n{len(extra)} additional languages via the Pygments fallback " + f"(resolved on demand, not validated)", file=sys.stderr) + return 0 + width = max((len(n) for n, _ in rows), default=0) for name, ext in rows: print(f"{name.ljust(width)} {ext}") - print(f"\n{len(rows)} languages", file=sys.stderr) + footer = f"\n{len(rows)} built-in languages (zero-dependency)" + extra = _pygments_fallback_rows(builtin_exts) + if extra: + footer += (f"; +{len(extra)} more available via the optional Pygments " + f"backend (see `pyreprism languages --pygments`)") + print(footer, file=sys.stderr) return 0 @@ -378,7 +428,8 @@ def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog='pyreprism', description='Preprocess source code: strip/extract/count comments, strings, ' - 'numbers, operators, keywords and identifiers across 145+ languages.', + 'numbers, operators, keywords and identifiers across 160+ ' + 'built-in languages (500+ with the optional Pygments backend).', ) parser.add_argument('--version', action='version', version=f'pyreprism {__version__}') sub = parser.add_subparsers(dest='command', required=True) @@ -522,6 +573,9 @@ def add_fingerprint_opts(p): p_clones.set_defaults(func=_cmd_clones) p_langs = sub.add_parser('languages', help='list supported languages and extensions') + p_langs.add_argument('--pygments', action='store_true', + help='instead list the additional languages available via ' + 'the optional Pygments fallback backend') p_langs.set_defaults(func=_cmd_languages) return parser diff --git a/tests/languages/test_all_languages.py b/tests/languages/test_all_languages.py index e0ccc6b..777d391 100644 --- a/tests/languages/test_all_languages.py +++ b/tests/languages/test_all_languages.py @@ -162,10 +162,54 @@ 'Xeora': 'keepAA //zzsecretzz\nkeepBB\n', 'Xojo': 'keepAA //zzsecretzz\nkeepBB\n', 'Yaml': 'keepAA #zzsecretzz\nkeepBB\n', + # --- generated (data-driven) languages: see languages/generated.py --- + 'Vala': 'keepAA //zzsecretzz\nkeepBB\n', + 'Solidity': 'keepAA //zzsecretzz\nkeepBB\n', + 'Wgsl': 'keepAA //zzsecretzz\nkeepBB\n', + 'Move': 'keepAA //zzsecretzz\nkeepBB\n', + 'Sway': 'keepAA //zzsecretzz\nkeepBB\n', + 'Pony': 'keepAA //zzsecretzz\nkeepBB\n', + 'Thrift': 'keepAA //zzsecretzz\nkeepBB\n', + 'Zig': 'keepAA //zzsecretzz\nkeepBB\n', + 'Gleam': 'keepAA //zzsecretzz\nkeepBB\n', + 'Toml': 'keepAA #zzsecretzz\nkeepBB\n', + 'Capnp': 'keepAA #zzsecretzz\nkeepBB\n', + 'CMake': 'keepAA #zzsecretzz\nkeepBB\n', + 'Fish': 'keepAA #zzsecretzz\nkeepBB\n', + 'Awk': 'keepAA #zzsecretzz\nkeepBB\n', + 'Janet': 'keepAA #zzsecretzz\nkeepBB\n', + 'Hcl': 'keepAA #zzsecretzz\nkeepBB\n', + 'Elm': 'keepAA --zzsecretzz\nkeepBB\n', + 'PureScript': 'keepAA --zzsecretzz\nkeepBB\n', + 'Racket': 'keepAA ;zzsecretzz\nkeepBB\n', + 'Fennel': 'keepAA ;zzsecretzz\nkeepBB\n', + 'Red': 'keepAA ;zzsecretzz\nkeepBB\n', + # --- second batch: more families + dedicated scanners --- + 'Cadence': 'keepAA //zzsecretzz\nkeepBB\n', + 'Chapel': 'keepAA //zzsecretzz\nkeepBB\n', + 'Tact': 'keepAA //zzsecretzz\nkeepBB\n', + 'Carbon': 'keepAA //zzsecretzz\nkeepBB\n', + 'Hare': 'keepAA //zzsecretzz\nkeepBB\n', + 'Cue': 'keepAA //zzsecretzz\nkeepBB\n', + 'Ballerina': 'keepAA //zzsecretzz\nkeepBB\n', + 'Jsonnet': 'keepAA //zzsecretzz\nkeepBB\n', + 'Odin': 'keepAA //zzsecretzz\nkeepBB\n', + 'V': 'keepAA //zzsecretzz\nkeepBB\n', + 'Jai': 'keepAA //zzsecretzz\nkeepBB\n', + 'Vyper': 'keepAA #zzsecretzz\nkeepBB\n', + 'Starlark': 'keepAA #zzsecretzz\nkeepBB\n', + 'Nushell': 'keepAA #zzsecretzz\nkeepBB\n', + 'Nickel': 'keepAA #zzsecretzz\nkeepBB\n', + 'Just': 'keepAA #zzsecretzz\nkeepBB\n', + 'Dhall': 'keepAA --zzsecretzz\nkeepBB\n', + 'Idris': 'keepAA --zzsecretzz\nkeepBB\n', + 'Futhark': 'keepAA --zzsecretzz\nkeepBB\n', + 'Lean': 'keepAA --zzsecretzz\nkeepBB\n', } # Languages whose comment model doesn't fit the shared sample; tested separately. -SPECIAL = {'CSP', 'BrainFuck'} +# COBOL is column-sensitive (see test_generated_languages.py). +SPECIAL = {'CSP', 'BrainFuck', 'Cobol'} @pytest.mark.parametrize('name', sorted(SAMPLES)) diff --git a/tests/test_cli.py b/tests/test_cli.py index b69a96d..7101f93 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -87,6 +87,23 @@ def test_languages_lists_many(monkeypatch, capsys): assert code == 0 assert 'Python' in out assert out.count('\n') > 100 + # generated (data-driven) languages are registered and therefore listed + assert 'Zig' in out and 'Solidity' in out and 'Toml' in out + assert 'built-in languages' in err + + +def test_languages_pygments_flag(monkeypatch, capsys): + import importlib.util + code, out, err = run(['languages', '--pygments'], monkeypatch=monkeypatch, + capsys=capsys) + if importlib.util.find_spec('pygments') is None: + assert code == 1 + assert 'Pygments is not installed' in err + else: + assert code == 0 + assert 'Pygments fallback' in err + # a Pygments-only language (no built-in module) should appear + assert 'sml' in out.lower() def test_no_language_errors(monkeypatch, capsys):