diff --git a/README.md b/README.md index fa86999..93617b5 100644 --- a/README.md +++ b/README.md @@ -106,13 +106,31 @@ True ' Body texts...' ``` +### Read named tables + +Tables in `node.body_rich` expose their `#+NAME:` through `Table.name`. +The name is `None` for unnamed tables. + +``` pycon +>>> from orgparse.extra import Table +>>> root = loads(''' +... #+NAME: measurements +... | x | y | +... |---+---| +... | 1 | 2 | +... ''') +>>> [table] = [part for part in root.body_rich if isinstance(part, Table) and part.name == 'measurements'] +>>> list(table.as_dicts) +[{'x': '1', 'y': '2'}] +``` + ### More examples The tests show additional supported features: - [Custom TODO keywords](https://github.com/karlicoss/orgparse/blob/master/src/orgparse/tests/test_misc.py#L72-L97) - [File-level tags](https://github.com/karlicoss/orgparse/blob/master/src/orgparse/tests/test_misc.py#L155-L166) -- [Reading tables](https://github.com/karlicoss/orgparse/blob/master/src/orgparse/tests/test_rich.py#L11-L60) +- [Reading tables](https://github.com/karlicoss/orgparse/blob/master/src/orgparse/tests/test_rich.py#L11-L61) ## Development and documentation diff --git a/README.qmd b/README.qmd index 7fff685..f8e313a 100644 --- a/README.qmd +++ b/README.qmd @@ -135,6 +135,25 @@ True ``` +### Read named tables + +Tables in `node.body_rich` expose their `#+NAME:` through `Table.name`. +The name is `None` for unnamed tables. + +```pycon +>>> from orgparse.extra import Table +>>> root = loads(''' +... #+NAME: measurements +... | x | y | +... |---+---| +... | 1 | 2 | +... ''') +>>> [table] = [part for part in root.body_rich if isinstance(part, Table) and part.name == 'measurements'] +>>> list(table.as_dicts) +[{'x': '1', 'y': '2'}] + +``` + ### More examples The tests show additional supported features: diff --git a/src/orgparse/extra.py b/src/orgparse/extra.py index 0c9c1ce..efc430f 100644 --- a/src/orgparse/extra.py +++ b/src/orgparse/extra.py @@ -5,6 +5,14 @@ RE_TABLE_SEPARATOR = re.compile(r'\s*\|(\-+\+)*\-+\|') RE_TABLE_ROW = re.compile(r'\s*\|([^|]+)+\|') +# Org's affiliated keywords, including legacy spellings of NAME. +RE_AFFILIATED_KEYWORD = re.compile( + r'[ \t]*#\+' + r'(?:(?PNAME|TBLNAME|DATA|LABEL|RESNAME|SOURCE|SRCNAME)' + r'|(?:CAPTION|RESULTS)(?:\[.*\])?|HEADERS?|PLOT|RESULT|ATTR_[-_A-Za-z0-9]+)' + r':[ \t]*(?P.*)', + re.IGNORECASE, +) STRIP_CELL_WHITESPACE = True @@ -12,8 +20,17 @@ class Table: - def __init__(self, lines: list[str]) -> None: + def __init__(self, lines: list[str], *, name: str | None = None) -> None: self._lines = lines + self._name = name + + @property + def name(self) -> str | None: + """The affiliated ``#+NAME:`` value, or ``None`` for an unnamed table. + + Legacy spellings such as ``#+TBLNAME:`` are also recognized. + """ + return self._name @property def blocks(self) -> Iterator[Sequence[Row]]: @@ -84,6 +101,18 @@ class Gap: Rich = Table | Gap +def _table_name(lines: Sequence[str]) -> str | None: + # Only consecutive affiliated keywords immediately before the table apply. + # Searching backwards gives the last NAME precedence, as in Org. + for line in reversed(lines): + match = RE_AFFILIATED_KEYWORD.match(line) + if match is None: + break + if match['name'] is not None: + return match['value'].strip() + return None + + def to_rich_text(text: str) -> Iterator[Rich]: ''' Convert an org-mode text into a 'rich' text, e.g. tables/lists/etc, interleaved by gaps. @@ -95,13 +124,14 @@ def to_rich_text(text: str) -> Iterator[Rich]: lines = text.splitlines(keepends=True) group: list[str] = [] last: type[Rich] = Gap + table_name: str | None = None def emit() -> Rich: nonlocal group, last if last is Gap: res = Gap() elif last is Table: - res = Table(group) # type: ignore[assignment] + res = Table(group, name=table_name) # type: ignore[assignment] else: raise RuntimeError(f'Unexpected type {last}') group = [] @@ -113,6 +143,8 @@ def emit() -> Rich: else: cur = Gap # type: ignore[assignment] if cur is not last: + if cur is Table: + table_name = _table_name(group) if len(group) > 0: yield emit() last = cur diff --git a/src/orgparse/tests/corpus/__init__.py b/src/orgparse/tests/corpus/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/orgparse/tests/corpus/test_tables.py b/src/orgparse/tests/corpus/test_tables.py new file mode 100644 index 0000000..e2770e9 --- /dev/null +++ b/src/orgparse/tests/corpus/test_tables.py @@ -0,0 +1,199 @@ +"""Check concrete table contents and names in the pinned upstream corpus. + +Load complete Org documents through the public API. +Expected values come from the source files, including tables embedded in larger documents. +""" + +from pathlib import Path + +import pytest + +from ... import OrgRootNode, load +from ...extra import Table + +CORPUS = Path(__file__).resolve().parents[4] / 'testdata' / 'external' +ORG_EXAMPLES = CORPUS / 'org-mode' / 'testing' / 'examples' + + +@pytest.fixture(scope='module') +def sachac() -> OrgRootNode: + return load(CORPUS / 'sachac' / 'Sacha.org') + + +@pytest.mark.parametrize( + ('filename', 'heading', 'expected_tables'), + [ + ( + 'ob-maxima-test.org', + 'Table input', + [ + ('test_tbl_col', [['1.0'], ['2.0']]), + ('test_tbl_row', [['1.0', '2.0']]), + ('test_tbl_mtr', [['1.0', '1.0']]), + ], + ), + ( + 'ob-fortran-test.org', + 'matrix', + [ + ('fortran-input-matrix1', [['0.0', '42.0'], ['0.0', '0.0'], ['0.0', '0.0']]), + ('fortran-input-matrix2', [['0.0', '0.0', '0.0'], ['0.0', '0.0', '42.0']]), + ], + ), + ], + ids=['maxima', 'fortran'], +) +def test_org_named_matrices(*, filename: str, heading: str, expected_tables: list[tuple[str, list[list[str]]]]) -> None: + """Distinguish named tables in one heading, preserving their shapes and numeric text.""" + root = load(ORG_EXAMPLES / filename) + [node] = [node for node in root[1:] if node.heading == heading] + tables = [part for part in node.body_rich if isinstance(part, Table)] + assert [table.name for table in tables] == [name for name, _ in expected_tables] + for table, (_, rows) in zip(tables, expected_tables, strict=True): + assert list(table.rows) == rows + assert list(table.blocks) == [rows] + + +def test_org_captioned_table() -> None: + """Associate an uppercase NAME and preceding caption with a single-cell table.""" + root = load(ORG_EXAMPLES / 'include.org') + [node] = [node for node in root[1:] if node.get_property('CUSTOM_ID') == 'ht'] + [table] = [part for part in node.body_rich if isinstance(part, Table)] + assert table.name == 'tbl' + assert list(table) == [['1']] + assert list(table.blocks) == [[['1']]] + + +def test_org_heterogeneous_table() -> None: + """Read indented rows with text and numbers, using the header for dictionary keys.""" + root = load(ORG_EXAMPLES / 'ob-C-test.org') + [node] = [node for node in root[1:] if node.heading == 'Inhomogeneous table'] + [table] = [part for part in node.body_rich if isinstance(part, Table)] + assert table.name == 'tinomogen' + assert table.as_dicts.columns == ['day', 'quty'] + assert list(table.as_dicts) == [ + {'day': 'monday', 'quty': '34'}, + {'day': 'tuesday', 'quty': '41'}, + {'day': 'wednesday', 'quty': '56'}, + {'day': 'thursday', 'quty': '17'}, + {'day': 'friday', 'quty': '12'}, + {'day': 'saturday', 'quty': '7'}, + {'day': 'sunday', 'quty': '4'}, + ] # fmt: skip + + +def test_org_multiple_blocks() -> None: + """Preserve separator-delimited sections and refuse ambiguous dictionary conversion.""" + root = load(ORG_EXAMPLES / 'ob-header-arg-defaults.org') + [node] = [node for node in root[1:] if node.heading == 'Overwrite'] + [table] = [part for part in node.body_rich if isinstance(part, Table)] + assert table.name is None + assert list(table.blocks) == [ + [ + ['Global', 't1', 't2', 't3', 't4', 't5', 't6', 't7', 't8', 't9'], + ], + [ + ['header-args', 'gh1', 'gh2', '---', 'gh4', '---', '---', '---', '---', '---'], + ['header-args:emacs-lisp', 'ge1', '---', '---', 'ge4', 'ge5', '---', '---', '---', '---'], + ], + [ + ['Tree', 't1', 't2', 't3', 't4', 't5', 't6', 't7', 't8', 't9'], + ], + [ + ['header-args', '---', '---', '---', '---', '---', '---', 'th7', '---', '---'], + ['header-args:emacs-lisp', '---', '---', '---', '---', '---', '---', '---', 'te8', '---'], + ], + [ + ['Result #+CALL', 'ge1', 'gh2', '--3', 'ge4', 'ge5', '--6', 'th7', 'te8', '--9'], + ['Result noweb', '--1', '--2', '--3', '--4', '--5', '--6', 'th7', 'te8', '--9'], + ], + ] # fmt: skip + with pytest.raises(RuntimeError, match='Need two-block table'): + list(table.as_dicts) + + +def test_sachac_lispy_bindings(sachac: OrgRootNode) -> None: + """Read an indented named table inside a special block, including empty cells and punctuation keys.""" + [node] = [node for node in sachac[1:] if node.get_property('CUSTOM_ID') == 'hydra-lispy'] + [table] = [part for part in node.body_rich if isinstance(part, Table)] + assert table.name == 'bindings' + [header, data] = table.blocks + assert header == [['key', 'function', 'column']] + assert len(data) == 69 + records = list(table.as_dicts) + assert records[0] == {'key': '<', 'function': 'lispy-barf', 'column': ''} + assert records[-1] == {'key': 'm', 'function': 'lispy-mark-list', 'column': 'Other'} + assert [row for row in records if row['key'] == '\\'] == [ + {'key': '\\', 'function': 'lispy-splice', 'column': 'Edit'}, + ] + assert [row for row in records if row['key'] == '-'] == [ + {'key': '-', 'function': 'lispy-ace-subword', 'column': 'Nav'}, + ] + + +def test_sachac_unicode_and_links(sachac: OrgRootNode) -> None: + """Keep accented headers and raw Org links when converting an unnamed table to dictionaries.""" + [node] = [node for node in sachac[1:] if node.heading == 'Fréquence des erreurs par catégorie'] + [table] = [part for part in node.body_rich if isinstance(part, Table)] + assert table.name is None + assert table.as_dicts.columns == ['Thématique (KwizIQ)', "Nombre d'erreurs"] + records = list(table.as_dicts) + assert len(records) == 7 + assert records[0] == { + 'Thématique (KwizIQ)': '[[https://french.kwiziq.com/revision/grammar/topics/nouns-articles][Nouns & Articles]]', + "Nombre d'erreurs": '18', + } + assert records[-1] == { + 'Thématique (KwizIQ)': '[[https://french.kwiziq.com/revision/grammar/topics/numbers-time-date][Numbers, Time & Date]]', + "Nombre d'erreurs": '8', + } + + +def test_sachac_link_syntax(sachac: OrgRootNode) -> None: + """Preserve inline code and several link syntaxes in a table without a header separator.""" + [node] = [ + node + for node in sachac[1:] + if node.heading == 'Adding Org Mode link awesomeness elsewhere: sacha-org-insert-link-dwim' + ] + [table] = [part for part in node.body_rich if isinstance(part, Table)] + assert table.name is None + rows = [ + ['HTML', '~title~'], + ['Org', '~[[https://example.com][title]]~'], + ['Plain text', '~title https://example.com~'], + ['Markdown', '~[https://example.com](title)~'], + ['Oddmuse', '~[https://example.com title]~'], + ] # fmt: skip + assert list(table.rows) == rows + assert list(table.blocks) == [rows] + with pytest.raises(RuntimeError, match='Need two-block table'): + list(table.as_dicts) + + +def test_sachac_babel_results(sachac: OrgRootNode) -> None: + """Read separate result tables in drawers without mistaking source/result labels for table names.""" + [node] = [ + node + for node in sachac[1:] + if node.heading == 'Emacs and whisper.el: Trying out different speech-to-text backends and models' + ] + [cpu, gpu] = [part for part in node.body_rich if isinstance(part, Table)] + assert cpu.name is None + assert gpu.name is None + assert list(cpu.rows) == [ + ['3.694', 'parakeet'], + ['2.484', 'whisper.cpp base-q4_0'], + ['1.547', 'speaches whisper-base'], + ['1.425', 'speaches whisper-base.en'], + ['4.076', 'speaches whisper-small'], + ['3.735', 'speaches whisper-small.en'], + ['2.870', 'speaches lorneluo/whisper-small-ct2-int8'], + ['4.537', 'whisperx-server Systran/faster-whisper-small'], + ] + assert list(gpu.rows) == [ + ['0.596', 'speaches whisper-tiny'], + ['0.940', 'speaches whisper-base'], + ['2.909', 'speaches whisper-small'], + ['8.740', 'speaches whisper-medium'], + ] diff --git a/src/orgparse/tests/test_corpus.py b/src/orgparse/tests/test_corpus.py index 8c527ed..1c1942c 100644 --- a/src/orgparse/tests/test_corpus.py +++ b/src/orgparse/tests/test_corpus.py @@ -318,6 +318,7 @@ def by_custom_id(node_id: str) -> OrgNode: abbreviations = by_custom_id('completion-define-abbreviations') assert abbreviations.heading == 'Define abbreviations' [table] = [part for part in abbreviations.body_rich if isinstance(part, Table)] + assert table.name == 'global-abbrev' expected_rows = [ ['meweb', 'https://sachachua.com'], ['mehub', 'https://github.com/sachac'], diff --git a/src/orgparse/tests/test_rich.py b/src/orgparse/tests/test_rich.py index 5171bb0..214dcba 100644 --- a/src/orgparse/tests/test_rich.py +++ b/src/orgparse/tests/test_rich.py @@ -43,6 +43,7 @@ def test_table() -> None: t2 = Table(root._lines[11:19]) t3 = Table(root._lines[22:26]) + assert t1.name is None assert ilen(t1.blocks) == 4 assert list(t1.blocks)[2] == [] assert ilen(t1.rows) == 6 @@ -82,8 +83,74 @@ def test_table_2() -> None: [[file:plot.png]] ''') [_, t, _] = root.children[0].body_rich + assert isinstance(t, Table) + assert t.name == 'something' assert ilen(t.as_dicts) == 3 +def test_named_table() -> None: + """Expose the table name requested in https://github.com/karlicoss/orgparse/issues/34.""" + root = loads(''' +#+caption: Some caption for a table +#+name: tabname +| x | y | +|---+---| +| 1 | 2 | +''') + [table] = [part for part in root.body_rich if isinstance(part, Table)] + assert table.name == 'tabname' + assert list(table.rows) == [['x', 'y'], ['1', '2']] + assert list(table.as_dicts) == [{'x': '1', 'y': '2'}] + assert '#+name: tabname' in root.get_body(format='raw') + + +@pytest.mark.parametrize( + ('metadata', 'expected_name'), + [ + ('', None), + (' #+NaMe:\t Mixed.Name \t\n', 'Mixed.Name'), + ('#+NAME:\n', ''), + ('#+NAME: first\n#+NAME: last\n', 'last'), + ( + '#+NAME: data\n#+CAPTION[short]: A longer caption\n#+ATTR_HTML: :border 2\n#+PLOT: title:"Data"\n', + 'data', + ), + ('#+NAME: data\n#+HEADER: :results table\n#+RESULTS[hash]: source\n', 'data'), + ('#+NAME: data\n\n', None), + ('#+NAME: data\n \t\n', None), + ('#+NAME: data\nSome intervening text\n', None), + ('#+NAME: data\n# A comment\n', None), + ('#+NAME: data\n#+TITLE: A different keyword\n', None), + ('#+NAME: data\n#+NAME[invalid]: other\n', None), + ('#+NAME: data\n#+BEGIN_SRC python\nprint(1)\n#+END_SRC\n', None), + ], +) +def test_table_name_affiliation(metadata: str, expected_name: str | None) -> None: + root = loads(metadata + ' | value |\n') + [table] = [part for part in root.body_rich if isinstance(part, Table)] + assert table.name == expected_name + assert list(table.rows) == [['value']] + + +def test_table_names_are_local() -> None: + """Keep each name on its own table, without leaking across tables or headings.""" + root = loads(''' +#+NAME: first +| 1 | + +| 2 | +#+NAME: third +| 3 | +* Heading +| 4 | +#+NAME: fifth +| 5 | +''') + tables = [part for part in root.body_rich if isinstance(part, Table)] + assert [table.name for table in tables] == ['first', None, 'third'] + child_tables = [part for part in root.children[0].body_rich if isinstance(part, Table)] + assert [table.name for table in child_tables] == [None, 'fifth'] + + def ilen(x) -> int: return len(list(x))