Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 19 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -106,13 +106,31 @@ True
' Body texts...'
```

### Read named tables

Tables in `node.body_rich` expose their `#+NAME:` through `Table.name`.
The name is `None` for unnamed tables.

``` pycon
>>> from orgparse.extra import Table
>>> root = loads('''
... #+NAME: measurements
... | x | y |
... |---+---|
... | 1 | 2 |
... ''')
>>> [table] = [part for part in root.body_rich if isinstance(part, Table) and part.name == 'measurements']
>>> list(table.as_dicts)
[{'x': '1', 'y': '2'}]
```

### More examples

The tests show additional supported features:

- [Custom TODO keywords](https://github.com/karlicoss/orgparse/blob/master/src/orgparse/tests/test_misc.py#L72-L97)
- [File-level tags](https://github.com/karlicoss/orgparse/blob/master/src/orgparse/tests/test_misc.py#L155-L166)
- [Reading tables](https://github.com/karlicoss/orgparse/blob/master/src/orgparse/tests/test_rich.py#L11-L60)
- [Reading tables](https://github.com/karlicoss/orgparse/blob/master/src/orgparse/tests/test_rich.py#L11-L61)

## Development and documentation

Expand Down
19 changes: 19 additions & 0 deletions README.qmd
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,25 @@ True

```

### Read named tables

Tables in `node.body_rich` expose their `#+NAME:` through `Table.name`.
The name is `None` for unnamed tables.

```pycon
>>> from orgparse.extra import Table
>>> root = loads('''
... #+NAME: measurements
... | x | y |
... |---+---|
... | 1 | 2 |
... ''')
>>> [table] = [part for part in root.body_rich if isinstance(part, Table) and part.name == 'measurements']
>>> list(table.as_dicts)
[{'x': '1', 'y': '2'}]

```

### More examples

The tests show additional supported features:
Expand Down
36 changes: 34 additions & 2 deletions src/orgparse/extra.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,32 @@

RE_TABLE_SEPARATOR = re.compile(r'\s*\|(\-+\+)*\-+\|')
RE_TABLE_ROW = re.compile(r'\s*\|([^|]+)+\|')
# Org's affiliated keywords, including legacy spellings of NAME.
RE_AFFILIATED_KEYWORD = re.compile(
r'[ \t]*#\+'
r'(?:(?P<name>NAME|TBLNAME|DATA|LABEL|RESNAME|SOURCE|SRCNAME)'
r'|(?:CAPTION|RESULTS)(?:\[.*\])?|HEADERS?|PLOT|RESULT|ATTR_[-_A-Za-z0-9]+)'
r':[ \t]*(?P<value>.*)',
re.IGNORECASE,
)
STRIP_CELL_WHITESPACE = True


Row = Sequence[str]


class Table:
def __init__(self, lines: list[str]) -> None:
def __init__(self, lines: list[str], *, name: str | None = None) -> None:
self._lines = lines
self._name = name

@property
def name(self) -> str | None:
"""The affiliated ``#+NAME:`` value, or ``None`` for an unnamed table.

Legacy spellings such as ``#+TBLNAME:`` are also recognized.
"""
return self._name

@property
def blocks(self) -> Iterator[Sequence[Row]]:
Expand Down Expand Up @@ -84,6 +101,18 @@ class Gap:
Rich = Table | Gap


def _table_name(lines: Sequence[str]) -> str | None:
# Only consecutive affiliated keywords immediately before the table apply.
# Searching backwards gives the last NAME precedence, as in Org.
for line in reversed(lines):
match = RE_AFFILIATED_KEYWORD.match(line)
if match is None:
break
if match['name'] is not None:
return match['value'].strip()
return None


def to_rich_text(text: str) -> Iterator[Rich]:
'''
Convert an org-mode text into a 'rich' text, e.g. tables/lists/etc, interleaved by gaps.
Expand All @@ -95,13 +124,14 @@ def to_rich_text(text: str) -> Iterator[Rich]:
lines = text.splitlines(keepends=True)
group: list[str] = []
last: type[Rich] = Gap
table_name: str | None = None

def emit() -> Rich:
nonlocal group, last
if last is Gap:
res = Gap()
elif last is Table:
res = Table(group) # type: ignore[assignment]
res = Table(group, name=table_name) # type: ignore[assignment]
else:
raise RuntimeError(f'Unexpected type {last}')
group = []
Expand All @@ -113,6 +143,8 @@ def emit() -> Rich:
else:
cur = Gap # type: ignore[assignment]
if cur is not last:
if cur is Table:
table_name = _table_name(group)
if len(group) > 0:
yield emit()
last = cur
Expand Down
Empty file.
199 changes: 199 additions & 0 deletions src/orgparse/tests/corpus/test_tables.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,199 @@
"""Check concrete table contents and names in the pinned upstream corpus.

Load complete Org documents through the public API.
Expected values come from the source files, including tables embedded in larger documents.
"""

from pathlib import Path

import pytest

from ... import OrgRootNode, load
from ...extra import Table

CORPUS = Path(__file__).resolve().parents[4] / 'testdata' / 'external'
ORG_EXAMPLES = CORPUS / 'org-mode' / 'testing' / 'examples'


@pytest.fixture(scope='module')
def sachac() -> OrgRootNode:
return load(CORPUS / 'sachac' / 'Sacha.org')


@pytest.mark.parametrize(
('filename', 'heading', 'expected_tables'),
[
(
'ob-maxima-test.org',
'Table input',
[
('test_tbl_col', [['1.0'], ['2.0']]),
('test_tbl_row', [['1.0', '2.0']]),
('test_tbl_mtr', [['1.0', '1.0']]),
],
),
(
'ob-fortran-test.org',
'matrix',
[
('fortran-input-matrix1', [['0.0', '42.0'], ['0.0', '0.0'], ['0.0', '0.0']]),
('fortran-input-matrix2', [['0.0', '0.0', '0.0'], ['0.0', '0.0', '42.0']]),
],
),
],
ids=['maxima', 'fortran'],
)
def test_org_named_matrices(*, filename: str, heading: str, expected_tables: list[tuple[str, list[list[str]]]]) -> None:
"""Distinguish named tables in one heading, preserving their shapes and numeric text."""
root = load(ORG_EXAMPLES / filename)
[node] = [node for node in root[1:] if node.heading == heading]
tables = [part for part in node.body_rich if isinstance(part, Table)]
assert [table.name for table in tables] == [name for name, _ in expected_tables]
for table, (_, rows) in zip(tables, expected_tables, strict=True):
assert list(table.rows) == rows
assert list(table.blocks) == [rows]


def test_org_captioned_table() -> None:
"""Associate an uppercase NAME and preceding caption with a single-cell table."""
root = load(ORG_EXAMPLES / 'include.org')
[node] = [node for node in root[1:] if node.get_property('CUSTOM_ID') == 'ht']
[table] = [part for part in node.body_rich if isinstance(part, Table)]
assert table.name == 'tbl'
assert list(table) == [['1']]
assert list(table.blocks) == [[['1']]]


def test_org_heterogeneous_table() -> None:
"""Read indented rows with text and numbers, using the header for dictionary keys."""
root = load(ORG_EXAMPLES / 'ob-C-test.org')
[node] = [node for node in root[1:] if node.heading == 'Inhomogeneous table']
[table] = [part for part in node.body_rich if isinstance(part, Table)]
assert table.name == 'tinomogen'
assert table.as_dicts.columns == ['day', 'quty']
assert list(table.as_dicts) == [
{'day': 'monday', 'quty': '34'},
{'day': 'tuesday', 'quty': '41'},
{'day': 'wednesday', 'quty': '56'},
{'day': 'thursday', 'quty': '17'},
{'day': 'friday', 'quty': '12'},
{'day': 'saturday', 'quty': '7'},
{'day': 'sunday', 'quty': '4'},
] # fmt: skip


def test_org_multiple_blocks() -> None:
"""Preserve separator-delimited sections and refuse ambiguous dictionary conversion."""
root = load(ORG_EXAMPLES / 'ob-header-arg-defaults.org')
[node] = [node for node in root[1:] if node.heading == 'Overwrite']
[table] = [part for part in node.body_rich if isinstance(part, Table)]
assert table.name is None
assert list(table.blocks) == [
[
['Global', 't1', 't2', 't3', 't4', 't5', 't6', 't7', 't8', 't9'],
],
[
['header-args', 'gh1', 'gh2', '---', 'gh4', '---', '---', '---', '---', '---'],
['header-args:emacs-lisp', 'ge1', '---', '---', 'ge4', 'ge5', '---', '---', '---', '---'],
],
[
['Tree', 't1', 't2', 't3', 't4', 't5', 't6', 't7', 't8', 't9'],
],
[
['header-args', '---', '---', '---', '---', '---', '---', 'th7', '---', '---'],
['header-args:emacs-lisp', '---', '---', '---', '---', '---', '---', '---', 'te8', '---'],
],
[
['Result #+CALL', 'ge1', 'gh2', '--3', 'ge4', 'ge5', '--6', 'th7', 'te8', '--9'],
['Result noweb', '--1', '--2', '--3', '--4', '--5', '--6', 'th7', 'te8', '--9'],
],
] # fmt: skip
with pytest.raises(RuntimeError, match='Need two-block table'):
list(table.as_dicts)


def test_sachac_lispy_bindings(sachac: OrgRootNode) -> None:
"""Read an indented named table inside a special block, including empty cells and punctuation keys."""
[node] = [node for node in sachac[1:] if node.get_property('CUSTOM_ID') == 'hydra-lispy']
[table] = [part for part in node.body_rich if isinstance(part, Table)]
assert table.name == 'bindings'
[header, data] = table.blocks
assert header == [['key', 'function', 'column']]
assert len(data) == 69
records = list(table.as_dicts)
assert records[0] == {'key': '<', 'function': 'lispy-barf', 'column': ''}
assert records[-1] == {'key': 'm', 'function': 'lispy-mark-list', 'column': 'Other'}
assert [row for row in records if row['key'] == '\\'] == [
{'key': '\\', 'function': 'lispy-splice', 'column': 'Edit'},
]
assert [row for row in records if row['key'] == '-'] == [
{'key': '-', 'function': 'lispy-ace-subword', 'column': 'Nav'},
]


def test_sachac_unicode_and_links(sachac: OrgRootNode) -> None:
"""Keep accented headers and raw Org links when converting an unnamed table to dictionaries."""
[node] = [node for node in sachac[1:] if node.heading == 'Fréquence des erreurs par catégorie']
[table] = [part for part in node.body_rich if isinstance(part, Table)]
assert table.name is None
assert table.as_dicts.columns == ['Thématique (KwizIQ)', "Nombre d'erreurs"]
records = list(table.as_dicts)
assert len(records) == 7
assert records[0] == {
'Thématique (KwizIQ)': '[[https://french.kwiziq.com/revision/grammar/topics/nouns-articles][Nouns & Articles]]',
"Nombre d'erreurs": '18',
}
assert records[-1] == {
'Thématique (KwizIQ)': '[[https://french.kwiziq.com/revision/grammar/topics/numbers-time-date][Numbers, Time & Date]]',
"Nombre d'erreurs": '8',
}


def test_sachac_link_syntax(sachac: OrgRootNode) -> None:
"""Preserve inline code and several link syntaxes in a table without a header separator."""
[node] = [
node
for node in sachac[1:]
if node.heading == 'Adding Org Mode link awesomeness elsewhere: sacha-org-insert-link-dwim'
]
[table] = [part for part in node.body_rich if isinstance(part, Table)]
assert table.name is None
rows = [
['HTML', '~<a href="https://example.com">title</a>~'],
['Org', '~[[https://example.com][title]]~'],
['Plain text', '~title https://example.com~'],
['Markdown', '~[https://example.com](title)~'],
['Oddmuse', '~[https://example.com title]~'],
] # fmt: skip
assert list(table.rows) == rows
assert list(table.blocks) == [rows]
with pytest.raises(RuntimeError, match='Need two-block table'):
list(table.as_dicts)


def test_sachac_babel_results(sachac: OrgRootNode) -> None:
"""Read separate result tables in drawers without mistaking source/result labels for table names."""
[node] = [
node
for node in sachac[1:]
if node.heading == 'Emacs and whisper.el: Trying out different speech-to-text backends and models'
]
[cpu, gpu] = [part for part in node.body_rich if isinstance(part, Table)]
assert cpu.name is None
assert gpu.name is None
assert list(cpu.rows) == [
['3.694', 'parakeet'],
['2.484', 'whisper.cpp base-q4_0'],
['1.547', 'speaches whisper-base'],
['1.425', 'speaches whisper-base.en'],
['4.076', 'speaches whisper-small'],
['3.735', 'speaches whisper-small.en'],
['2.870', 'speaches lorneluo/whisper-small-ct2-int8'],
['4.537', 'whisperx-server Systran/faster-whisper-small'],
]
assert list(gpu.rows) == [
['0.596', 'speaches whisper-tiny'],
['0.940', 'speaches whisper-base'],
['2.909', 'speaches whisper-small'],
['8.740', 'speaches whisper-medium'],
]
1 change: 1 addition & 0 deletions src/orgparse/tests/test_corpus.py
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,7 @@ def by_custom_id(node_id: str) -> OrgNode:
abbreviations = by_custom_id('completion-define-abbreviations')
assert abbreviations.heading == 'Define abbreviations'
[table] = [part for part in abbreviations.body_rich if isinstance(part, Table)]
assert table.name == 'global-abbrev'
expected_rows = [
['meweb', 'https://sachachua.com'],
['mehub', 'https://github.com/sachac'],
Expand Down
Loading
Loading