diff --git a/doc/.gitignore b/doc/.gitignore new file mode 100644 index 00000000..3e609b0d --- /dev/null +++ b/doc/.gitignore @@ -0,0 +1,7 @@ +/*.html +/.quarto/ +**/*.quarto_ipynb +/_freeze/ +/_site/ +/*_files/libs/ +/quarto/ diff --git a/doc/README.md b/doc/README.md new file mode 100644 index 00000000..db281a1f --- /dev/null +++ b/doc/README.md @@ -0,0 +1,36 @@ +# HPI docs + + +- [Docs](#docs) +- [Legacy (org, being migrated)](#legacy-org-being-migrated) + +Render all documentation with `tox -e quarto`. For live browser preview, +run `uv run --only-group quarto quarto preview doc --profile preview` +from the repository root. + +With `freeze: auto`, Quarto only detects changes to `.qmd` files. After +editing `quarto_helpers.py`, run `tox -e quarto -- --execute` or delete +`_freeze/` to re-execute the docs that import it. + +# Docs + +User guides: + +- [Configuring modules](configuring_modules.md) +- [Querying HPI](QUERY.md) +- [Denylists](DENYLIST.md) + +Architecture decisions: + +- [Configuration requirements](configuration_requirements.md) + +# Legacy (org, being migrated) + +- [CONFIGURING](CONFIGURING.org) +- [CONTRIBUTING](CONTRIBUTING.org) +- [DESIGN](DESIGN.org) +- [DEVELOPMENT](DEVELOPMENT.org) +- [MODULES](MODULES.org) +- [MODULE_DESIGN](MODULE_DESIGN.org) +- [OVERLAYS](OVERLAYS.org) +- [SETUP](SETUP.org) diff --git a/doc/README.qmd b/doc/README.qmd new file mode 100644 index 00000000..98108352 --- /dev/null +++ b/doc/README.qmd @@ -0,0 +1,32 @@ +--- +title: "HPI docs" +--- + +Render all documentation with `tox -e quarto`. +For live browser preview, run `uv run --only-group quarto quarto preview doc --profile preview` from the repository root. + +With `freeze: auto`, Quarto only detects changes to `.qmd` files. +After editing `quarto_helpers.py`, run `tox -e quarto -- --execute` or delete `_freeze/` to re-execute the docs that import it. + +# Docs + +User guides: + +- [Configuring modules](configuring_modules.md) +- [Querying HPI](QUERY.md) +- [Denylists](DENYLIST.md) + +Architecture decisions: + +- [Configuration requirements](configuration_requirements.md) + +# Legacy (org, being migrated) + +- [CONFIGURING](CONFIGURING.org) +- [CONTRIBUTING](CONTRIBUTING.org) +- [DESIGN](DESIGN.org) +- [DEVELOPMENT](DEVELOPMENT.org) +- [MODULES](MODULES.org) +- [MODULE_DESIGN](MODULE_DESIGN.org) +- [OVERLAYS](OVERLAYS.org) +- [SETUP](SETUP.org) diff --git a/doc/_quarto-preview.yml b/doc/_quarto-preview.yml new file mode 100644 index 00000000..7bba82a9 --- /dev/null +++ b/doc/_quarto-preview.yml @@ -0,0 +1,5 @@ +# Website mode is only for project-wide live preview and qmd-to-html link resolution. +project: + type: website + +format: html diff --git a/doc/quarto/_quarto.yml b/doc/_quarto.yml similarity index 70% rename from doc/quarto/_quarto.yml rename to doc/_quarto.yml index 62924c07..7dfe5896 100644 --- a/doc/quarto/_quarto.yml +++ b/doc/_quarto.yml @@ -4,14 +4,12 @@ project: - "*.qmd" # otherwise picks up .md for some reason?? - "README.qmd" # for whatever reason quarto excludes it by default format: - gfm: - toc: true - toc-depth: 3 - html: - toc: true - toc-depth: 3 - anchor-sections: true - # code-fold: true + gfm: default + html: default +toc: true +toc-depth: 3 +anchor-sections: true +# code-fold: true execute: freeze: auto # re-render only when source changes diff --git a/doc/quarto/configuration_requirements.md b/doc/configuration_requirements.md similarity index 99% rename from doc/quarto/configuration_requirements.md rename to doc/configuration_requirements.md index c60bcdb3..06c7280b 100644 --- a/doc/quarto/configuration_requirements.md +++ b/doc/configuration_requirements.md @@ -22,8 +22,7 @@ This doc describes the technical decisions behind HPI configuration system. It’s more of a ‘design doc’ rather than usage guide. If you just -want to know how to set up HPI or configure it, see -[SETUP](../SETUP.org). +want to know how to set up HPI or configure it, see [SETUP](SETUP.org). I feel like it’s good to keep the rationales in the documentation, but happy to [discuss](https://github.com/karlicoss/HPI/issues/46) it here. diff --git a/doc/quarto/configuration_requirements.qmd b/doc/configuration_requirements.qmd similarity index 96% rename from doc/quarto/configuration_requirements.qmd rename to doc/configuration_requirements.qmd index 6bc238ae..ec466068 100644 --- a/doc/quarto/configuration_requirements.qmd +++ b/doc/configuration_requirements.qmd @@ -12,26 +12,12 @@ format: ```{python} #| echo: false -from pathlib import Path -import jedi - -src_dir = Path.cwd().absolute().parent.parent / 'src' -assert src_dir.exists(), src_dir - -project = jedi.Project(src_dir) - -def github_link(name: str) -> str: - completions = list(project.complete_search(name, all_scopes=True)) - assert len(completions) == 1, f"Expected one completion for {name}, got {completions}" - [c] = completions - [c] = c.goto() - rpath = Path(c.module_path).relative_to(src_dir) - return f"https://github.com/karlicoss/HPI/blob/master/src/{rpath}#L{c.line}" +from quarto_helpers import github_link ``` This doc describes the technical decisions behind HPI configuration system. It's more of a 'design doc' rather than usage guide. -If you just want to know how to set up HPI or configure it, see [SETUP](../SETUP.org). +If you just want to know how to set up HPI or configure it, see [SETUP](SETUP.org). I feel like it's good to keep the rationales in the documentation, but happy to [discuss](https://github.com/karlicoss/HPI/issues/46) it here. diff --git a/doc/quarto/configuring_modules.md b/doc/configuring_modules.md similarity index 99% rename from doc/quarto/configuring_modules.md rename to doc/configuring_modules.md index 6c7e28c2..fc7b4b5d 100644 --- a/doc/quarto/configuring_modules.md +++ b/doc/configuring_modules.md @@ -144,7 +144,7 @@ Cons: instantiate the config on first use (TODO link to it later). See more about this approach -[here](https://github.com/karlicoss/HPI/blob/master/src/my/core/docs/test_configuration.py#L384). +[here](https://github.com/karlicoss/HPI/blob/master/src/my/core/docs/test_configuration.py#L386). # Other options diff --git a/doc/quarto/configuring_modules.qmd b/doc/configuring_modules.qmd similarity index 90% rename from doc/quarto/configuring_modules.qmd rename to doc/configuring_modules.qmd index 37230d79..932b09bd 100644 --- a/doc/quarto/configuring_modules.qmd +++ b/doc/configuring_modules.qmd @@ -9,21 +9,7 @@ format: ```{python} #| echo: false -from pathlib import Path -import jedi - -src_dir = Path.cwd().absolute().parent.parent / 'src' -assert src_dir.exists(), src_dir - -project = jedi.Project(src_dir) - -def github_link(name: str) -> str: - completions = list(project.complete_search(name, all_scopes=True)) - assert len(completions) == 1, f"Expected one completion for {name}, got {completions}" - [c] = completions - [c] = c.goto() - rpath = Path(c.module_path).relative_to(src_dir) - return f"https://github.com/karlicoss/HPI/blob/master/src/{rpath}#L{c.line}" +from quarto_helpers import github_link ``` @@ -152,4 +138,3 @@ Many actual HPI modules still have different implementations for historic reason If you're interested to dig more, see [test_configuration](`{python} github_link("my.core.docs.test_configuration")`) that covers various approaches. However, we've found that the property-based approach described above has the fewest quirks, so we recommend it going forward. - diff --git a/doc/index.qmd b/doc/index.qmd new file mode 100644 index 00000000..74bd4af3 --- /dev/null +++ b/doc/index.qmd @@ -0,0 +1,10 @@ +--- +title: "HPI docs" +format: html +--- + + + + + +[Open the documentation index](README.html). diff --git a/doc/quarto/.gitignore b/doc/quarto/.gitignore deleted file mode 100644 index 1eeaca25..00000000 --- a/doc/quarto/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -/.quarto/ -**/*.quarto_ipynb -_freeze/ -*_files/ diff --git a/doc/quarto/README.md b/doc/quarto/README.md deleted file mode 100644 index de1a992a..00000000 --- a/doc/quarto/README.md +++ /dev/null @@ -1,32 +0,0 @@ -# HPI docs - - -- [Misc](#misc) - - - - - -User guides: - -- [Configuring modules](configuring_modules.qmd) - -Architecture decisions: - -- [Configuration requirements](configuration_requirements.qmd) - -# Misc - -Just experimenting with using Quarto for docs instead of org-mode. - -- to rerender everything `tox -e quarto` - -- use `./validate-references.py .qmd` to check for broken - referneces - - TODO need to integrate to do automatically - -- interactive preview `quarto preview .qmd` - -TODO some issues with Quarto - in markdown, cross-references are not -output, and github uses custom cross-references syntax diff --git a/doc/quarto/README.qmd b/doc/quarto/README.qmd deleted file mode 100644 index bf8f232d..00000000 --- a/doc/quarto/README.qmd +++ /dev/null @@ -1,29 +0,0 @@ ---- -title: "HPI docs" ---- - - - - -User guides: - -- [Configuring modules](configuring_modules.qmd) - -Architecture decisions: - -- [Configuration requirements](configuration_requirements.qmd) - - -# Misc -Just experimenting with using Quarto for docs instead of org-mode. - -- to rerender everything `tox -e quarto` -- use `./validate-references.py .qmd` to check for broken referneces - - TODO need to integrate to do automatically - -- interactive preview `quarto preview .qmd` - - -TODO some issues with Quarto -- in markdown, cross-references are not output, and github uses custom cross-references syntax diff --git a/doc/quarto/validate_references.py b/doc/quarto/validate_references.py deleted file mode 100755 index 1ed35099..00000000 --- a/doc/quarto/validate_references.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 -""" -Sadly quarto doesn't have builtin reference validation? -See https://github.com/quarto-dev/quarto-cli/discussions/13259 -""" - -# TODO validate file links as well? -import re -import sys -from pathlib import Path - -defs = {} -refs = [] -relative_links = [] - -path = Path(sys.argv[1]) -for i, line in enumerate(path.read_text().splitlines()): - lineno = i + 1 - # FIXME might be multiple per line? - ## cross-reference definition - m = re.search(r'\{(#\S+)\}', line) - if m is not None: - xx = m.group(1) - assert xx not in defs, (xx, defs) - defs[xx] = (lineno, line) - continue - ## cross-reference usage - m = re.search(r'\((#\S+)\)', line) - if m is not None: - xx = m.group(1) - refs.append((xx, lineno, line)) - continue - m = re.search(r'\]\((\S+)\)', line) - if m is not None: - url = m.group(1) - if not url.startswith(('http://', 'https://')): - relative_links.append((url, lineno, line)) - - -errors = [] -for ref, lineno, line in refs: - if ref in defs: - continue - print(f"{path}:{lineno}: reference {ref} not found\n {line}. Known: {defs}", file=sys.stderr) - errors.append(ref) - -for url, lineno, line in relative_links: - relpath = Path(url) - if not relpath.exists(): - print(f"{path}:{lineno}: link {url} not found\n {line}", file=sys.stderr) - errors.append(url) - -assert len(errors) == 0 diff --git a/doc/quarto_helpers.py b/doc/quarto_helpers.py new file mode 100644 index 00000000..a4aa747e --- /dev/null +++ b/doc/quarto_helpers.py @@ -0,0 +1,24 @@ +import tomllib +from pathlib import Path + +import jedi + +REPO = Path(__file__).parent.parent +SRC = REPO / "src" + +with (REPO / "pyproject.toml").open("rb") as fo: + pyproject = tomllib.load(fo) +GITHUB = pyproject["project"]["urls"]["Homepage"] +assert isinstance(GITHUB, str), GITHUB +assert GITHUB.startswith("https://github.com/"), GITHUB + +project = jedi.Project(SRC) + + +def github_link(name: str) -> str: + completions = list(project.complete_search(name, all_scopes=True)) + assert len(completions) == 1, f"Expected one completion for {name}, got {completions}" + [c] = completions + [c] = c.goto() + rpath = Path(c.module_path).relative_to(SRC) + return f"{GITHUB}/blob/master/src/{rpath}#L{c.line}" diff --git a/doc/validate_references.py b/doc/validate_references.py new file mode 100755 index 00000000..b9f332d6 --- /dev/null +++ b/doc/validate_references.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +""" +Sadly quarto doesn't have builtin reference validation? +See https://github.com/quarto-dev/quarto-cli/discussions/13259 +""" + +# TODO validate file links as well? +import re +import sys +from pathlib import Path + + +def validate(path: Path) -> list[str]: + defs = {} + refs = [] + relative_links = [] + + for i, line in enumerate(path.read_text().splitlines()): + lineno = i + 1 + # FIXME might be multiple per line? + ## cross-reference definition + m = re.search(r'\{(#\S+)\}', line) + if m is not None: + xx = m.group(1) + assert xx not in defs, (xx, defs) + defs[xx] = (lineno, line) + continue + ## cross-reference usage + m = re.search(r'\((#\S+)\)', line) + if m is not None: + xx = m.group(1) + refs.append((xx, lineno, line)) + continue + m = re.search(r'\]\((\S+)\)', line) + if m is not None: + url = m.group(1) + if not url.startswith(('http://', 'https://')): + relative_links.append((url, lineno, line)) + + errors = [] + for ref, lineno, line in refs: + if ref in defs: + continue + print(f"{path}:{lineno}: reference {ref} not found\n {line}. Known: {defs}", file=sys.stderr) + errors.append(ref) + + for url, lineno, line in relative_links: + file_url = url.partition('#')[0] + relpath = path.parent / file_url + if not relpath.exists(): + print(f"{path}:{lineno}: link {url} not found\n {line}", file=sys.stderr) + errors.append(url) + + return errors + + +if len(sys.argv) == 1: + paths = sorted(Path(__file__).parent.rglob('*.qmd')) +else: + paths = [Path(arg) for arg in sys.argv[1:]] + +errors = [] +for path in paths: + errors.extend(validate(path)) +assert len(errors) == 0, errors diff --git a/pyproject.toml b/pyproject.toml index edb39fc3..75db0a85 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,6 +83,7 @@ typecheck-all = [ ] quarto = [ "quarto-cli", + "jedi", # used by quarto_helpers.py ] @@ -527,9 +528,13 @@ dependency_groups = ["quarto"] # so need to unset set_env.PYTHONSAFEPATH = "" commands = [ + # NOTE: for interactive browser preview can use uv run --group quarto quarto preview doc --profile preview [ "quarto", "render", - "doc/quarto", + "doc", { replace = "posargs", default = [], extend = true }, ], + [ + "{env_python}", "doc/validate_references.py", + ], ]