From a5f6f1cfac64cb3ac6fc8c1251be49d1b79db745 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ola=20Gr=C3=B8ttvik?= Date: Tue, 27 Oct 2026 13:10:24 +0000 Subject: [PATCH 1/2] Add Pyodide CI job and pure Python dependency test --- .github/workflows/python_tests.yml | 30 ++++++++++++++ .gitignore | 3 ++ CHANGELOG.md | 1 + docs/development.md | 14 ++++++- docs/getting-started.md | 13 ++++++ scripts/pyodide/generate_examples.py | 26 ++++++++++++ scripts/pyodide/package-lock.json | 53 +++++++++++++++++++++++++ scripts/pyodide/package.json | 9 +++++ scripts/pyodide/run.mjs | 41 +++++++++++++++++++ tests/test_pure_python.py | 59 ++++++++++++++++++++++++++++ 10 files changed, 248 insertions(+), 1 deletion(-) create mode 100644 scripts/pyodide/generate_examples.py create mode 100644 scripts/pyodide/package-lock.json create mode 100644 scripts/pyodide/package.json create mode 100644 scripts/pyodide/run.mjs create mode 100644 tests/test_pure_python.py diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml index 8c1865cb..e31ce12f 100644 --- a/.github/workflows/python_tests.yml +++ b/.github/workflows/python_tests.yml @@ -98,3 +98,33 @@ jobs: with: name: dist path: dist/* + + # cloudbust runs bust in the browser. Install the wheel in Pyodide with only + # the core dependencies and check that it writes what CPython writes. + pyodide: + runs-on: ubuntu-latest + needs: build + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.11" + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: dist + path: dist + - name: Generate the examples with CPython + run: | + python -m venv "$RUNNER_TEMP/wheel-venv" + "$RUNNER_TEMP/wheel-venv/bin/pip" install dist/*.whl + for example in example/*.json; do + "$RUNNER_TEMP/wheel-venv/bin/bust" "$example" -o "$RUNNER_TEMP/cpython" -p + done + - name: Generate the examples with Pyodide + working-directory: scripts/pyodide + run: | + npm ci + node run.mjs ../../dist/*.whl "$RUNNER_TEMP/pyodide" + - name: Compare + # The header line holds the minute each run generated the file. + run: diff -r -I 'Generated by bust ' "$RUNNER_TEMP/cpython" "$RUNNER_TEMP/pyodide" diff --git a/.gitignore b/.gitignore index db556d0b..6ee028d2 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,6 @@ example/*/doc/*.pdf # bust writes its log to the working directory debug.log + +# npm install in scripts/pyodide +node_modules/ diff --git a/CHANGELOG.md b/CHANGELOG.md index bb0b2f8d..21d7b11f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ the git log and on [PyPI](https://pypi.org/project/bust/#history). - `doc/.md`, the documentation as Markdown, with the register list and a bits table per register, readable on GitHub. - The documentation hosts the register map JSON Schema at https://bust.readthedocs.io/en/stable/register-map.schema.json. A register map may point to it with a `"$schema"` key, which gives completion, hover text and errors in editors with JSON Schema support. The editor adds the key to new register maps. - `bust.bundle_zip` turns the result of `generate_files` into one zip, in the folder layout the CLI writes, with a `README.md` that lists what each file is. +- bust runs under Pyodide in the browser. Getting started lists the dependencies to install there, and CI checks that the wheel generates every example under Pyodide as it does under CPython. ### Changed - The documentation and the editor overview show the bits that no field covers as `reserved` instead of `unused`, and list them among the fields with "Read as 0, writes have no effect." diff --git a/docs/development.md b/docs/development.md index 21369605..faef058d 100644 --- a/docs/development.md +++ b/docs/development.md @@ -27,9 +27,21 @@ bust runs on Python 3.9 to 3.14. mypy checks against 3.9. CI runs mypy and the tests on Python 3.9 to 3.14 (plus the next Python while it is in pre-release), builds the docs, builds the package and smoke-tests the -wheel, runs the pre-commit hooks and simulates the generated VHDL. A pull +wheel, checks that the wheel generates the same files under Pyodide as under +CPython, runs the pre-commit hooks and simulates the generated VHDL. A pull request needs all of it green. +To run the Pyodide check locally, with Node.js 18 or newer: + +```bash +python -m build -w +cd scripts/pyodide +npm ci +node run.mjs ../../dist/bust-*.whl /tmp/bust-pyodide +``` + +`tests/test_pure_python.py` fails if a runtime dependency is not pure Python, since Pyodide cannot install it. + ## Golden files Most tests regenerate the outputs of the examples in `example/` and compare diff --git a/docs/getting-started.md b/docs/getting-started.md index 2b904e59..ef4ecd6b 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -13,6 +13,19 @@ pip install bust bust --version ``` +### In the browser + +bust runs under [Pyodide](https://pyodide.org), Python in the browser. Install it without its dependencies and add the ones that generating code needs: + +```python +import micropip + +await micropip.install(["json5", "Jinja2", "pylatexenc"]) +await micropip.install("bust", deps=False) +``` + +That leaves out Textual, which only the editor (`-c`, `-e`) uses, and docopt, which only the command line uses. Generate the files with [`bust.generate_files`](command-line.md#from-python). + ## Describe your registers A register map is one JSON file with a `bus` section and a `module` section. diff --git a/scripts/pyodide/generate_examples.py b/scripts/pyodide/generate_examples.py new file mode 100644 index 00000000..eddf3b71 --- /dev/null +++ b/scripts/pyodide/generate_examples.py @@ -0,0 +1,26 @@ +"""Generate every example register map with bust as Pyodide runs it. + +run.mjs installs bust without its dependencies and imports this module. Only +the core dependencies are present, so generation fails if it needs any other. +""" + +import glob +import os + +import bust + + +def generate_examples(example_dir: str, output_dir: str) -> None: + """Write what ``bust FILE -o output_dir -p`` writes for each example.""" + json_paths = sorted(glob.glob(os.path.join(example_dir, "*.json"))) + if not json_paths: + raise FileNotFoundError(f"No register maps in {example_dir}") + for json_path in json_paths: + print(f"Generating {json_path}") + with open(json_path, encoding="utf-8", newline="") as fp: + files = bust.generate_files(fp.read()) + for relative_path, content in files.items(): + path = os.path.join(output_dir, relative_path) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as fp: + fp.write(content) diff --git a/scripts/pyodide/package-lock.json b/scripts/pyodide/package-lock.json new file mode 100644 index 00000000..f303028a --- /dev/null +++ b/scripts/pyodide/package-lock.json @@ -0,0 +1,53 @@ +{ + "name": "bust-pyodide-check", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "bust-pyodide-check", + "dependencies": { + "pyodide": "314.0.7" + } + }, + "node_modules/@types/emscripten": { + "version": "1.41.6", + "resolved": "https://registry.npmjs.org/@types/emscripten/-/emscripten-1.41.6.tgz", + "integrity": "sha512-uN+9i8bFT5CUcZfyIEYDrSueACEyKGbUs5kC/72DGlZZoinh84sJfVV0i8UOJD1asdzkvLPBRrKs41kZ8MdEXg==", + "license": "MIT" + }, + "node_modules/pyodide": { + "version": "314.0.7", + "resolved": "https://registry.npmjs.org/pyodide/-/pyodide-314.0.7.tgz", + "integrity": "sha512-0YvXxEhfEdpLfb/XkM2BFAeMROq0iMUX2bzzH9pOttyMcWkwq+HbE5uyuGD82LN7y2q+SNvi/6V5JEsOlD2R1A==", + "license": "MPL-2.0", + "dependencies": { + "@types/emscripten": "^1.41.4", + "ws": "^8.5.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/ws": { + "version": "8.22.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.22.0.tgz", + "integrity": "sha512-Ydggc987+RO0AnWtZ/7Wq9FtNvcrL1b/RO0ud9mWjUPgDrsAAwQSF51sm2hm1XofbU/4jkpGEsLFsZZxU+1DOg==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + } + } +} diff --git a/scripts/pyodide/package.json b/scripts/pyodide/package.json new file mode 100644 index 00000000..12e249d8 --- /dev/null +++ b/scripts/pyodide/package.json @@ -0,0 +1,9 @@ +{ + "name": "bust-pyodide-check", + "private": true, + "description": "Runs bust under Pyodide, as the browser does", + "type": "module", + "dependencies": { + "pyodide": "314.0.7" + } +} diff --git a/scripts/pyodide/run.mjs b/scripts/pyodide/run.mjs new file mode 100644 index 00000000..1ed3c25b --- /dev/null +++ b/scripts/pyodide/run.mjs @@ -0,0 +1,41 @@ +// Install a bust wheel in Pyodide the way the browser does and generate every +// example into OUTPUT_DIR, as `bust FILE -o OUTPUT_DIR -p` would. +// Usage: node scripts/pyodide/run.mjs WHEEL OUTPUT_DIR +import { mkdirSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { loadPyodide } from "pyodide"; + +// The set docs/getting-started.md lists. The browser leaves out Textual (the +// editor) and docopt (the command line, and PyPI has no wheel of it). +const CORE_DEPENDENCIES = ["json5", "Jinja2", "pylatexenc"]; + +const [wheel, outputDir] = process.argv.slice(2); +if (!wheel || !outputDir) { + console.error("Usage: node scripts/pyodide/run.mjs WHEEL OUTPUT_DIR"); + process.exit(2); +} +const repo = fileURLToPath(new URL("../..", import.meta.url)); +mkdirSync(outputDir, { recursive: true }); + +const pyodide = await loadPyodide(); +const mounts = { + "/repo": repo, + "/dist": path.dirname(path.resolve(wheel)), + "/out": path.resolve(outputDir), +}; +for (const [mountPoint, root] of Object.entries(mounts)) { + pyodide.FS.mkdirTree(mountPoint); + pyodide.FS.mount(pyodide.FS.filesystems.NODEFS, { root }, mountPoint); +} + +await pyodide.loadPackage("micropip"); +const micropip = pyodide.pyimport("micropip"); +await micropip.install(CORE_DEPENDENCIES); +await micropip.install.callKwargs(`emfs:/dist/${path.basename(wheel)}`, { + deps: false, +}); + +const sys = pyodide.pyimport("sys"); +sys.path.append("/repo/scripts/pyodide"); +pyodide.pyimport("generate_examples").generate_examples("/repo/example", "/out"); diff --git a/tests/test_pure_python.py b/tests/test_pure_python.py new file mode 100644 index 00000000..2e52ec25 --- /dev/null +++ b/tests/test_pure_python.py @@ -0,0 +1,59 @@ +"""bust runs in the browser under Pyodide, which installs only pure Python +wheels from PyPI. Every runtime dependency must stay pure Python.""" + +import email +from importlib import metadata +from typing import Dict, List + +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name + +# Pyodide ships its own build of MarkupSafe, the one Jinja2 needs. MarkupSafe +# also runs without its C speedups, so it works everywhere else too. +PYODIDE_BUILDS = {"markupsafe"} + + +def runtime_requirements(name: str) -> List[Requirement]: + requirements = [Requirement(line) for line in metadata.requires(name) or []] + return [ + requirement + for requirement in requirements + if requirement.marker is None or requirement.marker.evaluate({"extra": ""}) + ] + + +def runtime_dependencies() -> Dict[str, metadata.Distribution]: + """Every distribution bust needs at run time, by canonical name.""" + found: Dict[str, metadata.Distribution] = {} + pending = runtime_requirements("bust") + while pending: + name = canonicalize_name(pending.pop().name) + if name not in found: + found[name] = metadata.distribution(name) + pending.extend(runtime_requirements(name)) + return found + + +def is_pure_python(distribution: metadata.Distribution) -> bool: + wheel = email.message_from_string(distribution.read_text("WHEEL") or "") + tags = wheel.get_all("Tag") or [] + return wheel["Root-Is-Purelib"] == "true" and all( + tag.endswith("-none-any") for tag in tags + ) + + +def test_the_walk_reaches_indirect_dependencies(): + dependencies = runtime_dependencies() + assert {"jinja2", "markupsafe", "textual", "rich"} <= dependencies.keys() + + +def test_every_runtime_dependency_is_pure_python(): + impure = [ + name + for name, distribution in sorted(runtime_dependencies().items()) + if name not in PYODIDE_BUILDS and not is_pure_python(distribution) + ] + assert impure == [], ( + f"{', '.join(impure)} is not pure Python, so bust cannot install it " + "in the browser under Pyodide" + ) From b95fe99d4aabc85c9fd177deb9a91f9eb060fd2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ola=20Gr=C3=B8ttvik?= Date: Thu, 29 Oct 2026 15:07:44 +0000 Subject: [PATCH 2/2] Fold Pyodide check into build job and derive its dependencies --- .github/workflows/python_tests.yml | 34 +++++++++------------------- scripts/pyodide/generate_examples.py | 8 +++---- scripts/pyodide/run.mjs | 31 +++++++++++++++++++------ 3 files changed, 39 insertions(+), 34 deletions(-) diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml index e31ce12f..43fdfa6a 100644 --- a/.github/workflows/python_tests.yml +++ b/.github/workflows/python_tests.yml @@ -94,37 +94,25 @@ jobs: # -I keeps the checkout off sys.path, so this reads the installed package. version="$("$RUNNER_TEMP/wheel-venv/bin/python" -I -c 'from bust._version import __VERSION__; print(__VERSION__)')" PATH="$RUNNER_TEMP/wheel-venv/bin:$PATH" scripts/smoke-test.sh "$version" - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: dist - path: dist/* - - # cloudbust runs bust in the browser. Install the wheel in Pyodide with only - # the core dependencies and check that it writes what CPython writes. - pyodide: - runs-on: ubuntu-latest - needs: build - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.11" - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: dist - path: dist + # cloudbust runs bust in the browser. Install the wheel in Pyodide with only + # the core dependencies and check that it writes what CPython writes. - name: Generate the examples with CPython + env: + SOURCE_DATE_EPOCH: 0 run: | - python -m venv "$RUNNER_TEMP/wheel-venv" - "$RUNNER_TEMP/wheel-venv/bin/pip" install dist/*.whl for example in example/*.json; do "$RUNNER_TEMP/wheel-venv/bin/bust" "$example" -o "$RUNNER_TEMP/cpython" -p done - name: Generate the examples with Pyodide working-directory: scripts/pyodide + env: + SOURCE_DATE_EPOCH: 0 run: | npm ci node run.mjs ../../dist/*.whl "$RUNNER_TEMP/pyodide" - name: Compare - # The header line holds the minute each run generated the file. - run: diff -r -I 'Generated by bust ' "$RUNNER_TEMP/cpython" "$RUNNER_TEMP/pyodide" + run: diff -r "$RUNNER_TEMP/cpython" "$RUNNER_TEMP/pyodide" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: dist + path: dist/* diff --git a/scripts/pyodide/generate_examples.py b/scripts/pyodide/generate_examples.py index eddf3b71..578cd348 100644 --- a/scripts/pyodide/generate_examples.py +++ b/scripts/pyodide/generate_examples.py @@ -8,6 +8,7 @@ import os import bust +from bust.utils.file_utils import write_string_to_file def generate_examples(example_dir: str, output_dir: str) -> None: @@ -20,7 +21,6 @@ def generate_examples(example_dir: str, output_dir: str) -> None: with open(json_path, encoding="utf-8", newline="") as fp: files = bust.generate_files(fp.read()) for relative_path, content in files.items(): - path = os.path.join(output_dir, relative_path) - os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, "w", encoding="utf-8") as fp: - fp.write(content) + directory, filename = os.path.split(os.path.join(output_dir, relative_path)) + os.makedirs(directory, exist_ok=True) + write_string_to_file(content, filename, directory, True) diff --git a/scripts/pyodide/run.mjs b/scripts/pyodide/run.mjs index 1ed3c25b..010644ea 100644 --- a/scripts/pyodide/run.mjs +++ b/scripts/pyodide/run.mjs @@ -6,10 +6,6 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import { loadPyodide } from "pyodide"; -// The set docs/getting-started.md lists. The browser leaves out Textual (the -// editor) and docopt (the command line, and PyPI has no wheel of it). -const CORE_DEPENDENCIES = ["json5", "Jinja2", "pylatexenc"]; - const [wheel, outputDir] = process.argv.slice(2); if (!wheel || !outputDir) { console.error("Usage: node scripts/pyodide/run.mjs WHEEL OUTPUT_DIR"); @@ -18,7 +14,12 @@ if (!wheel || !outputDir) { const repo = fileURLToPath(new URL("../..", import.meta.url)); mkdirSync(outputDir, { recursive: true }); -const pyodide = await loadPyodide(); +// SOURCE_DATE_EPOCH pins the time bust writes into the files. +const env = { HOME: "/home/pyodide" }; +if (process.env.SOURCE_DATE_EPOCH !== undefined) { + env.SOURCE_DATE_EPOCH = process.env.SOURCE_DATE_EPOCH; +} +const pyodide = await loadPyodide({ env }); const mounts = { "/repo": repo, "/dist": path.dirname(path.resolve(wheel)), @@ -29,12 +30,28 @@ for (const [mountPoint, root] of Object.entries(mounts)) { pyodide.FS.mount(pyodide.FS.filesystems.NODEFS, { root }, mountPoint); } -await pyodide.loadPackage("micropip"); +await pyodide.loadPackage(["micropip", "packaging"]); const micropip = pyodide.pyimport("micropip"); -await micropip.install(CORE_DEPENDENCIES); await micropip.install.callKwargs(`emfs:/dist/${path.basename(wheel)}`, { deps: false, }); +// The browser leaves out Textual (the editor) and docopt (the command line, +// and PyPI has no wheel of it). docs/getting-started.md lists the rest. +const coreRequirements = pyodide + .runPython( + ` +from importlib.metadata import requires +from packaging.requirements import Requirement + +[ + str(requirement) + for requirement in map(Requirement, requires("bust")) + if requirement.marker is None and requirement.name not in ("textual", "docopt") +] +`, + ) + .toJs(); +await micropip.install(coreRequirements); const sys = pyodide.pyimport("sys"); sys.path.append("/repo/scripts/pyodide");