diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml index 8c1865c..43fdfa6 100644 --- a/.github/workflows/python_tests.yml +++ b/.github/workflows/python_tests.yml @@ -94,6 +94,24 @@ jobs: # -I keeps the checkout off sys.path, so this reads the installed package. version="$("$RUNNER_TEMP/wheel-venv/bin/python" -I -c 'from bust._version import __VERSION__; print(__VERSION__)')" PATH="$RUNNER_TEMP/wheel-venv/bin:$PATH" scripts/smoke-test.sh "$version" + # cloudbust runs bust in the browser. Install the wheel in Pyodide with only + # the core dependencies and check that it writes what CPython writes. + - name: Generate the examples with CPython + env: + SOURCE_DATE_EPOCH: 0 + run: | + for example in example/*.json; do + "$RUNNER_TEMP/wheel-venv/bin/bust" "$example" -o "$RUNNER_TEMP/cpython" -p + done + - name: Generate the examples with Pyodide + working-directory: scripts/pyodide + env: + SOURCE_DATE_EPOCH: 0 + run: | + npm ci + node run.mjs ../../dist/*.whl "$RUNNER_TEMP/pyodide" + - name: Compare + run: diff -r "$RUNNER_TEMP/cpython" "$RUNNER_TEMP/pyodide" - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: dist diff --git a/.gitignore b/.gitignore index db556d0..6ee028d 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,6 @@ example/*/doc/*.pdf # bust writes its log to the working directory debug.log + +# npm install in scripts/pyodide +node_modules/ diff --git a/CHANGELOG.md b/CHANGELOG.md index bb0b2f8..21d7b11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ the git log and on [PyPI](https://pypi.org/project/bust/#history). - `doc/.md`, the documentation as Markdown, with the register list and a bits table per register, readable on GitHub. - The documentation hosts the register map JSON Schema at https://bust.readthedocs.io/en/stable/register-map.schema.json. A register map may point to it with a `"$schema"` key, which gives completion, hover text and errors in editors with JSON Schema support. The editor adds the key to new register maps. - `bust.bundle_zip` turns the result of `generate_files` into one zip, in the folder layout the CLI writes, with a `README.md` that lists what each file is. +- bust runs under Pyodide in the browser. Getting started lists the dependencies to install there, and CI checks that the wheel generates every example under Pyodide as it does under CPython. ### Changed - The documentation and the editor overview show the bits that no field covers as `reserved` instead of `unused`, and list them among the fields with "Read as 0, writes have no effect." diff --git a/docs/development.md b/docs/development.md index 2136960..faef058 100644 --- a/docs/development.md +++ b/docs/development.md @@ -27,9 +27,21 @@ bust runs on Python 3.9 to 3.14. mypy checks against 3.9. CI runs mypy and the tests on Python 3.9 to 3.14 (plus the next Python while it is in pre-release), builds the docs, builds the package and smoke-tests the -wheel, runs the pre-commit hooks and simulates the generated VHDL. A pull +wheel, checks that the wheel generates the same files under Pyodide as under +CPython, runs the pre-commit hooks and simulates the generated VHDL. A pull request needs all of it green. +To run the Pyodide check locally, with Node.js 18 or newer: + +```bash +python -m build -w +cd scripts/pyodide +npm ci +node run.mjs ../../dist/bust-*.whl /tmp/bust-pyodide +``` + +`tests/test_pure_python.py` fails if a runtime dependency is not pure Python, since Pyodide cannot install it. + ## Golden files Most tests regenerate the outputs of the examples in `example/` and compare diff --git a/docs/getting-started.md b/docs/getting-started.md index 2b904e5..ef4ecd6 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -13,6 +13,19 @@ pip install bust bust --version ``` +### In the browser + +bust runs under [Pyodide](https://pyodide.org), Python in the browser. Install it without its dependencies and add the ones that generating code needs: + +```python +import micropip + +await micropip.install(["json5", "Jinja2", "pylatexenc"]) +await micropip.install("bust", deps=False) +``` + +That leaves out Textual, which only the editor (`-c`, `-e`) uses, and docopt, which only the command line uses. Generate the files with [`bust.generate_files`](command-line.md#from-python). + ## Describe your registers A register map is one JSON file with a `bus` section and a `module` section. diff --git a/scripts/pyodide/generate_examples.py b/scripts/pyodide/generate_examples.py new file mode 100644 index 0000000..578cd34 --- /dev/null +++ b/scripts/pyodide/generate_examples.py @@ -0,0 +1,26 @@ +"""Generate every example register map with bust as Pyodide runs it. + +run.mjs installs bust without its dependencies and imports this module. Only +the core dependencies are present, so generation fails if it needs any other. +""" + +import glob +import os + +import bust +from bust.utils.file_utils import write_string_to_file + + +def generate_examples(example_dir: str, output_dir: str) -> None: + """Write what ``bust FILE -o output_dir -p`` writes for each example.""" + json_paths = sorted(glob.glob(os.path.join(example_dir, "*.json"))) + if not json_paths: + raise FileNotFoundError(f"No register maps in {example_dir}") + for json_path in json_paths: + print(f"Generating {json_path}") + with open(json_path, encoding="utf-8", newline="") as fp: + files = bust.generate_files(fp.read()) + for relative_path, content in files.items(): + directory, filename = os.path.split(os.path.join(output_dir, relative_path)) + os.makedirs(directory, exist_ok=True) + write_string_to_file(content, filename, directory, True) diff --git a/scripts/pyodide/package-lock.json b/scripts/pyodide/package-lock.json new file mode 100644 index 0000000..f303028 --- /dev/null +++ b/scripts/pyodide/package-lock.json @@ -0,0 +1,53 @@ +{ + "name": "bust-pyodide-check", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "bust-pyodide-check", + "dependencies": { + "pyodide": "314.0.7" + } + }, + "node_modules/@types/emscripten": { + "version": "1.41.6", + "resolved": "https://registry.npmjs.org/@types/emscripten/-/emscripten-1.41.6.tgz", + "integrity": "sha512-uN+9i8bFT5CUcZfyIEYDrSueACEyKGbUs5kC/72DGlZZoinh84sJfVV0i8UOJD1asdzkvLPBRrKs41kZ8MdEXg==", + "license": "MIT" + }, + "node_modules/pyodide": { + "version": "314.0.7", + "resolved": "https://registry.npmjs.org/pyodide/-/pyodide-314.0.7.tgz", + "integrity": "sha512-0YvXxEhfEdpLfb/XkM2BFAeMROq0iMUX2bzzH9pOttyMcWkwq+HbE5uyuGD82LN7y2q+SNvi/6V5JEsOlD2R1A==", + "license": "MPL-2.0", + "dependencies": { + "@types/emscripten": "^1.41.4", + "ws": "^8.5.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/ws": { + "version": "8.22.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.22.0.tgz", + "integrity": "sha512-Ydggc987+RO0AnWtZ/7Wq9FtNvcrL1b/RO0ud9mWjUPgDrsAAwQSF51sm2hm1XofbU/4jkpGEsLFsZZxU+1DOg==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + } + } +} diff --git a/scripts/pyodide/package.json b/scripts/pyodide/package.json new file mode 100644 index 0000000..12e249d --- /dev/null +++ b/scripts/pyodide/package.json @@ -0,0 +1,9 @@ +{ + "name": "bust-pyodide-check", + "private": true, + "description": "Runs bust under Pyodide, as the browser does", + "type": "module", + "dependencies": { + "pyodide": "314.0.7" + } +} diff --git a/scripts/pyodide/run.mjs b/scripts/pyodide/run.mjs new file mode 100644 index 0000000..010644e --- /dev/null +++ b/scripts/pyodide/run.mjs @@ -0,0 +1,58 @@ +// Install a bust wheel in Pyodide the way the browser does and generate every +// example into OUTPUT_DIR, as `bust FILE -o OUTPUT_DIR -p` would. +// Usage: node scripts/pyodide/run.mjs WHEEL OUTPUT_DIR +import { mkdirSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { loadPyodide } from "pyodide"; + +const [wheel, outputDir] = process.argv.slice(2); +if (!wheel || !outputDir) { + console.error("Usage: node scripts/pyodide/run.mjs WHEEL OUTPUT_DIR"); + process.exit(2); +} +const repo = fileURLToPath(new URL("../..", import.meta.url)); +mkdirSync(outputDir, { recursive: true }); + +// SOURCE_DATE_EPOCH pins the time bust writes into the files. +const env = { HOME: "/home/pyodide" }; +if (process.env.SOURCE_DATE_EPOCH !== undefined) { + env.SOURCE_DATE_EPOCH = process.env.SOURCE_DATE_EPOCH; +} +const pyodide = await loadPyodide({ env }); +const mounts = { + "/repo": repo, + "/dist": path.dirname(path.resolve(wheel)), + "/out": path.resolve(outputDir), +}; +for (const [mountPoint, root] of Object.entries(mounts)) { + pyodide.FS.mkdirTree(mountPoint); + pyodide.FS.mount(pyodide.FS.filesystems.NODEFS, { root }, mountPoint); +} + +await pyodide.loadPackage(["micropip", "packaging"]); +const micropip = pyodide.pyimport("micropip"); +await micropip.install.callKwargs(`emfs:/dist/${path.basename(wheel)}`, { + deps: false, +}); +// The browser leaves out Textual (the editor) and docopt (the command line, +// and PyPI has no wheel of it). docs/getting-started.md lists the rest. +const coreRequirements = pyodide + .runPython( + ` +from importlib.metadata import requires +from packaging.requirements import Requirement + +[ + str(requirement) + for requirement in map(Requirement, requires("bust")) + if requirement.marker is None and requirement.name not in ("textual", "docopt") +] +`, + ) + .toJs(); +await micropip.install(coreRequirements); + +const sys = pyodide.pyimport("sys"); +sys.path.append("/repo/scripts/pyodide"); +pyodide.pyimport("generate_examples").generate_examples("/repo/example", "/out"); diff --git a/tests/test_pure_python.py b/tests/test_pure_python.py new file mode 100644 index 0000000..2e52ec2 --- /dev/null +++ b/tests/test_pure_python.py @@ -0,0 +1,59 @@ +"""bust runs in the browser under Pyodide, which installs only pure Python +wheels from PyPI. Every runtime dependency must stay pure Python.""" + +import email +from importlib import metadata +from typing import Dict, List + +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name + +# Pyodide ships its own build of MarkupSafe, the one Jinja2 needs. MarkupSafe +# also runs without its C speedups, so it works everywhere else too. +PYODIDE_BUILDS = {"markupsafe"} + + +def runtime_requirements(name: str) -> List[Requirement]: + requirements = [Requirement(line) for line in metadata.requires(name) or []] + return [ + requirement + for requirement in requirements + if requirement.marker is None or requirement.marker.evaluate({"extra": ""}) + ] + + +def runtime_dependencies() -> Dict[str, metadata.Distribution]: + """Every distribution bust needs at run time, by canonical name.""" + found: Dict[str, metadata.Distribution] = {} + pending = runtime_requirements("bust") + while pending: + name = canonicalize_name(pending.pop().name) + if name not in found: + found[name] = metadata.distribution(name) + pending.extend(runtime_requirements(name)) + return found + + +def is_pure_python(distribution: metadata.Distribution) -> bool: + wheel = email.message_from_string(distribution.read_text("WHEEL") or "") + tags = wheel.get_all("Tag") or [] + return wheel["Root-Is-Purelib"] == "true" and all( + tag.endswith("-none-any") for tag in tags + ) + + +def test_the_walk_reaches_indirect_dependencies(): + dependencies = runtime_dependencies() + assert {"jinja2", "markupsafe", "textual", "rich"} <= dependencies.keys() + + +def test_every_runtime_dependency_is_pure_python(): + impure = [ + name + for name, distribution in sorted(runtime_dependencies().items()) + if name not in PYODIDE_BUILDS and not is_pure_python(distribution) + ] + assert impure == [], ( + f"{', '.join(impure)} is not pure Python, so bust cannot install it " + "in the browser under Pyodide" + )