From 4c9c07ad38fd7628852be8c5460af10fa3dcaaeb Mon Sep 17 00:00:00 2001 From: paugier Date: Wed, 14 May 2025 17:18:44 +0200 Subject: [PATCH 1/8] Helper microbench --- .gitignore | 4 ++++ microbench/Makefile | 37 ++++++++++++++++++++++++++++++ microbench/pixi.toml | 12 ++++++++++ microbench/print_other_vs_cpy.py | 39 ++++++++++++++++++++++++++++++++ microbench/pyproject.toml | 22 ++++++++++++++++++ microbench/setup.py | 6 +++-- 6 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 microbench/Makefile create mode 100644 microbench/pixi.toml create mode 100644 microbench/print_other_vs_cpy.py create mode 100644 microbench/pyproject.toml diff --git a/.gitignore b/.gitignore index 287706698..dc3f00389 100644 --- a/.gitignore +++ b/.gitignore @@ -90,3 +90,7 @@ docs/_build/ # vscode .vscode + +microbench/pixi.lock +microbench/.venv_* +microbench/tmp_* diff --git a/microbench/Makefile b/microbench/Makefile new file mode 100644 index 000000000..d7140b386 --- /dev/null +++ b/microbench/Makefile @@ -0,0 +1,37 @@ +ifeq ($(PYTHON),) +PYTHON := python3 +endif + +install: + $(PYTHON) -m pip install . + +install-univ: + $(PYTHON) -m pip install . --config-settings="--global-option=--hpy-abi=universal" + +uninstall: + $(PYTHON) -m pip uninstall hpy.microbench + +test: + $(PYTHON) -m pytest -v | tee tmp_results_$(shell $(PYTHON) -c "import sys; print(sys.implementation.name)").txt + +clean: + rm -f src/*.so src/hpy_simple.py + +create_venv_cpy: + $(shell uv python find 3.12) -m venv .venv_cpy --upgrade-deps + +create_venv_pypy: + $(shell uv python find pypy) -m venv .venv_pypy --upgrade-deps + +create_venv_graalpy: + # cannot use --upgrade-deps because pip is patched for GraalPy + $(shell uv python find graalpy) -m venv .venv_graalpy + +print_other_vs_cpy: + @$(PYTHON) print_other_vs_cpy.py + +print_cpy: + @tail tmp_results_cpython.txt -n 30 + +print_pypy: + @tail tmp_results_pypy.txt -n 30 diff --git a/microbench/pixi.toml b/microbench/pixi.toml new file mode 100644 index 000000000..ea458f9e0 --- /dev/null +++ b/microbench/pixi.toml @@ -0,0 +1,12 @@ +[workspace] +authors = ["The HPy team "] +channels = ["conda-forge"] +name = "hpy.microbench" +platforms = ["linux-64"] +version = "0.1.0" + +[tasks] + +[dependencies] +gcc = ">=15.1.0,<15.2" +valgrind = ">=3.25.0,<4" diff --git a/microbench/print_other_vs_cpy.py b/microbench/print_other_vs_cpy.py new file mode 100644 index 000000000..0e27eb9bb --- /dev/null +++ b/microbench/print_other_vs_cpy.py @@ -0,0 +1,39 @@ +from pathlib import Path + +path_result_cpy = Path("tmp_results_cpython.txt") +path_result_other = Path("tmp_results_pypy.txt") + +assert path_result_cpy.exists() +assert path_result_other.exists() + + +def data_from_path(path, index_time): + txt = path.read_text() + _, txt = txt.split( + "================================== BENCHMARKS ==================================" + ) + lines = txt.splitlines()[3:-2] + + names = [] + times = [] + + for line in lines: + parts = line.split() + names.append(parts[0]) + times.append(float(parts[index_time])) + + return names, times + + +names, times_cpy = data_from_path(path_result_cpy, 1) +names, times_other = data_from_path(path_result_other, 3) + +max_length_name = 45 +fmt_name = f"{{:{max_length_name}s}}" + +print("# PyPy HPy univ / CPy native (time ratio, smaller is better)") + +for index, t_other in enumerate(times_other): + ratio = t_other / times_cpy[index] + name = fmt_name.format(names[index]) + print(f"{name} {ratio:.2f}") diff --git a/microbench/pyproject.toml b/microbench/pyproject.toml new file mode 100644 index 000000000..024825605 --- /dev/null +++ b/microbench/pyproject.toml @@ -0,0 +1,22 @@ +[project] +name = "hpy.microbench" +authors = [ + {name = "The HPy team", email = "hpy-dev@python.org"}, +] +version = "0.1.0" +description = "HPy microbenchmarks." +readme = "README.md" +keywords = ["hpy"] +requires-python = ">=3.8" +dependencies = [ + "hpy>=0.9.0; implementation_name == 'cpython'", + "pytest", + "cffi; implementation_name == 'cpython'", + ] + +[build-system] +requires = [ + "setuptools>=64.0", + "hpy>=0.9.0; implementation_name == 'cpython'", + "cffi", +] diff --git a/microbench/setup.py b/microbench/setup.py index 16988b0ae..a25c8cf53 100644 --- a/microbench/setup.py +++ b/microbench/setup.py @@ -1,8 +1,10 @@ from setuptools import setup, Extension setup( - name="hpy.microbench", - setup_requires=['cffi', 'hpy'], + # Workaround: HPy adds files to the sources list and uses absolute paths. + # Newer setuptools complain about that if package data should be included. + # Therefore, we explicitly disable this here. + include_package_data=False, ext_modules = [ Extension('cpy_simple', ['src/cpy_simple.c'], From 32f8732bb75749cad74d6fc4a4363b2891438e74 Mon Sep 17 00:00:00 2001 From: Pierre Augier Date: Wed, 14 May 2025 23:36:33 +0200 Subject: [PATCH 2/8] Helper microbench: compatibility GraalPy --- microbench/Makefile | 10 ++++++++-- microbench/print_other_vs_cpy.py | 11 +++++++++-- microbench/pyproject.toml | 2 +- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/microbench/Makefile b/microbench/Makefile index d7140b386..5dbf43b89 100644 --- a/microbench/Makefile +++ b/microbench/Makefile @@ -27,11 +27,17 @@ create_venv_graalpy: # cannot use --upgrade-deps because pip is patched for GraalPy $(shell uv python find graalpy) -m venv .venv_graalpy -print_other_vs_cpy: - @$(PYTHON) print_other_vs_cpy.py +print_pypy_vs_cpy: + @$(PYTHON) print_other_vs_cpy.py PyPy + +print_graalpy_vs_cpy: + @$(PYTHON) print_other_vs_cpy.py GraalPy print_cpy: @tail tmp_results_cpython.txt -n 30 print_pypy: @tail tmp_results_pypy.txt -n 30 + +print_graalpy: + @tail tmp_results_graalpy.txt -n 30 diff --git a/microbench/print_other_vs_cpy.py b/microbench/print_other_vs_cpy.py index 0e27eb9bb..6ddd3805a 100644 --- a/microbench/print_other_vs_cpy.py +++ b/microbench/print_other_vs_cpy.py @@ -1,7 +1,14 @@ +import sys + from pathlib import Path +try: + other = sys.argv[1] +except IndexError: + other = "PyPy" + path_result_cpy = Path("tmp_results_cpython.txt") -path_result_other = Path("tmp_results_pypy.txt") +path_result_other = Path(f"tmp_results_{other.lower()}.txt") assert path_result_cpy.exists() assert path_result_other.exists() @@ -31,7 +38,7 @@ def data_from_path(path, index_time): max_length_name = 45 fmt_name = f"{{:{max_length_name}s}}" -print("# PyPy HPy univ / CPy native (time ratio, smaller is better)") +print(f"# {other} HPy univ / CPy native (time ratio, smaller is better)") for index, t_other in enumerate(times_other): ratio = t_other / times_cpy[index] diff --git a/microbench/pyproject.toml b/microbench/pyproject.toml index 024825605..b8b50f26b 100644 --- a/microbench/pyproject.toml +++ b/microbench/pyproject.toml @@ -11,7 +11,7 @@ requires-python = ">=3.8" dependencies = [ "hpy>=0.9.0; implementation_name == 'cpython'", "pytest", - "cffi; implementation_name == 'cpython'", + "cffi; implementation_name != 'pypy'", ] [build-system] From 2ab1a082a827438f5e2352b6507a086b9cc3d9f8 Mon Sep 17 00:00:00 2001 From: Pierre Augier Date: Wed, 14 May 2025 23:59:11 +0200 Subject: [PATCH 3/8] microbench: more robust usage of UV --- microbench/Makefile | 3 +++ microbench/README.md | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/microbench/Makefile b/microbench/Makefile index 5dbf43b89..c0cb30fc7 100644 --- a/microbench/Makefile +++ b/microbench/Makefile @@ -18,12 +18,15 @@ clean: rm -f src/*.so src/hpy_simple.py create_venv_cpy: + uv python install 3.12 $(shell uv python find 3.12) -m venv .venv_cpy --upgrade-deps create_venv_pypy: + uv python install pypy $(shell uv python find pypy) -m venv .venv_pypy --upgrade-deps create_venv_graalpy: + uv python install graalpy # cannot use --upgrade-deps because pip is patched for GraalPy $(shell uv python find graalpy) -m venv .venv_graalpy diff --git a/microbench/README.md b/microbench/README.md index c25916088..4a1f7c951 100644 --- a/microbench/README.md +++ b/microbench/README.md @@ -1,7 +1,7 @@ To run the microbenchmarks -------------------------- -1. You need to have `hpy` installed in your virtuanenv. The easiest way +1. You need to have `hpy` installed in your virtualenv. The easiest way to do it is: $ cd /path/to/hpy From c444258a2c077e124268d53a46d0b0081bc64ead Mon Sep 17 00:00:00 2001 From: paugier Date: Thu, 15 May 2025 09:21:41 +0200 Subject: [PATCH 4/8] ci microbench: avoid setup.py call --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f68ace7e..8a73ba4a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -423,5 +423,5 @@ jobs: - name: Run microbenchmarks run: | cd microbench - python setup.py build_ext -i + python -m pip install . --no-build-isolation --no-deps python -m pytest -v From 5a61a1d28304c6d2ced4be55f818651edac26587 Mon Sep 17 00:00:00 2001 From: paugier Date: Thu, 15 May 2025 09:22:12 +0200 Subject: [PATCH 5/8] ci microbench: bench universal ABI --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a73ba4a5..506979c72 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -425,3 +425,9 @@ jobs: cd microbench python -m pip install . --no-build-isolation --no-deps python -m pytest -v + + - name: Run microbenchmarks universal ABI + run: | + python -m pip uninstall hpy.microbench --yes + python -m pip install -e . --no-build-isolation --no-deps --config-settings="--global-option=--hpy-abi=universal" + python -m pytest -v -m hpy From 48bf0e5983da55cf63688670082e641818b02752 Mon Sep 17 00:00:00 2001 From: paugier Date: Thu, 15 May 2025 10:27:22 +0200 Subject: [PATCH 6/8] microbench: update README and Makefile --- microbench/Makefile | 24 ++++-- microbench/README.md | 126 ++++++++++++++++++++++++++----- microbench/pixi.toml | 1 + microbench/print_other_vs_cpy.py | 15 +++- 4 files changed, 136 insertions(+), 30 deletions(-) diff --git a/microbench/Makefile b/microbench/Makefile index c0cb30fc7..72057f581 100644 --- a/microbench/Makefile +++ b/microbench/Makefile @@ -5,18 +5,26 @@ endif install: $(PYTHON) -m pip install . -install-univ: +install_universal: $(PYTHON) -m pip install . --config-settings="--global-option=--hpy-abi=universal" uninstall: - $(PYTHON) -m pip uninstall hpy.microbench + $(PYTHON) -m pip uninstall hpy.microbench --yes test: $(PYTHON) -m pytest -v | tee tmp_results_$(shell $(PYTHON) -c "import sys; print(sys.implementation.name)").txt +bench: test + +bench_hpy: + $(PYTHON) -m pytest -v -m hpy | tee tmp_results_$(shell $(PYTHON) -c "import sys; print(sys.implementation.name)").txt + clean: rm -f src/*.so src/hpy_simple.py +cleanall: clean + rm -rf .venv_* tmp_*.txt + create_venv_cpy: uv python install 3.12 $(shell uv python find 3.12) -m venv .venv_cpy --upgrade-deps @@ -30,12 +38,6 @@ create_venv_graalpy: # cannot use --upgrade-deps because pip is patched for GraalPy $(shell uv python find graalpy) -m venv .venv_graalpy -print_pypy_vs_cpy: - @$(PYTHON) print_other_vs_cpy.py PyPy - -print_graalpy_vs_cpy: - @$(PYTHON) print_other_vs_cpy.py GraalPy - print_cpy: @tail tmp_results_cpython.txt -n 30 @@ -44,3 +46,9 @@ print_pypy: print_graalpy: @tail tmp_results_graalpy.txt -n 30 + +print_pypy_vs_cpy: + @$(PYTHON) print_other_vs_cpy.py PyPy + +print_graalpy_vs_cpy: + @$(PYTHON) print_other_vs_cpy.py GraalPy diff --git a/microbench/README.md b/microbench/README.md index 4a1f7c951..6d49489a9 100644 --- a/microbench/README.md +++ b/microbench/README.md @@ -1,29 +1,117 @@ -To run the microbenchmarks --------------------------- +# To run the microbenchmarks -1. You need to have `hpy` installed in your virtualenv. The easiest way +## Non-Python dependencies + +The benchmarks depends on Valgrind, which can be installed with + +```sh +sudo apt update && sudo apt install -y valgrind +``` + +Alternatively, you can also use [Pixi] to get it in a new shell with + +```sh +pixi shell +``` + +Similarly, building with GraalPy requires `libffi` to build the Python package `ffi`. +`pixi shell` provides it. + +[UV] can be useful since it is used in few Makefile targets to install Python interpreters. + +## Python virtual environments + +We assume in the following that a virtual environment is activated. One can create +environments with the Makefile targets `create_venv_...` as + +```sh +cd /path/to/hpy/microbench +make create_venv_pypy +. .venv_pypy/bin/activate +``` + +## Non-editable install with build isolation + +One can build these microbenchmarks with HPy from PyPI (on CPython) or bundled with the Python implementation. + +```sh +pip install . +``` + +This builds the HPy extension with the CPython ABI for CPython and with the universal ABI for other implementations. +To build this extension with the universal ABI with CPython: + +```sh +pip install . --config-settings="--global-option=--hpy-abi=universal" +``` + +## Editable install without build isolation + +1. On CPython, you need to have `hpy` installed in your virtualenv. The easiest way to do it is: - $ cd /path/to/hpy - $ python setup.py develop + ```sh + cd /path/to/hpy + pip install -e . + ``` + +2. Install build and runtime dependencies + + ```sh + # cffi needed to build _valgrind + pip install cffi pytest + ``` + +3. Build and install the extension modules needed for the microbenchmarks + + ```sh + cd /path/to/hpy/microbench + pip install . --no-build-isolation + # or for the universal ABI (on with CPython) + rm -f src/*.so src/hpy_simple.py + pip install -e . --no-build-isolation --config-settings="--global-option=--hpy-abi=universal" + ``` + +## Run the benchmarks + +```sh +pytest -v +``` + +To run only cpy or hpy tests, use -m (to select markers): + +```sh +pytest -v -m hpy +pytest -v -m cpy +``` + +## Comparing alternative Python implementations to CPython -2. Build the extension modules needed for the microbenchmarks +One can run things like - $ cd /path/to/hpy/microbench - $ pip install cffi # needed to build _valgrind - $ python setup.py build_ext --inplace +```sh +make cleanall +pixi shell +make create_venv_cpy +make create_venv_pypy +make create_venv_graalpy -2. `py.test -v` +make install PYTHON=.venv_cpy/bin/python +make install PYTHON=.venv_pypy/bin/python +make install PYTHON=.venv_graalpy/bin/python -3. To run only cpy or hpy tests, use -m (to select markers): +make bench PYTHON=.venv_cpy/bin/python +make bench PYTHON=.venv_pypy/bin/python +# only HPy for GraalPy since the full benchmarks are a bit too long +make bench_hpy PYTHON=.venv_graalpy/bin/python - $ py.test -v -m hpy - $ py.test -v -m cpy +make print_cpy +make print_pypy +make print_graalpy -4. Step (2) build `hpy_simple` using the CPython ABI by default. If you want - to benchmark the universal mode, you need to build it explicitly: +make print_pypy_vs_cpy +make print_graalpy_vs_cpy +``` - $ cd /path/to/hpy/microbench - $ rm *.so # make sure to delete CPython-ABI versions - $ python setup.py --hpy-abi=universal build_ext --inplace - $ py.test -v +[Pixi]: https://pixi.sh +[UV]: https://docs.astral.sh/uv/ diff --git a/microbench/pixi.toml b/microbench/pixi.toml index ea458f9e0..83b1c4f6f 100644 --- a/microbench/pixi.toml +++ b/microbench/pixi.toml @@ -10,3 +10,4 @@ version = "0.1.0" [dependencies] gcc = ">=15.1.0,<15.2" valgrind = ">=3.25.0,<4" +libffi = ">=3.4.6,<4" diff --git a/microbench/print_other_vs_cpy.py b/microbench/print_other_vs_cpy.py index 6ddd3805a..b7ad0abea 100644 --- a/microbench/print_other_vs_cpy.py +++ b/microbench/print_other_vs_cpy.py @@ -14,13 +14,22 @@ assert path_result_other.exists() -def data_from_path(path, index_time): +def data_from_path(path): txt = path.read_text() _, txt = txt.split( "================================== BENCHMARKS ==================================" ) lines = txt.splitlines()[3:-2] + if "cpy" in path.name: + index_time = 1 + else: + parts = lines[0].split() + if len(parts) == 3: + index_time = 1 + else: + index_time = 3 + names = [] times = [] @@ -32,8 +41,8 @@ def data_from_path(path, index_time): return names, times -names, times_cpy = data_from_path(path_result_cpy, 1) -names, times_other = data_from_path(path_result_other, 3) +names, times_cpy = data_from_path(path_result_cpy) +names, times_other = data_from_path(path_result_other) max_length_name = 45 fmt_name = f"{{:{max_length_name}s}}" From 95e2a336853b9f1e653c01961386d3851afc9382 Mon Sep 17 00:00:00 2001 From: paugier Date: Thu, 15 May 2025 12:02:37 +0200 Subject: [PATCH 7/8] ci microbench: fix and update --- .github/workflows/ci.yml | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 506979c72..5f05e0b11 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -412,22 +412,32 @@ jobs: - name: Install / Upgrade Python requirements run: | - python -m pip install --upgrade pip wheel 'setuptools>=60.2' - python -m pip install pytest cffi + pip install --upgrade pip wheel 'setuptools>=60.2' setuptools-scm + pip install pytest cffi - name: Build and install HPy run: | make - python -m pip install . + pip install . + + - name: Build and install hpy.microbench + run: | + pip install ./microbench --no-build-isolation --no-deps - name: Run microbenchmarks run: | cd microbench - python -m pip install . --no-build-isolation --no-deps - python -m pytest -v + pytest -v + + - name: Uninstall hpy.microbench + run: | + pip uninstall hpy.microbench --yes + + - name: Build and install hpy.microbench universal ABI + run: | + pip install -e ./microbench --no-build-isolation --no-deps --config-settings="--global-option=--hpy-abi=universal" - name: Run microbenchmarks universal ABI run: | - python -m pip uninstall hpy.microbench --yes - python -m pip install -e . --no-build-isolation --no-deps --config-settings="--global-option=--hpy-abi=universal" - python -m pytest -v -m hpy + cd microbench + pytest -v -m hpy From b824d878fda8643e59ec899a156b915477b7ef9e Mon Sep 17 00:00:00 2001 From: paugier Date: Thu, 15 May 2025 13:19:27 +0200 Subject: [PATCH 8/8] microbench: improve print results --- microbench/Makefile | 9 ++++++--- microbench/print_other_vs_cpy.py | 6 +++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/microbench/Makefile b/microbench/Makefile index 72057f581..3044a8617 100644 --- a/microbench/Makefile +++ b/microbench/Makefile @@ -39,13 +39,16 @@ create_venv_graalpy: $(shell uv python find graalpy) -m venv .venv_graalpy print_cpy: - @tail tmp_results_cpython.txt -n 30 + @echo =================================== CPython ==================================== + @tail tmp_results_cpython.txt -n 29 print_pypy: - @tail tmp_results_pypy.txt -n 30 + @echo ==================================== PyPy ====================================== + @tail tmp_results_pypy.txt -n 29 print_graalpy: - @tail tmp_results_graalpy.txt -n 30 + @echo =================================== GraalPy ==================================== + @tail tmp_results_graalpy.txt -n 29 print_pypy_vs_cpy: @$(PYTHON) print_other_vs_cpy.py PyPy diff --git a/microbench/print_other_vs_cpy.py b/microbench/print_other_vs_cpy.py index b7ad0abea..a303d6f4a 100644 --- a/microbench/print_other_vs_cpy.py +++ b/microbench/print_other_vs_cpy.py @@ -47,7 +47,11 @@ def data_from_path(path): max_length_name = 45 fmt_name = f"{{:{max_length_name}s}}" -print(f"# {other} HPy univ / CPy native (time ratio, smaller is better)") +out = f" {other} HPy univ / CPy native (time ratio, smaller is better) " +num_chars = 81 +num_equals = (num_chars - len(out)) // 2 + +print("\n" + num_equals * "=" + out + num_equals * "=") for index, t_other in enumerate(times_other): ratio = t_other / times_cpy[index]