Skip to content

Commit 409621b

Browse files
soswowsoswow
authored
feat: Nanobind for python bindings (first steps -- pybind11 still working) (#5084)
First PR on the way to a migration from pybind11 to nanobind. New env flag introduced `OIIO_PYTHON_BINDINGS_BACKEND` = `pybind11` or `nanobind` or `both` > Select which Python binding backend(s) to configure. `both` keeps the existing pybind11 module and also builds the experimental nanobind module. When it is nanobine one is build it is in `PyOpenImageIONanobindExperimental` target. ``` cmake --fresh -S . -B build \ -DUSE_CCACHE=OFF \ -Dfmt_DIR=/opt/homebrew/lib/cmake/fmt \ -DOpenColorIO_DIR=/opt/homebrew/lib/cmake/OpenColorIO \ -DOIIO_INTERNALIZE_FMT=ON \ -DOIIO_PYTHON_BINDINGS_BACKEND=both ``` ### Checklist: <!-- Put an 'x' in the boxes as you complete the checklist items --> - [x] **I have read the guidelines** on [contributions](https://github.com/AcademySoftwareFoundation/OpenImageIO/blob/main/CONTRIBUTING.md) and [code review procedures](https://github.com/AcademySoftwareFoundation/OpenImageIO/blob/main/docs/dev/CodeReview.md). - [x] **I have updated the documentation** if my PR adds features or changes behavior. - [x] **I am sure that this PR's changes are tested somewhere in the testsuite**. - [x] **I have run and passed the testsuite in CI** *before* submitting the PR, by pushing the changes to my fork and seeing that the automated CI passed there. (Exceptions: If most tests pass and you can't figure out why the remaining ones fail, it's ok to submit the PR and ask for help. Or if any failures seem entirely unrelated to your change; sometimes things break on the GitHub runners.) - [x] **My code follows the prevailing code style of this project** and I fixed any problems reported by the clang-format CI test. - [x] If I added or modified a public C++ API call, I have also amended the corresponding Python bindings. If altering ImageBufAlgo functions, I also exposed the new functionality as oiiotool options. This code contribution was assisted by Cursor/Composer-2 --------- Signed-off-by: Aleksandr Motsjonov <soswow@gmail.com>
1 parent d1de954 commit 409621b

24 files changed

Lines changed: 2059 additions & 40 deletions

‎CMakeLists.txt‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -314,7 +314,12 @@ else ()
314314
set (_py_dev_found Python3_Development.Module_FOUND)
315315
endif ()
316316
if (USE_PYTHON AND ${_py_dev_found} AND NOT BUILD_OIIOUTIL_ONLY)
317-
add_subdirectory (src/python)
317+
if (OIIO_BUILD_PYTHON_PYBIND11)
318+
add_subdirectory (src/python)
319+
endif ()
320+
if (OIIO_BUILD_PYTHON_NANOBIND)
321+
add_subdirectory (src/python-nanobind)
322+
endif ()
318323
else ()
319324
message (STATUS "Not building Python bindings: USE_PYTHON=${USE_PYTHON}, Python3_Development.Module_FOUND=${Python3_Development.Module_FOUND}")
320325
endif ()

‎INSTALL.md‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,10 @@ NEW or CHANGED MINIMUM dependencies since the last major release are **bold**.
4242
* Python >= 3.9 (tested through 3.13).
4343
* pybind11 >= 2.7 (tested through 3.0)
4444
* NumPy (tested through 2.4.4)
45+
* If you enable the optional nanobind (WIP) backend for source/CMake
46+
builds (`OIIO_PYTHON_BINDINGS_BACKEND` is `nanobind` or `both`):
47+
* nanobind discoverable by CMake, or installed in the active Python
48+
environment so `python -m nanobind --cmake_dir` works
4549
* If you want support for PNG files:
4650
* libPNG >= 1.6.0 (tested though 1.6.56)
4751
* If you want support for camera "RAW" formats:
@@ -157,6 +161,12 @@ Make wrapper (`make PkgName_ROOT=...`).
157161

158162
`USE_PYTHON=0` : Omits building the Python bindings.
159163

164+
`OIIO_PYTHON_BINDINGS_BACKEND=pybind11|nanobind|both` : Select which Python
165+
binding backend(s) to configure for source/CMake builds. `both` keeps the
166+
existing pybind11 module and also builds the nanobind (WIP) module. The
167+
Python packaging path driven by `pyproject.toml` still targets the production
168+
pybind11 bindings today.
169+
160170
`OIIO_BUILD_TESTS=0` : Omits building tests (you probably don't need them
161171
unless you are a developer of OIIO or want to verify that your build
162172
passes all tests).
@@ -247,6 +257,7 @@ Additionally, a few helpful modifiers alter some build-time options:
247257
| make USE_QT=0 ... | Skip anything that needs Qt |
248258
| make MYCC=xx MYCXX=yy ... | Use custom compilers |
249259
| make USE_PYTHON=0 ... | Don't build the Python binding |
260+
| make OIIO_PYTHON_BINDINGS_BACKEND=both ... | For source/CMake builds, build the existing pybind11 bindings and the nanobind (WIP) module |
250261
| make BUILD_SHARED_LIBS=0 | Build static library instead of shared |
251262
| make IGNORE_HOMEBREWED_DEPS=1 | Ignore homebrew-managed dependencies |
252263
| make LINKSTATIC=1 ... | Link with static external libraries when possible |

‎src/cmake/externalpackages.cmake‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,9 +118,13 @@ endif()
118118
if (USE_PYTHON)
119119
find_python()
120120
endif ()
121-
if (USE_PYTHON)
121+
if (USE_PYTHON AND OIIO_BUILD_PYTHON_PYBIND11)
122122
checked_find_package (pybind11 REQUIRED VERSION_MIN 2.7)
123123
endif ()
124+
if (USE_PYTHON AND OIIO_BUILD_PYTHON_NANOBIND)
125+
discover_nanobind_cmake_dir()
126+
checked_find_package (nanobind CONFIG REQUIRED)
127+
endif ()
124128

125129

126130
###########################################################################

‎src/cmake/pythonutils.cmake‎

Lines changed: 123 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,31 @@ set (PYTHON_VERSION "" CACHE STRING "Target version of python to find")
88
option (PYLIB_INCLUDE_SONAME "If ON, soname/soversion will be set for Python module library" OFF)
99
option (PYLIB_LIB_PREFIX "If ON, prefix the Python module with 'lib'" OFF)
1010
set (PYMODULE_SUFFIX "" CACHE STRING "Suffix to add to Python module init namespace")
11+
set (OIIO_PYTHON_BINDINGS_BACKEND "pybind11" CACHE STRING
12+
"Which Python binding backend(s) to build: pybind11, nanobind, or both")
13+
set_property (CACHE OIIO_PYTHON_BINDINGS_BACKEND PROPERTY STRINGS
14+
pybind11 nanobind both)
15+
16+
# Normalize and validate the user-facing backend selector early so the rest
17+
# of the file can make simple boolean decisions.
18+
string (TOLOWER "${OIIO_PYTHON_BINDINGS_BACKEND}" OIIO_PYTHON_BINDINGS_BACKEND)
19+
if (NOT OIIO_PYTHON_BINDINGS_BACKEND MATCHES "^(pybind11|nanobind|both)$")
20+
message (FATAL_ERROR
21+
"OIIO_PYTHON_BINDINGS_BACKEND must be one of: pybind11, nanobind, both")
22+
endif ()
23+
24+
# Derive internal switches used by the top-level CMakeLists and the Python
25+
# helper macros below.
26+
set (OIIO_BUILD_PYTHON_PYBIND11 OFF)
27+
set (OIIO_BUILD_PYTHON_NANOBIND OFF)
28+
if (OIIO_PYTHON_BINDINGS_BACKEND STREQUAL "pybind11"
29+
OR OIIO_PYTHON_BINDINGS_BACKEND STREQUAL "both")
30+
set (OIIO_BUILD_PYTHON_PYBIND11 ON)
31+
endif ()
32+
if (OIIO_PYTHON_BINDINGS_BACKEND STREQUAL "nanobind"
33+
OR OIIO_PYTHON_BINDINGS_BACKEND STREQUAL "both")
34+
set (OIIO_BUILD_PYTHON_NANOBIND ON)
35+
endif ()
1136
if (WIN32)
1237
set (PYLIB_LIB_TYPE SHARED CACHE STRING "Type of library to build for python module (MODULE or SHARED)")
1338
else ()
@@ -54,6 +79,15 @@ macro (find_python)
5479
Python3_Development.Module_FOUND
5580
Python3_Interpreter_FOUND )
5681

82+
if (OIIO_BUILD_PYTHON_NANOBIND)
83+
# nanobind's CMake package expects the generic FindPython targets and
84+
# variables (Python::Module, Python_EXECUTABLE, etc.), not the
85+
# versioned Python3::* targets that the rest of OIIO uses today.
86+
find_package (Python ${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}
87+
EXACT REQUIRED
88+
COMPONENTS ${_py_components})
89+
endif ()
90+
5791
# The version that was found may not be the default or user
5892
# defined one.
5993
set (PYTHON_VERSION_FOUND ${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR})
@@ -63,15 +97,44 @@ macro (find_python)
6397
set (PythonInterp3_FIND_VERSION PYTHON_VERSION_FOUND)
6498
set (PythonInterp3_FIND_VERSION_MAJOR ${Python3_VERSION_MAJOR})
6599

100+
if (NOT DEFINED PYTHON_SITE_ROOT_DIR)
101+
set (PYTHON_SITE_ROOT_DIR
102+
"${CMAKE_INSTALL_LIBDIR}/python${PYTHON_VERSION_FOUND}/site-packages")
103+
endif ()
66104
if (NOT DEFINED PYTHON_SITE_DIR)
67-
set (PYTHON_SITE_DIR "${CMAKE_INSTALL_LIBDIR}/python${PYTHON_VERSION_FOUND}/site-packages/OpenImageIO")
105+
set (PYTHON_SITE_DIR "${PYTHON_SITE_ROOT_DIR}/OpenImageIO")
68106
endif ()
69107
message (VERBOSE " Python site packages dir ${PYTHON_SITE_DIR}")
108+
message (VERBOSE " Python site packages root ${PYTHON_SITE_ROOT_DIR}")
70109
message (VERBOSE " Python to include 'lib' prefix: ${PYLIB_LIB_PREFIX}")
71110
message (VERBOSE " Python to include SO version: ${PYLIB_INCLUDE_SONAME}")
72111
endmacro()
73112

74113

114+
# Help CMake locate nanobind when it was installed as a Python package.
115+
macro (discover_nanobind_cmake_dir)
116+
if (nanobind_DIR OR nanobind_ROOT OR "$ENV{nanobind_DIR}" OR "$ENV{nanobind_ROOT}")
117+
return()
118+
endif ()
119+
120+
if (NOT Python3_Interpreter_FOUND)
121+
return()
122+
endif ()
123+
124+
execute_process (
125+
COMMAND ${Python3_EXECUTABLE} -m nanobind --cmake_dir
126+
RESULT_VARIABLE _oiio_nanobind_result
127+
OUTPUT_VARIABLE _oiio_nanobind_cmake_dir
128+
OUTPUT_STRIP_TRAILING_WHITESPACE
129+
ERROR_QUIET)
130+
if (_oiio_nanobind_result EQUAL 0
131+
AND EXISTS "${_oiio_nanobind_cmake_dir}/nanobind-config.cmake")
132+
set (nanobind_DIR "${_oiio_nanobind_cmake_dir}" CACHE PATH
133+
"Path to the nanobind CMake package" FORCE)
134+
endif ()
135+
endmacro()
136+
137+
75138
###########################################################################
76139
# pybind11
77140

@@ -163,3 +226,62 @@ macro (setup_python_module)
163226

164227
endmacro ()
165228

229+
230+
###########################################################################
231+
# nanobind
232+
233+
macro (setup_python_module_nanobind)
234+
cmake_parse_arguments (lib "" "TARGET;MODULE"
235+
"SOURCES;LIBS;INCLUDES;SYSTEM_INCLUDE_DIRS;PACKAGE_FILES"
236+
${ARGN})
237+
238+
set (target_name ${lib_TARGET})
239+
240+
if (NOT COMMAND nanobind_add_module)
241+
discover_nanobind_cmake_dir()
242+
find_package (nanobind CONFIG REQUIRED)
243+
endif ()
244+
245+
nanobind_add_module(${target_name} ${lib_SOURCES})
246+
if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND TARGET nanobind-static)
247+
target_compile_options (nanobind-static PRIVATE -Wno-error=format-nonliteral)
248+
endif ()
249+
250+
target_include_directories (${target_name}
251+
PRIVATE ${lib_INCLUDES})
252+
target_include_directories (${target_name}
253+
SYSTEM PRIVATE ${lib_SYSTEM_INCLUDE_DIRS})
254+
target_link_libraries (${target_name}
255+
PRIVATE ${lib_LIBS})
256+
257+
set (_module_LINK_FLAGS "${VISIBILITY_MAP_COMMAND} ${EXTRA_DSO_LINK_ARGS}")
258+
if (UNIX AND NOT APPLE)
259+
set (_module_LINK_FLAGS "${_module_LINK_FLAGS} -Wl,--exclude-libs,ALL")
260+
endif ()
261+
set_target_properties (${target_name} PROPERTIES
262+
LINK_FLAGS ${_module_LINK_FLAGS}
263+
OUTPUT_NAME ${lib_MODULE}
264+
DEBUG_POSTFIX "")
265+
266+
if (SKBUILD)
267+
set (_nanobind_install_dir .)
268+
else ()
269+
set (_nanobind_install_dir ${PYTHON_SITE_DIR})
270+
endif ()
271+
272+
# Keep nanobind modules isolated in the build tree so they don't alter
273+
# how the existing top-level OpenImageIO module is imported during tests.
274+
set_target_properties (${target_name} PROPERTIES
275+
LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib/python/nanobind/OpenImageIO
276+
ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib/python/nanobind/OpenImageIO
277+
)
278+
279+
install (TARGETS ${target_name}
280+
RUNTIME DESTINATION ${_nanobind_install_dir} COMPONENT user
281+
LIBRARY DESTINATION ${_nanobind_install_dir} COMPONENT user)
282+
283+
if (lib_PACKAGE_FILES)
284+
install (FILES ${lib_PACKAGE_FILES}
285+
DESTINATION ${_nanobind_install_dir} COMPONENT user)
286+
endif ()
287+
endmacro ()

‎src/cmake/testing.cmake‎

Lines changed: 72 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,23 @@ set(OIIO_TESTSUITE_IMAGEDIR "${PROJECT_BINARY_DIR}/testsuite" CACHE PATH
2121
"Location of oiio-images, openexr-images, libtiffpic, etc.." )
2222

2323

24+
# Build a single ENVIRONMENT list entry "PYTHONPATH=..." for CTest.
25+
# On Windows, keep this deterministic and do not append inherited PYTHONPATH:
26+
# semicolon-separated values can be split by CMake list processing when used
27+
# as test ENVIRONMENT entries.
28+
function (oiio_tests_pythonpath_env_entry out_var prefix_dir)
29+
if (WIN32)
30+
set (_pythonpath "${prefix_dir}")
31+
else ()
32+
if (DEFINED ENV{PYTHONPATH} AND NOT "$ENV{PYTHONPATH}" STREQUAL "")
33+
string (CONCAT _pythonpath "${prefix_dir}" ":" "$ENV{PYTHONPATH}")
34+
else ()
35+
set (_pythonpath "${prefix_dir}")
36+
endif ()
37+
endif ()
38+
set (${out_var} "PYTHONPATH=${_pythonpath}" PARENT_SCOPE)
39+
endfunction ()
40+
2441

2542
# oiio_add_tests() - add a set of test cases.
2643
#
@@ -229,25 +246,60 @@ macro (oiio_add_all_tests)
229246
# Python interpreter itself won't be linked with the right asan
230247
# libraries to run correctly.
231248
if (USE_PYTHON AND NOT BUILD_OIIOUTIL_ONLY AND NOT SANITIZE)
232-
oiio_add_tests (
233-
docs-examples-python
234-
python-colorconfig
235-
python-deep
236-
python-imagebuf
237-
python-imagecache
238-
python-imageoutput
239-
python-imagespec
240-
python-paramlist
241-
python-roi
242-
python-texturesys
243-
python-typedesc
244-
filters
245-
)
246-
# These Python tests also need access to oiio-images
247-
oiio_add_tests (
248-
python-imageinput python-imagebufalgo
249-
IMAGEDIR oiio-images
250-
)
249+
if (WIN32)
250+
# On Windows CI we run the install target before tests. Use the
251+
# installed package path to avoid multi-config output layout quirks.
252+
set (_installed_python_site_packages
253+
"${CMAKE_INSTALL_PREFIX}/${PYTHON_SITE_ROOT_DIR}")
254+
oiio_tests_pythonpath_env_entry (_pybind_tests_pythonpath
255+
"${_installed_python_site_packages}")
256+
oiio_tests_pythonpath_env_entry (_nanobind_tests_pythonpath
257+
"${CMAKE_BINARY_DIR}/lib/python/nanobind/$<CONFIG>")
258+
else ()
259+
oiio_tests_pythonpath_env_entry (_pybind_tests_pythonpath
260+
"${CMAKE_BINARY_DIR}/lib/python/site-packages")
261+
oiio_tests_pythonpath_env_entry (_nanobind_tests_pythonpath
262+
"${CMAKE_BINARY_DIR}/lib/python/nanobind")
263+
endif ()
264+
set (nanobind_python_tests
265+
python-imagespec
266+
python-paramlist
267+
python-roi
268+
python-typedesc)
269+
set (nanobind_python_test_suffix ".nanobind")
270+
if (OIIO_BUILD_PYTHON_PYBIND11)
271+
oiio_add_tests (
272+
docs-examples-python
273+
python-colorconfig
274+
python-deep
275+
python-imagebuf
276+
python-imagecache
277+
python-imageoutput
278+
python-imagespec
279+
python-paramlist
280+
python-roi
281+
python-texturesys
282+
python-typedesc
283+
filters
284+
ENVIRONMENT "${_pybind_tests_pythonpath}"
285+
)
286+
# These Python tests also need access to oiio-images
287+
oiio_add_tests (
288+
python-imageinput python-imagebufalgo
289+
IMAGEDIR oiio-images
290+
ENVIRONMENT "${_pybind_tests_pythonpath}"
291+
)
292+
else ()
293+
set (nanobind_python_test_suffix "")
294+
endif ()
295+
296+
if (OIIO_BUILD_PYTHON_NANOBIND)
297+
oiio_add_tests (
298+
${nanobind_python_tests}
299+
SUFFIX ${nanobind_python_test_suffix}
300+
ENVIRONMENT "${_nanobind_tests_pythonpath}"
301+
)
302+
endif ()
251303
endif ()
252304

253305
oiio_add_tests (oiiotool-color
@@ -267,7 +319,7 @@ macro (oiio_add_all_tests)
267319

268320
oiio_add_tests ( python-imagebufalgo
269321
FOUNDVAR hwy_FOUND
270-
ENABLEVAR OIIO_USE_HWY USE_PYTHON
322+
ENABLEVAR OIIO_USE_HWY USE_PYTHON OIIO_BUILD_PYTHON_PYBIND11
271323
DISABLEVAR BUILD_OIIOUTIL_ONLY SANITIZE
272324
SUFFIX ".hwy"
273325
ENVIRONMENT "OPENIMAGEIO_ENABLE_HWY=1"

‎src/python-nanobind/CMakeLists.txt‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# Copyright Contributors to the OpenImageIO project.
2+
# SPDX-License-Identifier: Apache-2.0
3+
# https://github.com/AcademySoftwareFoundation/OpenImageIO
4+
5+
set (nanobind_srcs
6+
py_oiio.cpp
7+
py_paramvalue.cpp
8+
py_roi.cpp
9+
py_imagespec.cpp
10+
py_typedesc.cpp)
11+
12+
set (nanobind_build_package_dir ${CMAKE_BINARY_DIR}/lib/python/nanobind/OpenImageIO)
13+
file (MAKE_DIRECTORY ${nanobind_build_package_dir})
14+
configure_file (__init__.py
15+
${nanobind_build_package_dir}/__init__.py
16+
COPYONLY)
17+
18+
setup_python_module_nanobind (
19+
TARGET PyOpenImageIONanobind
20+
MODULE _OpenImageIO
21+
SOURCES ${nanobind_srcs}
22+
LIBS OpenImageIO
23+
)
24+
25+
if (OIIO_PYTHON_BINDINGS_BACKEND STREQUAL "nanobind")
26+
if (SKBUILD)
27+
install (FILES __init__.py DESTINATION . COMPONENT user)
28+
else ()
29+
install (FILES __init__.py
30+
DESTINATION ${PYTHON_SITE_DIR} COMPONENT user)
31+
endif ()
32+
endif ()

0 commit comments

Comments
 (0)