From a7288723e74e091fb0f49ac26c40d53b46931ad0 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Mon, 6 Jul 2026 13:48:02 -0700 Subject: [PATCH 1/3] Build shared libraries by default, single pybind module, simpler device strings Cherry-picks the shared-library-by-default and pybind-simplification work from copilot/separate-pybind-and-libopen3d without the host/device (libOpen3D vs libOpen3D_cuda/xpu) library split. - BUILD_SHARED_LIBS now defaults to ON. - Single open3d.pybind module; CUDA/SYCL detected at runtime via core.cuda.is_available()/core.sycl.is_available() instead of shipping separate cpu/cuda package variants. - Device strings accept a bare type name (e.g. "cuda"), defaulting id to 0. - New utility::filesystem::GetSelfBinaryDirectory() helper; Logging.cpp print function is now a function-local static. - Renamed pybind/core/tensor_type_caster -> type_caster and added an implicit Device<->str caster. - Flattened ML custom-op output layout (no cpu/cuda subfolder) and fixed the torch/tf ops loaders to match. - Fixed a static-link ordering bug (curl/BoringSSL) exposed by building Open3D as a shared library, using CMake's LINK_GROUP:RESCAN genex. - Ported unrelated CI/Docker/WebRTC improvements from the source branch. - Updated docs/compilation.rst to describe the single combined library. Verified: BUILD_SHARED_LIBS=ON by default, cpp tests (Device, MemoryManager) and python/test/core/test_core.py device tests pass, and python import/ smoke test shows no cpu/cuda subpackages. Co-authored-by: Cursor --- .github/workflows/README.md | 2 +- .github/workflows/macos.yml | 26 ++- .github/workflows/ubuntu-cuda.yml | 9 +- .github/workflows/ubuntu-openblas.yml | 44 ++++- .github/workflows/ubuntu-sycl.yml | 112 ++++++++--- .github/workflows/ubuntu-wheel.yml | 43 +++++ .github/workflows/vtk_packages.yml | 2 +- .github/workflows/webrtc.yml | 175 ++++++++++++------ .github/workflows/windows.yml | 53 +++--- 3rdparty/find_dependencies.cmake | 15 +- 3rdparty/webrtc/CMakeLists.txt | 1 + 3rdparty/webrtc/webrtc_build.sh | 10 +- 3rdparty/webrtc/webrtc_common.cmake | 11 ++ CMakeLists.txt | 2 +- cpp/open3d/core/Device.cpp | 66 ++++--- cpp/open3d/core/Device.h | 6 +- cpp/open3d/ml/pytorch/CMakeLists.txt | 8 +- cpp/open3d/ml/tensorflow/CMakeLists.txt | 6 +- cpp/open3d/utility/FileSystem.cpp | 51 +++++ cpp/open3d/utility/FileSystem.h | 2 + cpp/open3d/utility/Logging.cpp | 18 +- cpp/open3d/visualization/gui/Application.cpp | 68 +------ cpp/pybind/CMakeLists.txt | 9 +- cpp/pybind/core/CMakeLists.txt | 2 +- cpp/pybind/core/tensor.cpp | 2 +- ...tensor_type_caster.cpp => type_caster.cpp} | 15 +- .../{tensor_type_caster.h => type_caster.h} | 15 +- cpp/pybind/io/rpc.cpp | 2 +- cpp/pybind/make_python_package.cmake | 16 +- cpp/pybind/open3d_pybind.h | 6 +- cpp/pybind/t/geometry/raycasting_scene.cpp | 2 +- cpp/tests/core/Device.cpp | 18 ++ docker/Dockerfile.openblas | 6 +- docker/Dockerfile.wheel | 21 +-- docker/docker_build.sh | 129 +++++++------ docker/docker_test.sh | 24 +-- docs/compilation.rst | 105 ++++++++++- python/open3d/__init__.py | 129 ++++--------- python/open3d/core/__init__.py | 9 + python/open3d/ml/__init__.py | 7 +- python/open3d/ml/contrib/__init__.py | 6 +- python/open3d/ml/tf/python/ops/lib.py | 9 +- python/open3d/ml/torch/__init__.py | 26 ++- python/open3d/visualization/__init__.py | 11 +- python/test/core/test_core.py | 14 ++ util/ci_utils.sh | 70 +++---- util/run_ci.sh | 4 +- 47 files changed, 871 insertions(+), 516 deletions(-) mode change 100644 => 100755 .github/workflows/macos.yml mode change 100644 => 100755 .github/workflows/windows.yml rename cpp/pybind/core/{tensor_type_caster.cpp => type_caster.cpp} (73%) rename cpp/pybind/core/{tensor_type_caster.h => type_caster.h} (67%) create mode 100644 python/open3d/core/__init__.py mode change 100644 => 100755 util/ci_utils.sh diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 97e6e17b878..5098525e7a1 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -134,7 +134,7 @@ either due to lack of resources or GPU quota exhaustion. The custom VM image has NVIDIA drivers, `nvidia-container-toolkit` and `docker` installed. It contains today's date in the name and the image family is set to -`ubuntu-os-docker-gpu-2004-lts`. The latest image from this family is +`ubuntu-os-docker-gpu-2204-lts`. The latest image from this family is used for running CI. #### Step 3: GitHub diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml old mode 100644 new mode 100755 index 495049bbb35..4957a6302f5 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -26,7 +26,7 @@ env: BUILD_CUDA_MODULE: OFF jobs: - MacOS: + build-lib: permissions: contents: write # upload id-token: write @@ -75,9 +75,21 @@ jobs: run: | PATH=/usr/local/var/homebrew/linked/ccache/libexec:$PATH ccache -s + export BUILD_PYTHON_MODULE=OFF ./util/run_ci.sh DEVEL_PKG_NAME="$(basename build/package/open3d-devel-*.tar.xz)" echo "DEVEL_PKG_NAME=$DEVEL_PKG_NAME" >> $GITHUB_ENV + + # Compress build directory for reuse (exclude some large potentially unnecessary files if needed) + # We need 'build' directory. + tar -caf build.tar.xz build + + - name: Upload build artifact + if: ${{ matrix.CONFIG == 'OFF' }} + uses: actions/upload-artifact@v4 + with: + name: open3d-build-${{ matrix.os }} + path: build.tar.xz - name: Build Open3D viewer app if: ${{ env.BUILD_SHARED_LIBS == 'OFF' }} run: | @@ -129,6 +141,7 @@ jobs: fi build-wheel: + needs: build-lib name: Build wheel permissions: contents: write # upload @@ -171,6 +184,14 @@ jobs: ref: main path: ${{ env.OPEN3D_ML_ROOT }} + - name: Download build artifact + uses: actions/download-artifact@v4 + with: + name: open3d-build-${{ matrix.os }} + + - name: Unpack build artifact + run: tar -xf build.tar.xz + - name: Setup cache uses: actions/cache@v4 with: @@ -182,8 +203,7 @@ jobs: # Restore any ccache cache entry, if none for # ${{ runner.os }}-${{ runner.arch }}-ccache-${{ github.sha }} exists. # Common prefix will be used so that ccache can be used across commits. - restore-keys: | - ${{ runner.os }}-${{ runner.arch }}-ccache + restore-keys: ${{ runner.os }}-${{ runner.arch }}-ccache - name: Set up Python uses: actions/setup-python@v5 diff --git a/.github/workflows/ubuntu-cuda.yml b/.github/workflows/ubuntu-cuda.yml index b69b747155f..c1080d8635e 100644 --- a/.github/workflows/ubuntu-cuda.yml +++ b/.github/workflows/ubuntu-cuda.yml @@ -57,14 +57,13 @@ jobs: fail-fast: false matrix: include: - - CI_CONFIG: 2-jammy - - CI_CONFIG: 3-ml-shared-jammy - - CI_CONFIG: 5-ml-noble + - CI_CONFIG: 2-noble + - CI_CONFIG: 3-ml-shared-noble env: # Export everything from matrix to be easily used. # Docker tag and ccache names must be consistent with docker_build.sh CI_CONFIG : ${{ matrix.CI_CONFIG }} - BUILD_PACKAGE : ${{ contains(fromJson('["3-ml-shared-jammy"]'), matrix.CI_CONFIG) }} + BUILD_PACKAGE : ${{ contains(fromJson('["3-ml-shared-noble"]'), matrix.CI_CONFIG) }} GCE_INSTANCE_PREFIX: open3d-ci-${{ matrix.CI_CONFIG }} DOCKER_TAG : open3d-ci:${{ matrix.CI_CONFIG }} CCACHE_TAR_NAME : open3d-ci-${{ matrix.CI_CONFIG }} @@ -119,7 +118,7 @@ jobs: --machine-type=n1-standard-8 \ --boot-disk-size="128GB" \ --boot-disk-type="pd-ssd" \ - --image-family="ubuntu-os-docker-gpu-2004-lts" \ + --image-family="ubuntu-os-docker-gpu-2204-lts" \ --metadata-from-file=startup-script=./util/gcloud_auto_clean.sh \ --scopes="storage-full,compute-rw" \ --service-account="$GCE_GPU_CI_SA"; do diff --git a/.github/workflows/ubuntu-openblas.yml b/.github/workflows/ubuntu-openblas.yml index 46ad4b5a1ff..02d5a3d8f6b 100644 --- a/.github/workflows/ubuntu-openblas.yml +++ b/.github/workflows/ubuntu-openblas.yml @@ -34,13 +34,45 @@ jobs: - name: Docker test run: docker/docker_test.sh openblas-amd64-py312-dev - openblas-arm64: + build-lib-arm64: + name: Build Lib ARM64 permissions: contents: write # Release upload id-token: write attestations: write artifact-metadata: write runs-on: ubuntu-24.04-arm # latest + env: + DEVELOPER_BUILD: 'ON' + OPEN3D_CPU_RENDERING: true + steps: + - name: Checkout source code + uses: actions/checkout@v4 + - name: Maximize build space + run: | + source util/ci_utils.sh + maximize_ubuntu_github_actions_build_space + + - name: Docker build lib + run: | + # Use py310 as dummy + docker/docker_build.sh openblas-arm64-py310-dev build-lib + # Image name open3d-ci:openblas-arm64-py310-dev (check valid tag in script) + docker tag open3d-ci:openblas-arm64-py310-dev open3d-lib-arm64:latest + docker save open3d-lib-arm64:latest | gzip > open3d-lib-arm64.tar.gz + + - name: Upload Lib Image + uses: actions/upload-artifact@v4 + with: + name: open3d-lib-arm64 + path: open3d-lib-arm64.tar.gz + + build-wheel-arm64: + name: Build Wheel ARM64 + permissions: + contents: read + runs-on: ubuntu-24.04-arm + needs: build-lib-arm64 strategy: fail-fast: false matrix: @@ -68,6 +100,15 @@ jobs: source util/ci_utils.sh maximize_ubuntu_github_actions_build_space + - name: Download Lib Image + uses: actions/download-artifact@v4 + with: + name: open3d-lib-arm64 + path: . + + - name: Load Lib Image + run: docker load -i open3d-lib-arm64.tar.gz # Restore open3d-lib-arm64:latest + - name: Compute Docker tag for this matrix entry run: | # Strip the dot: 3.12 ➜ 312, 3.14 ➜ 314 … @@ -78,6 +119,7 @@ jobs: - name: Docker build run: | + export BASE_IMAGE=open3d-lib-arm64:latest docker/docker_build.sh "${DOCKER_TAG}" shopt -s failglob PIP_PKG_NAME="$(basename ${GITHUB_WORKSPACE}/open3d-[0-9]*.whl)" diff --git a/.github/workflows/ubuntu-sycl.yml b/.github/workflows/ubuntu-sycl.yml index e4903de1890..30fb79b600a 100644 --- a/.github/workflows/ubuntu-sycl.yml +++ b/.github/workflows/ubuntu-sycl.yml @@ -23,7 +23,8 @@ env: DEVELOPER_BUILD: ${{ github.event.inputs.developer_build || 'ON' }} jobs: - ubuntu-sycl: + build-lib: + name: Build Lib permissions: contents: write # Release upload id-token: write @@ -41,39 +42,36 @@ jobs: run: | source util/ci_utils.sh maximize_ubuntu_github_actions_build_space - - name: Docker build + - name: Docker build lib run: | + # Use py310 as dummy if [ "${{ matrix.BUILD_SHARED_LIBS }}" = "ON" ]; then - docker/docker_build.sh sycl-shared + docker/docker_build.sh sycl-shared build-lib + docker tag open3d-ci:sycl-shared open3d-lib-sycl-shared:latest + docker save open3d-lib-sycl-shared:latest -o open3d-lib-sycl-shared.tar + xz open3d-lib-sycl-shared.tar + ls -lhs open3d-lib-sycl-shared.tar.xz else - docker/docker_build.sh sycl-static - fi - - name: Docker test - run: | - du -hs $PWD - if [ "${{ matrix.BUILD_SHARED_LIBS }}" = "ON" ]; then - docker/docker_test.sh sycl-shared - else - docker/docker_test.sh sycl-static + docker/docker_build.sh sycl-static build-lib + # Static build usually doesn't produce wheels or shared libs for python + # But we might need artifacts. For now keep simple. fi - - name: Generate artifact attestation + - name: Upload Lib Image if: ${{ matrix.BUILD_SHARED_LIBS == 'ON' }} - uses: actions/attest@v4 + uses: actions/upload-artifact@v4 with: - subject-path: | - ${{ github.workspace }}/open3d*.whl - ${{ github.workspace }}/open3d-devel-*.tar.xz + name: open3d-lib-sycl-shared + path: open3d-lib-sycl-shared.tar.xz - - name: Upload Python wheel and C++ binary package to GitHub artifacts + - name: Upload C++ binary package to GitHub artifacts if: ${{ matrix.BUILD_SHARED_LIBS == 'ON' }} uses: actions/upload-artifact@v4 with: - name: open3d-sycl-linux-wheel-and-binary - path: | - open3d*.whl - open3d-devel-*.tar.xz + name: open3d-devel-sycl + path: open3d-devel-*.tar.xz if-no-files-found: error + - name: Update devel release if: ${{ github.ref == 'refs/heads/main' && matrix.BUILD_SHARED_LIBS == 'ON' }} env: @@ -97,3 +95,73 @@ jobs: if: ${{ github.ref == 'refs/heads/main' }} run: | gsutil cp ${GITHUB_WORKSPACE}/open3d-ci-sycl.tar.gz gs://open3d-ci-cache/ || true + + build-wheel: + name: Build Wheel + permissions: + contents: write # Release upload + id-token: write + attestations: write + artifact-metadata: write + runs-on: ubuntu-latest + needs: build-lib + strategy: + fail-fast: false + matrix: + python_version: ['3.10', '3.11', '3.12', '3.13'] + is_main: + - ${{ github.ref == 'refs/heads/main' }} + exclude: + - is_main: false + python_version: '3.10' + - is_main: false + python_version: '3.11' + - is_main: false + python_version: '3.12' + env: + PYTHON_VERSION: ${{ matrix.python_version }} + steps: + - name: Checkout source code + uses: actions/checkout@v4 + - name: Maximize build space + run: | + source util/ci_utils.sh + maximize_ubuntu_github_actions_build_space + + - name: Download Lib Image + uses: actions/download-artifact@v4 + with: + name: open3d-lib-sycl-shared + path: . + + - name: Load Lib Image + run: docker load -i open3d-lib-sycl-shared.tar.xz + + - name: Docker build + run: | + export BASE_IMAGE=open3d-lib-sycl-shared:latest + + # We execute docker buildscript with python version argument. + docker/docker_build.sh sycl-shared py${{ matrix.python_version }} + + - name: Generate artifact attestation + uses: actions/attest@v4 + with: + subject-path: | + ${{ github.workspace }}/open3d*.whl + + - name: Upload wheel to GitHub artifacts + uses: actions/upload-artifact@v4 + with: + name: open3d-sycl-wheel-py${{ matrix.python_version }} + path: open3d*.whl + if-no-files-found: error + + - name: Update devel release + if: ${{ github.ref == 'refs/heads/main' }} + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release upload main-devel open3d-*.whl --clobber + gh release view main-devel + diff --git a/.github/workflows/ubuntu-wheel.yml b/.github/workflows/ubuntu-wheel.yml index 1b16e1ec168..607dbf77464 100644 --- a/.github/workflows/ubuntu-wheel.yml +++ b/.github/workflows/ubuntu-wheel.yml @@ -23,7 +23,38 @@ env: BUILD_CUDA_MODULE: 'ON' jobs: + build-lib: + name: Build Lib + runs-on: ubuntu-latest + env: + DEVELOPER_BUILD: 'ON' + CCACHE_TAR_NAME: open3d-ubuntu-2204-cuda-ci-ccache + OPEN3D_CPU_RENDERING: true + steps: + - name: Checkout source code + uses: actions/checkout@v4 + - name: Maximize build space + run: | + source util/ci_utils.sh + maximize_ubuntu_github_actions_build_space + - name: Docker build lib + run: | + # Use py310 as dummy + docker/docker_build.sh cuda_wheel_py310_dev build-lib + # The image is tagged open3d-ci:wheel by default in docker_build.sh (cuda_wheel_build) + docker tag open3d-ci:wheel open3d-lib:latest + docker save open3d-lib:latest -o open3d-lib.tar + gzip open3d-lib.tar + ls -lhs open3d-lib.tar.gz + + - name: Upload Lib Image + uses: actions/upload-artifact@v4 + with: + name: open3d-lib-image + path: open3d-lib.tar.gz + build-wheel: + needs: build-lib permissions: contents: write # Release upload id-token: write @@ -61,10 +92,22 @@ jobs: run: | source util/ci_utils.sh maximize_ubuntu_github_actions_build_space + + - name: Download Lib Image + uses: actions/download-artifact@v4 + with: + name: open3d-lib-image + path: . + + - name: Load Lib Image + run: docker load -i open3d-lib.tar.gz # Restore open3d-lib:latest + # Be verbose and explicit here such that a developer can directly copy the # `docker/docker_build.sh xxx` command to execute locally. - name: Docker build run: | + export BASE_IMAGE=open3d-lib:latest + if [ "${{ env.PYTHON_VERSION }}" = "3.10" ] && [ "${{ env.DEVELOPER_BUILD }}" = "ON" ]; then docker/docker_build.sh cuda_wheel_py310_dev elif [ "${{ env.PYTHON_VERSION }}" = "3.11" ] && [ "${{ env.DEVELOPER_BUILD }}" = "ON" ]; then diff --git a/.github/workflows/vtk_packages.yml b/.github/workflows/vtk_packages.yml index f447193a5bd..9ff1a736771 100644 --- a/.github/workflows/vtk_packages.yml +++ b/.github/workflows/vtk_packages.yml @@ -11,7 +11,7 @@ jobs: permissions: contents: write # TODO: Convert to docker - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Checkout source code uses: actions/checkout@v4 diff --git a/.github/workflows/webrtc.yml b/.github/workflows/webrtc.yml index 45b27363ebc..9cb0562be13 100644 --- a/.github/workflows/webrtc.yml +++ b/.github/workflows/webrtc.yml @@ -12,10 +12,26 @@ on: description: 'Specify Depot Tools commit to to use for the build.' required: false default: 'e1a98941d3ab10549be6d82d0686bb0fb91ec903' # Date: Wed Apr 7 21:35:29 2021 +0000 - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true + build_linux: + description: 'Build Linux binary (x86_64)' + type: boolean + required: false + default: true + build_macos: + description: 'Build macOS binary (arm64)' + type: boolean + required: false + default: true + build_windows_static: + description: 'Build Windows Static Runtime binary (x86_64)' + type: boolean + required: false + default: true + build_windows_dll: + description: 'Build Windows DLL (Dynamic Runtime) binary (x86_64)' + type: boolean + required: false + default: true env: WEBRTC_COMMIT: ${{ github.event.inputs.webrtc_commit }} @@ -26,150 +42,201 @@ jobs: Unix: permissions: contents: write # upload - runs-on: ${{ matrix.os }} strategy: fail-fast: false + # `enabled` carries the workflow_dispatch toggle into the matrix so that + # individual entries can be skipped (the matrix context is not available + # in a job-level `if`, so per-entry gating is done at the step level). matrix: - os: [ubuntu-22.04, macos-13] + include: + - os: ubuntu-22.04 + platform: linux + enabled: ${{ github.event.inputs.build_linux }} + artifact_name: webrtc_release_ubuntu-22.04 + - os: macos-14 + platform: macos + enabled: ${{ github.event.inputs.build_macos }} + artifact_name: webrtc_release_macos-14-arm64 + runs-on: ${{ matrix.os }} steps: - name: Checkout source code uses: actions/checkout@v4 + - name: Set parallel build jobs + run: echo "NPROC=$(getconf _NPROCESSORS_ONLN)" >> "$GITHUB_ENV" + - name: Set up Python version uses: actions/setup-python@v5 with: - python-version: 3.10 + python-version: "3.10" - - name: Install dependencies - if: ${{ matrix.os == 'ubuntu-22.04' }} + - name: Install dependencies (Linux) + if: matrix.enabled == 'true' && matrix.platform == 'linux' run: | source 3rdparty/webrtc/webrtc_build.sh install_dependencies_ubuntu - name: Download WebRTC sources + if: matrix.enabled == 'true' run: | source 3rdparty/webrtc/webrtc_build.sh download_webrtc_sources - name: Build WebRTC + if: matrix.enabled == 'true' run: | source 3rdparty/webrtc/webrtc_build.sh build_webrtc - name: Upload WebRTC + if: matrix.enabled == 'true' uses: actions/upload-artifact@v4 with: - name: webrtc_release_${{ matrix.os }} + name: ${{ matrix.artifact_name }} path: | - webrtc_*.tar.gz - checksum_*.txt + webrtc_*.tar.gz + checksum_*.txt if-no-files-found: error Windows: permissions: contents: write # upload # https://chromium.googlesource.com/chromium/src/+/HEAD/docs/windows_build_instructions.md + strategy: + fail-fast: false + # `enabled` carries the workflow_dispatch toggle into the matrix so that + # individual entries can be skipped (the matrix context is not available + # in a job-level `if`, so per-entry gating is done at the step level). + matrix: + include: + - variant: static + static_runtime: ON + zip_suffix: win + checksum_file: checksum_win.txt + enabled: ${{ github.event.inputs.build_windows_static }} + artifact_name: webrtc_release_windows_static + - variant: dll + static_runtime: OFF + zip_suffix: win_dll + checksum_file: checksum_win_dll.txt + enabled: ${{ github.event.inputs.build_windows_dll }} + artifact_name: webrtc_release_windows_dll runs-on: windows-2022 env: WORK_DIR: "C:\\WebRTC" # Not enough space in D: OPEN3D_DIR: "D:\\a\\open3d\\open3d" DEPOT_TOOLS_UPDATE: 1 # Fix cannot find python3_bin_reldir.txt DEPOT_TOOLS_WIN_TOOLCHAIN: 0 - NPROC: 2 steps: - name: Checkout source code uses: actions/checkout@v4 + - name: Set parallel build jobs + shell: pwsh + run: echo "NPROC=$env:NUMBER_OF_PROCESSORS" >> $env:GITHUB_ENV + - name: Set up Python version uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: "3.10" - name: Disk space + if: matrix.enabled == 'true' + shell: pwsh run: | Get-PSDrive - mkdir "$env:WORK_DIR" + New-Item -ItemType Directory -Force -Path "$env:WORK_DIR" | Out-Null - - name: Setup PATH for Visual Studio # Required for Ninja + - name: Setup PATH for Visual Studio + if: matrix.enabled == 'true' uses: ilammy/msvc-dev-cmd@v1 with: arch: x64 - name: Download WebRTC sources + if: matrix.enabled == 'true' shell: pwsh working-directory: ${{ env.WORK_DIR }} run: | $ErrorActionPreference = 'Stop' echo "Get depot_tools" - # Checkout to a specific version - # Ref: https://chromium.googlesource.com/chromium/src/+/main/docs/building_old_revisions.md git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git git -C depot_tools checkout $env:DEPOT_TOOLS_COMMIT $env:Path = (Get-Item depot_tools).FullName + ";" + $env:Path echo "Get WebRTC" - mkdir webrtc - cd webrtc + New-Item -ItemType Directory -Force -Path webrtc | Out-Null + Push-Location webrtc fetch webrtc - git -C src checkout $env:WEBRTC_COMMIT git -C src submodule update --init --recursive echo "gclient sync" gclient sync -D --force --reset - cd .. - echo "random.org" + Pop-Location curl "https://www.random.org/cgi-bin/randbyte?nbytes=10&format=h" -o skipcache - name: Patch WebRTC + if: matrix.enabled == 'true' + shell: pwsh working-directory: ${{ env.WORK_DIR }} run: | $ErrorActionPreference = 'Stop' - cp "$env:OPEN3D_DIR/3rdparty/webrtc/CMakeLists.txt" webrtc/ - cp "$env:OPEN3D_DIR/3rdparty/webrtc/webrtc_common.cmake" webrtc/ + Copy-Item "$env:OPEN3D_DIR/3rdparty/webrtc/CMakeLists.txt" webrtc/ + Copy-Item "$env:OPEN3D_DIR/3rdparty/webrtc/webrtc_common.cmake" webrtc/ - - name: Build WebRTC (Release) - working-directory: ${{ env.WORK_DIR }} - run: | - $ErrorActionPreference = 'Stop' - $env:Path = (Get-Item depot_tools).FullName + ";" + $env:Path - mkdir webrtc/build - cd webrtc/build - cmake -G Ninja -D CMAKE_BUILD_TYPE=Release ` - -D CMAKE_INSTALL_PREFIX=${{ env.WORK_DIR }}/webrtc_release/Release ` - .. - ninja install - echo "Cleanup build folder for next config build" - cd .. - rm -r build - - - name: Build WebRTC (Debug) + - name: Build WebRTC (Release and Debug) + if: matrix.enabled == 'true' + shell: pwsh working-directory: ${{ env.WORK_DIR }} + env: + STATIC_WINDOWS_RUNTIME: ${{ matrix.static_runtime }} run: | $ErrorActionPreference = 'Stop' - $env:Path = (Get-Item depot_tools).FullName + ";" + $env:Path - mkdir webrtc/build - cd webrtc/build - cmake -G Ninja -D CMAKE_BUILD_TYPE=Debug ` - -D CMAKE_INSTALL_PREFIX=${{ env.WORK_DIR }}/webrtc_release/Debug ` - .. - ninja install + # Use real ninja.exe for CMake; depot_tools/ninja breaks generator detection. + $cmakeNinja = (Get-Command ninja.exe -ErrorAction SilentlyContinue).Source + if (-not $cmakeNinja) { + $cmakeNinja = "${env:ProgramFiles}\CMake\bin\ninja.exe" + } + if (-not (Test-Path -LiteralPath $cmakeNinja)) { + throw "Cannot find ninja.exe for CMake (tried PATH and $cmakeNinja)" + } + + foreach ($config in @('Release', 'Debug')) { + New-Item -ItemType Directory -Force -Path webrtc/build | Out-Null + Push-Location webrtc/build + cmake -G Ninja ` + -D CMAKE_MAKE_PROGRAM="$cmakeNinja" ` + -D CMAKE_BUILD_TYPE=$config ` + -D "CMAKE_INSTALL_PREFIX=$env:WORK_DIR/webrtc_release/$config" ` + -D "STATIC_WINDOWS_RUNTIME=$env:STATIC_WINDOWS_RUNTIME" ` + .. + ninja -j $env:NPROC install + Pop-Location + Remove-Item -Recurse -Force webrtc/build + } - name: Package WebRTC + if: matrix.enabled == 'true' + shell: pwsh working-directory: ${{ env.WORK_DIR }} + env: + ZIP_SUFFIX: ${{ matrix.zip_suffix }} + CHECKSUM_FILE: ${{ matrix.checksum_file }} run: | $ErrorActionPreference = 'Stop' $env:WEBRTC_COMMIT_SHORT = (git -C webrtc/src rev-parse --short=7 HEAD) - cmake -E tar cv webrtc_${env:WEBRTC_COMMIT_SHORT}_win.zip ` - --format=zip -- webrtc_release - cmake -E sha256sum webrtc_${env:WEBRTC_COMMIT_SHORT}_win.zip | Tee-Object -FilePath checksum_win.txt + $zipName = "webrtc_${env:WEBRTC_COMMIT_SHORT}_${env:ZIP_SUFFIX}.zip" + cmake -E tar cv $zipName --format=zip -- webrtc_release + cmake -E sha256sum $zipName | Tee-Object -FilePath $env:CHECKSUM_FILE - name: Upload WebRTC + if: matrix.enabled == 'true' uses: actions/upload-artifact@v4 with: - name: webrtc_release_windows + name: ${{ matrix.artifact_name }} path: | - ${{ env.WORK_DIR }}/webrtc_*.zip - ${{ env.WORK_DIR }}/checksum_*.txt + ${{ env.WORK_DIR }}/webrtc_*.zip + ${{ env.WORK_DIR }}/checksum_*.txt if-no-files-found: error diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml old mode 100644 new mode 100755 index bd24116faa2..a7180bf8cd4 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -36,7 +36,7 @@ env: DEVELOPER_BUILD: ${{ github.event.inputs.developer_build || 'ON' }} jobs: - windows: + build-lib: permissions: contents: write # upload id-token: write @@ -56,7 +56,7 @@ jobs: - BUILD_CUDA_MODULE: ON # FIXME CONFIG: Debug env: - BUILD_WEBRTC: ${{ ( matrix.BUILD_SHARED_LIBS == 'OFF' && matrix.STATIC_RUNTIME == 'ON' ) && 'ON' || 'OFF' }} + BUILD_WEBRTC: 'OFF' # TODO: WebRTC DLL not available for Windows; re-enable when fixed BUILD_PYTORCH_OPS: ${{ ( matrix.BUILD_CUDA_MODULE == 'ON' || matrix.CONFIG == 'Debug' ) && 'OFF' || 'ON' }} # FIXME steps: @@ -140,6 +140,7 @@ jobs: cmake -G "Visual Studio 17 2022" -A x64 ` -DDEVELOPER_BUILD=$Env:DEVELOPER_BUILD ` -DBUILD_EXAMPLES=OFF ` + -DBUILD_PYTHON_MODULE=OFF ` -DCMAKE_INSTALL_PREFIX="$env:INSTALL_DIR" ` -DBUILD_SHARED_LIBS=${{ matrix.BUILD_SHARED_LIBS }} ` -DSTATIC_WINDOWS_RUNTIME=${{ matrix.STATIC_RUNTIME }} ` @@ -257,23 +258,18 @@ jobs: .\${{ matrix.CONFIG }}\Draw.exe --skip-for-unit-test } Remove-Item -LiteralPath $env:INSTALL_DIR -Recurse -Force -ErrorAction SilentlyContinue - - name: Install Open3D python build requirements - working-directory: ${{ env.SRC_DIR }} - run: | - $ErrorActionPreference = 'Stop' - python -m pip install -U pip==${{ env.PIP_VER }} - python -m pip install -U -r python/requirements_build.txt - - name: Install Python package - working-directory: ${{ env.BUILD_DIR }} - run: | - $ErrorActionPreference = 'Stop' - cmake --build . --config ${{ matrix.CONFIG }} --target install-pip-package - - name: Import python package - # If BUILD_SHARED_LIBS == ON, Open3D.dll needs to be copied, which is not recommended for python. - if: ${{ matrix.BUILD_SHARED_LIBS == 'OFF' && matrix.BUILD_CUDA_MODULE == 'OFF' }} # FIXME + - name: Compress build directory + working-directory: "C:/" + if: ${{ matrix.BUILD_SHARED_LIBS == 'OFF' && matrix.STATIC_RUNTIME == 'ON' && matrix.BUILD_CUDA_MODULE == 'OFF' && matrix.CONFIG == 'Release' }} run: | - python -c "import open3d; print('Imported:', open3d)" - python -c "import open3d; print('CUDA enabled: ', open3d.core.cuda.is_available())" + Compress-Archive -Path "${{ env.BUILD_DIR }}" -DestinationPath "${{ env.SRC_DIR }}/open3d_build.zip" + + - name: Upload build artifact + if: ${{ matrix.BUILD_SHARED_LIBS == 'OFF' && matrix.STATIC_RUNTIME == 'ON' && matrix.BUILD_CUDA_MODULE == 'OFF' && matrix.CONFIG == 'Release' }} + uses: actions/upload-artifact@v4 + with: + name: open3d-build-windows + path: ${{ env.SRC_DIR }}/open3d_build.zip - name: Disk space used run: Get-PSDrive @@ -285,6 +281,7 @@ jobs: attestations: write artifact-metadata: write runs-on: windows-2022 + needs: build-lib strategy: fail-fast: false # https://github.community/t/how-to-conditionally-include-exclude-items-in-matrix-eg-based-on-branch/16853/6 @@ -329,6 +326,17 @@ jobs: with: python-version: ${{ matrix.python_version }} + - name: Download build artifact + uses: actions/download-artifact@v4 + with: + name: open3d-build-windows + path: ${{ env.SRC_DIR }} + + - name: Unpack build artifact + run: | + New-Item -Path ${{ env.BUILD_DIR }} -ItemType Directory -Force + Expand-Archive -Path ${{ env.SRC_DIR }}\open3d_build.zip -DestinationPath "C:\Open3D\" -Force + - name: Install Python dependencies working-directory: ${{ env.SRC_DIR }} run: | @@ -344,24 +352,21 @@ jobs: python -m pip install -U -r open3d_ml/requirements-torch.txt } - - name: Config run: | $ErrorActionPreference = 'Stop' - New-Item -Path ${{ env.BUILD_DIR }} -ItemType Directory cd ${{ env.BUILD_DIR }} if ($Env:DEVELOPER_BUILD -ne "OFF") { $Env:DEVELOPER_BUILD = "ON" } - cmake -G "Visual Studio 17 2022" -A x64 ` + cmake --fresh -G "Visual Studio 17 2022" -A x64 ` -DCMAKE_INSTALL_PREFIX="$env:INSTALL_DIR" ` -DDEVELOPER_BUILD="$Env:DEVELOPER_BUILD" ` - -DBUILD_SHARED_LIBS=OFF ` - -DSTATIC_WINDOWS_RUNTIME=ON ` -DBUILD_COMMON_ISPC_ISAS=ON ` + -DBUILD_PYTHON_MODULE=ON ` -DBUILD_AZURE_KINECT=ON ` -DBUILD_LIBREALSENSE=ON ` - -DBUILD_WEBRTC=ON ` + -DBUILD_WEBRTC=OFF ` -DBUILD_JUPYTER_EXTENSION=ON ` -DBUILD_PYTORCH_OPS=${{ env.BUILD_PYTORCH_OPS }} ` ${{ env.SRC_DIR }} diff --git a/3rdparty/find_dependencies.cmake b/3rdparty/find_dependencies.cmake index fb926d36ebb..31b5b567b56 100644 --- a/3rdparty/find_dependencies.cmake +++ b/3rdparty/find_dependencies.cmake @@ -951,7 +951,20 @@ if(NOT USE_SYSTEM_CURL) endif() target_link_libraries(3rdparty_curl INTERFACE 3rdparty_openssl) endif() -list(APPEND Open3D_3RDPARTY_PRIVATE_TARGETS_FROM_CUSTOM Open3D::3rdparty_curl Open3D::3rdparty_openssl) +# curl and openssl (BoringSSL) are mutually referential static archives: curl +# needs OpenSSL symbols and, depending on how the final link line gets +# flattened by CMake (e.g. when Open3D itself is a shared library and must +# fully resolve all symbols at build time), they can end up in an order where +# ld's single left-to-right archive scan fails to resolve symbols. Use +# CMake's LINK_GROUP genex (3.24+) so GNU ld rescans this group of libraries +# until all symbols resolve, regardless of order. This is a no-op on +# platforms without GNU ld (falls back to plain linking). +if(UNIX AND NOT APPLE) + list(APPEND Open3D_3RDPARTY_PRIVATE_TARGETS_FROM_CUSTOM + "$") +else() + list(APPEND Open3D_3RDPARTY_PRIVATE_TARGETS_FROM_CUSTOM Open3D::3rdparty_curl Open3D::3rdparty_openssl) +endif() # PNG if(USE_SYSTEM_PNG) diff --git a/3rdparty/webrtc/CMakeLists.txt b/3rdparty/webrtc/CMakeLists.txt index 34d052ae4df..c1871a19ae6 100644 --- a/3rdparty/webrtc/CMakeLists.txt +++ b/3rdparty/webrtc/CMakeLists.txt @@ -31,6 +31,7 @@ cmake_dependent_option(WEBRTC_IS_DEBUG "WebRTC Debug build. Use ON for Win32 Open3D Debug." OFF "NOT CMAKE_BUILD_TYPE STREQUAL Debug OR NOT WIN32" ON) option(GLIBCXX_USE_CXX11_ABI "Set -D_GLIBCXX_USE_CXX11_ABI=1" ON) +option(STATIC_WINDOWS_RUNTIME "Use static (MT/MTd) Windows runtime" ON) # Set paths set(WEBRTC_ROOT ${PROJECT_SOURCE_DIR}) diff --git a/3rdparty/webrtc/webrtc_build.sh b/3rdparty/webrtc/webrtc_build.sh index b4e48f126b2..e5e6ee6566e 100755 --- a/3rdparty/webrtc/webrtc_build.sh +++ b/3rdparty/webrtc/webrtc_build.sh @@ -58,10 +58,10 @@ install_dependencies_ubuntu() { git \ gnupg \ libglib2.0-dev \ - python \ - python-pip \ - python-setuptools \ - python-wheel \ + python-is-python3 \ + python3-pip \ + python3-setuptools \ + python3-wheel \ software-properties-common \ tree \ curl @@ -136,7 +136,7 @@ build_webrtc() { webrtc_release elif [[ $(uname -s) == 'Darwin' ]]; then tar -czf \ - "$OPEN3D_DIR/webrtc_${WEBRTC_COMMIT_SHORT}_macos.tar.gz" \ + "$OPEN3D_DIR/webrtc_${WEBRTC_COMMIT_SHORT}_macos-$(uname -m).tar.gz" \ webrtc_release fi popd # PWD=Open3D diff --git a/3rdparty/webrtc/webrtc_common.cmake b/3rdparty/webrtc/webrtc_common.cmake index 98c19336c6a..85f19e5045a 100644 --- a/3rdparty/webrtc/webrtc_common.cmake +++ b/3rdparty/webrtc/webrtc_common.cmake @@ -18,6 +18,17 @@ function(get_webrtc_args WEBRTC_ARGS) endif() endif() + if(MSVC) + if(NOT DEFINED STATIC_WINDOWS_RUNTIME) + set(STATIC_WINDOWS_RUNTIME ON) + endif() + if(NOT STATIC_WINDOWS_RUNTIME) + set(WEBRTC_ARGS "dynamic_crt=true\n${WEBRTC_ARGS}") + else() + set(WEBRTC_ARGS "dynamic_crt=false\n${WEBRTC_ARGS}") + endif() + endif() + if (APPLE) # WebRTC default set(WEBRTC_ARGS is_clang=true\n${WEBRTC_ARGS}) else() diff --git a/CMakeLists.txt b/CMakeLists.txt index 3283c8e8b21..593eb6c422d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,7 +34,7 @@ endif() include(CMakeDependentOption) # Open3D build options -option(BUILD_SHARED_LIBS "Build shared libraries" OFF) +option(BUILD_SHARED_LIBS "Build shared libraries" ON ) option(BUILD_EXAMPLES "Build Open3D examples programs" ON ) option(BUILD_UNIT_TESTS "Build Open3D unit tests" OFF) option(BUILD_BENCHMARKS "Build the micro benchmarks" OFF) diff --git a/cpp/open3d/core/Device.cpp b/cpp/open3d/core/Device.cpp index e81c386a67e..e74b622084f 100644 --- a/cpp/open3d/core/Device.cpp +++ b/cpp/open3d/core/Device.cpp @@ -18,42 +18,49 @@ namespace open3d { namespace core { -static Device::DeviceType StringToDeviceType(const std::string& type_colon_id) { +static Device::DeviceType ParseDeviceTypeToken(const std::string& token) { + const std::string device_type_lower = utility::ToLower(token); + if (device_type_lower == "cpu") { + return Device::DeviceType::CPU; + } else if (device_type_lower == "cuda") { + return Device::DeviceType::CUDA; + } else if (device_type_lower == "sycl") { + return Device::DeviceType::SYCL; + } + utility::LogError( + "Invalid device type \"{}\". Expected \"CPU\", \"CUDA\", or " + "\"SYCL\".", + token); +} + +static Device::DeviceType StringToDeviceType(const std::string& device_str) { const std::vector tokens = - utility::SplitString(type_colon_id, ":", true); + utility::SplitString(device_str, ":", true); + if (tokens.size() == 1) { + return ParseDeviceTypeToken(tokens[0]); + } if (tokens.size() == 2) { - std::string device_type_lower = utility::ToLower(tokens[0]); - if (device_type_lower == "cpu") { - return Device::DeviceType::CPU; - } else if (device_type_lower == "cuda") { - return Device::DeviceType::CUDA; - } else if (device_type_lower == "sycl") { - return Device::DeviceType::SYCL; - } else { - utility::LogError( - "Invalid device string {}. Valid device strings are like " - "\"CPU:0\", \"CUDA:1\" or \"SYCL:0\"", - type_colon_id); - } - } else { - utility::LogError( - "Invalid device string {}. Valid device strings are like " - "\"CPU:0\", \"CUDA:1\" or \"SYCL:0\"", - type_colon_id); + return ParseDeviceTypeToken(tokens[0]); } + utility::LogError( + "Invalid device string {}. Valid device strings are like " + "\"CPU\", \"CUDA:0\", or \"SYCL:1\"", + device_str); } -static int StringToDeviceId(const std::string& type_colon_id) { +static int StringToDeviceId(const std::string& device_str) { const std::vector tokens = - utility::SplitString(type_colon_id, ":", true); + utility::SplitString(device_str, ":", true); + if (tokens.size() == 1) { + return 0; + } if (tokens.size() == 2) { return std::stoi(tokens[1]); - } else { - utility::LogError( - "Invalid device string {}. Valid device strings are like " - "\"CPU:0\", \"CUDA:1\" or \"SYCL:0\"", - type_colon_id); } + utility::LogError( + "Invalid device string {}. Valid device strings are like " + "\"CPU\", \"CUDA:0\", or \"SYCL:1\"", + device_str); } Device::Device(DeviceType device_type, int device_id) @@ -68,9 +75,8 @@ Device::Device(DeviceType device_type, int device_id) Device::Device(const std::string& device_type, int device_id) : Device(device_type + ":" + std::to_string(device_id)) {} -Device::Device(const std::string& type_colon_id) - : Device(StringToDeviceType(type_colon_id), - StringToDeviceId(type_colon_id)) {} +Device::Device(const std::string& device_str) + : Device(StringToDeviceType(device_str), StringToDeviceId(device_str)) {} bool Device::operator==(const Device& other) const { return this->device_type_ == other.device_type_ && diff --git a/cpp/open3d/core/Device.h b/cpp/open3d/core/Device.h index d215d16e55e..8c3d33c0a89 100644 --- a/cpp/open3d/core/Device.h +++ b/cpp/open3d/core/Device.h @@ -31,10 +31,12 @@ class Device { explicit Device(DeviceType device_type, int device_id); /// Constructor from device type string and device id. + /// Use ``Device("cuda")`` or ``Device("CUDA:0")`` for id 0. explicit Device(const std::string& device_type, int device_id); - /// Constructor from string, e.g. "CUDA:0". - explicit Device(const std::string& type_colon_id); + /// Constructor from string, e.g. ``"CUDA:0"``, ``"cuda"``, or ``"cpu"``. + /// Bare type names use device id 0. + Device(const std::string& device_str); bool operator==(const Device& other) const; diff --git a/cpp/open3d/ml/pytorch/CMakeLists.txt b/cpp/open3d/ml/pytorch/CMakeLists.txt index cb47c201f7a..8983e48fabb 100644 --- a/cpp/open3d/ml/pytorch/CMakeLists.txt +++ b/cpp/open3d/ml/pytorch/CMakeLists.txt @@ -124,12 +124,10 @@ endif() # Set output directory according to architecture (cpu/cuda) get_target_property(TORCH_OPS_DIR open3d_torch_ops LIBRARY_OUTPUT_DIRECTORY) -set(TORCH_OPS_ARCH_DIR - "${TORCH_OPS_DIR}/$,cuda,cpu>") set_target_properties(open3d_torch_ops PROPERTIES - LIBRARY_OUTPUT_DIRECTORY "${TORCH_OPS_ARCH_DIR}" - ARCHIVE_OUTPUT_DIRECTORY "${TORCH_OPS_ARCH_DIR}" - RUNTIME_OUTPUT_DIRECTORY "${TORCH_OPS_ARCH_DIR}") + LIBRARY_OUTPUT_DIRECTORY "${TORCH_OPS_DIR}" + ARCHIVE_OUTPUT_DIRECTORY "${TORCH_OPS_DIR}" + RUNTIME_OUTPUT_DIRECTORY "${TORCH_OPS_DIR}") # Do not add "lib" prefix set_target_properties(open3d_torch_ops PROPERTIES PREFIX "") diff --git a/cpp/open3d/ml/tensorflow/CMakeLists.txt b/cpp/open3d/ml/tensorflow/CMakeLists.txt index 5e25e3b9cb0..21c7e47ca28 100644 --- a/cpp/open3d/ml/tensorflow/CMakeLists.txt +++ b/cpp/open3d/ml/tensorflow/CMakeLists.txt @@ -126,11 +126,9 @@ open3d_enable_strip(open3d_tf_ops) # Set output directory according to architecture (cpu/cuda) get_target_property(TF_OPS_DIR open3d_tf_ops LIBRARY_OUTPUT_DIRECTORY) -set(TF_OPS_ARCH_DIR - "${TF_OPS_DIR}/$,cuda,cpu>") set_target_properties(open3d_tf_ops PROPERTIES - LIBRARY_OUTPUT_DIRECTORY "${TF_OPS_ARCH_DIR}" - ARCHIVE_OUTPUT_DIRECTORY "${TF_OPS_ARCH_DIR}") + LIBRARY_OUTPUT_DIRECTORY "${TF_OPS_DIR}" + ARCHIVE_OUTPUT_DIRECTORY "${TF_OPS_DIR}") # Do not add "lib" prefix set_target_properties(open3d_tf_ops PROPERTIES PREFIX "") diff --git a/cpp/open3d/utility/FileSystem.cpp b/cpp/open3d/utility/FileSystem.cpp index 8e6389a8d84..31c6e470d21 100644 --- a/cpp/open3d/utility/FileSystem.cpp +++ b/cpp/open3d/utility/FileSystem.cpp @@ -23,6 +23,7 @@ #endif #else #include +#include #include #include #include @@ -132,6 +133,56 @@ std::string GetWorkingDirectory() { return std::string(buff); } +std::string GetSelfBinaryDirectory() { +#if defined(__APPLE__) || defined(__linux__) + // dladdr() resolves which shared-library image owns the given code address. + // Passing the address of this very function gives us *this* dylib/so/exe. + // This is POSIX-standard and works identically on macOS and Linux. + ::Dl_info info; + if (::dladdr(reinterpret_cast(&GetSelfBinaryDirectory), &info) && + info.dli_fname && info.dli_fname[0]) { + char resolved[PATH_MAX]; + const char *p = ::realpath(info.dli_fname, resolved) ? resolved + : info.dli_fname; + std::string path(p); + auto slash = path.rfind('/'); + return (slash != std::string::npos) ? path.substr(0, slash) : path; + } + +#elif defined(_WIN32) + // GetModuleHandleExW with FLAG_FROM_ADDRESS retrieves the HMODULE of + // whichever DLL/EXE contains the given virtual address – i.e. this module. + HMODULE hModule = nullptr; + if (::GetModuleHandleExW( + GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | + GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + reinterpret_cast(&GetSelfBinaryDirectory), &hModule)) { + wchar_t buf[MAX_PATH]; + DWORD len = ::GetModuleFileNameW(hModule, buf, MAX_PATH); + if (len > 0 && len < MAX_PATH) { + // Strip the filename to get the directory. + wchar_t *last_sep = ::wcsrchr(buf, L'\\'); + if (last_sep) *last_sep = L'\0'; + + // Convert UTF-16 directory to UTF-8. + int sz = ::WideCharToMultiByte(CP_UTF8, 0, buf, -1, nullptr, 0, + nullptr, nullptr); + if (sz > 1) { + std::string result(static_cast(sz) - 1, '\0'); + ::WideCharToMultiByte(CP_UTF8, 0, buf, -1, &result[0], sz, + nullptr, nullptr); + // Normalise backslashes for the rest of the codebase. + for (char &c : result) { + if (c == '\\') c = '/'; + } + return result; + } + } + } +#endif + return {}; +} + std::vector GetPathComponents(const std::string &path) { auto SplitByPathSeparators = [](const std::string &path) { std::vector components; diff --git a/cpp/open3d/utility/FileSystem.h b/cpp/open3d/utility/FileSystem.h index f1f56f99a5e..6c7a86e2d1d 100644 --- a/cpp/open3d/utility/FileSystem.h +++ b/cpp/open3d/utility/FileSystem.h @@ -45,6 +45,8 @@ std::string GetRegularizedDirectoryName(const std::string &directory); std::string GetWorkingDirectory(); +std::string GetSelfBinaryDirectory(); + std::vector GetPathComponents(const std::string &path); std::string GetTempDirectoryPath(); diff --git a/cpp/open3d/utility/Logging.cpp b/cpp/open3d/utility/Logging.cpp index 7440212f097..f272fa9efd5 100644 --- a/cpp/open3d/utility/Logging.cpp +++ b/cpp/open3d/utility/Logging.cpp @@ -34,9 +34,6 @@ struct Logger::Impl { // The current print function. std::function print_fcn_; - // The default print function (that prints to console). - static std::function console_print_fcn_; - // Verbosity level. VerbosityLevel verbosity_level_; @@ -57,11 +54,18 @@ struct Logger::Impl { } }; -std::function Logger::Impl::console_print_fcn_ = - [](const std::string &msg) { std::cout << msg << std::endl; }; +namespace { + +const std::function &GetDefaultPrintFunction() { + static const std::function default_fcn = + [](const std::string &msg) { std::cout << msg << std::endl; }; + return default_fcn; +} + +} // namespace Logger::Logger() : impl_(new Logger::Impl()) { - impl_->print_fcn_ = Logger::Impl::console_print_fcn_; + impl_->print_fcn_ = GetDefaultPrintFunction(); impl_->verbosity_level_ = VerbosityLevel::Info; } @@ -118,7 +122,7 @@ const std::function Logger::GetPrintFunction() { } void Logger::ResetPrintFunction() { - impl_->print_fcn_ = impl_->console_print_fcn_; + impl_->print_fcn_ = GetDefaultPrintFunction(); } void Logger::SetVerbosityLevel(VerbosityLevel verbosity_level) { diff --git a/cpp/open3d/visualization/gui/Application.cpp b/cpp/open3d/visualization/gui/Application.cpp index ebc61d5dbc7..0619b180048 100644 --- a/cpp/open3d/visualization/gui/Application.cpp +++ b/cpp/open3d/visualization/gui/Application.cpp @@ -12,12 +12,6 @@ #include // so APIENTRY gets defined and GLFW doesn't define it #endif // _MSC_VER -// Platform headers for GetSelfBinaryDir() -#if defined(__APPLE__) || defined(__linux__) -#include // dladdr (POSIX; libSystem on macOS, libdl on Linux) -#include // PATH_MAX -#endif - #include #include #include // std::getenv @@ -53,69 +47,11 @@ namespace { const double RUNLOOP_DELAY_SEC = 0.010; -// Returns the directory that contains the currently-running binary or shared -// library (whichever loaded this translation unit). This works whether the -// caller is a standalone executable or a Python/app process that loaded the -// Open3D DLL/dylib/.so, making resource discovery robust across deployment -// scenarios. -std::string GetSelfBinaryDir() { - namespace o3dfs = open3d::utility::filesystem; - -#if defined(__APPLE__) || defined(__linux__) - // dladdr() resolves which shared-library image owns the given code address. - // Passing the address of this very function gives us *this* dylib/so/exe. - // This is POSIX-standard and works identically on macOS and Linux. - ::Dl_info info; - if (::dladdr(reinterpret_cast(&GetSelfBinaryDir), &info) && - info.dli_fname && info.dli_fname[0]) { - char resolved[PATH_MAX]; - const char *p = ::realpath(info.dli_fname, resolved) ? resolved - : info.dli_fname; - std::string path(p); - auto slash = path.rfind('/'); - return (slash != std::string::npos) ? path.substr(0, slash) : path; - } - -#elif defined(_WIN32) - // GetModuleHandleExW with FLAG_FROM_ADDRESS retrieves the HMODULE of - // whichever DLL/EXE contains the given virtual address – i.e. this module. - HMODULE hModule = nullptr; - if (::GetModuleHandleExW( - GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | - GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, - reinterpret_cast(&GetSelfBinaryDir), &hModule)) { - wchar_t buf[MAX_PATH]; - DWORD len = ::GetModuleFileNameW(hModule, buf, MAX_PATH); - if (len > 0 && len < MAX_PATH) { - // Strip the filename to get the directory. - wchar_t *last_sep = ::wcsrchr(buf, L'\\'); - if (last_sep) *last_sep = L'\0'; - - // Convert UTF-16 directory to UTF-8. - int sz = ::WideCharToMultiByte(CP_UTF8, 0, buf, -1, nullptr, 0, - nullptr, nullptr); - if (sz > 1) { - std::string result(static_cast(sz) - 1, '\0'); - ::WideCharToMultiByte(CP_UTF8, 0, buf, -1, &result[0], sz, - nullptr, nullptr); - // Normalise backslashes for the rest of the codebase. - for (char &c : result) { - if (c == '\\') c = '/'; - } - return result; - } - } - } -#endif - - return {}; -} - std::string FindResourcePath() { // Search order (stops at the first hit): // 1. OPEN3D_RESOURCE_PATH environment variable. // 2. Subpaths relative to the directory of this binary or shared library, - // discovered via GetSelfBinaryDir(). + // discovered via GetSelfBinaryDirectory(). namespace o3dfs = open3d::utility::filesystem; // ---- Priority 1: explicit environment variable ------------------------- @@ -130,7 +66,7 @@ std::string FindResourcePath() { } // ---- Priority 2: relative to this binary / shared library -------------- - std::string self_dir = GetSelfBinaryDir(); + std::string self_dir = o3dfs::GetSelfBinaryDirectory(); if (!self_dir.empty()) { #if defined(__APPLE__) // macOS bundle: .../Contents/MacOS/ -> .../Contents/Resources/ diff --git a/cpp/pybind/CMakeLists.txt b/cpp/pybind/CMakeLists.txt index 5a3d8b8893e..fb06323c8df 100644 --- a/cpp/pybind/CMakeLists.txt +++ b/cpp/pybind/CMakeLists.txt @@ -75,9 +75,9 @@ endif() # At `make`: open3d.so (or the equivalents) will be created at # PYTHON_COMPILED_MODULE_DIR. The default location is -# `build/lib/${CMAKE_BUILD_TYPE}/Python/{cpu|cuda}` +# `build/lib/${CMAKE_BUILD_TYPE}/Python` set(PYTHON_COMPILED_MODULE_DIR - "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/Python/$,cuda,cpu>") + "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/Python") if (UNIX AND NOT APPLE) # Use RPATH instead of RUNPATH in pybind so that needed libc++.so can find child dependant libc++abi.so in RPATH @@ -109,10 +109,9 @@ set(PYTHON_EXTRA_LIBRARIES $) if (BUILD_GUI AND CMAKE_SYSTEM_NAME STREQUAL "Linux") list(APPEND PYTHON_EXTRA_LIBRARIES ${CPP_LIBRARY}.1 ${CPPABI_LIBRARY}) endif() -if (WITH_OPENMP AND APPLE AND NOT BUILD_SHARED_LIBS) +if (WITH_OPENMP AND APPLE) # Package libomp v11.1.0, if it is not installed. Later version cause crash on -# x86_64 if PyTorch is already imported. Case of shared libopen3d.dylib is not -# handled. +# x86_64 if PyTorch is already imported. # https://github.com/microsoft/LightGBM/issues/4229 list(APPEND PYTHON_EXTRA_LIBRARIES ${OpenMP_libomp_LIBRARY}) execute_process(COMMAND brew list --versions libomp diff --git a/cpp/pybind/core/CMakeLists.txt b/cpp/pybind/core/CMakeLists.txt index 511fe4dfdd7..6366dde25c9 100644 --- a/cpp/pybind/core/CMakeLists.txt +++ b/cpp/pybind/core/CMakeLists.txt @@ -13,7 +13,7 @@ target_sources(pybind PRIVATE tensor_accessor.cpp tensor_converter.cpp tensor_function.cpp - tensor_type_caster.cpp + type_caster.cpp tensor.cpp ) diff --git a/cpp/pybind/core/tensor.cpp b/cpp/pybind/core/tensor.cpp index a2d217141da..68ed8bb0d8a 100644 --- a/cpp/pybind/core/tensor.cpp +++ b/cpp/pybind/core/tensor.cpp @@ -21,7 +21,7 @@ #include "open3d/core/TensorKey.h" #include "pybind/core/core.h" #include "pybind/core/tensor_converter.h" -#include "pybind/core/tensor_type_caster.h" +#include "pybind/core/type_caster.h" #include "pybind/docstring.h" #include "pybind/open3d_pybind.h" #include "pybind/pybind_utils.h" diff --git a/cpp/pybind/core/tensor_type_caster.cpp b/cpp/pybind/core/type_caster.cpp similarity index 73% rename from cpp/pybind/core/tensor_type_caster.cpp rename to cpp/pybind/core/type_caster.cpp index d48194414c7..237d086daa5 100644 --- a/cpp/pybind/core/tensor_type_caster.cpp +++ b/cpp/pybind/core/type_caster.cpp @@ -5,7 +5,7 @@ // SPDX-License-Identifier: MIT // ---------------------------------------------------------------------------- -#include "pybind/core/tensor_type_caster.h" +#include "pybind/core/type_caster.h" #include "pybind/core/tensor_converter.h" @@ -32,5 +32,18 @@ bool type_caster::load(handle src, bool convert) { return false; } +bool type_caster::load(handle src, bool convert) { + if (type_caster_base::load(src, convert)) { + return true; + } + if (convert && py::isinstance(src)) { + holder_ = std::make_unique( + py::cast(src)); + value = holder_.get(); + return true; + } + return false; +} + } // namespace detail } // namespace pybind11 diff --git a/cpp/pybind/core/tensor_type_caster.h b/cpp/pybind/core/type_caster.h similarity index 67% rename from cpp/pybind/core/tensor_type_caster.h rename to cpp/pybind/core/type_caster.h index 19455769f2f..8fd4b48dbea 100644 --- a/cpp/pybind/core/tensor_type_caster.h +++ b/cpp/pybind/core/type_caster.h @@ -7,6 +7,7 @@ #pragma once +#include "open3d/core/Device.h" #include "pybind/open3d_pybind.h" namespace open3d { @@ -15,8 +16,8 @@ class Tensor; } } // namespace open3d -// Define type caster allowing implicit conversion to Tensor from common types. -// Needs to be included in each compilation unit. +// Type casters for implicit Python conversions. Include in each compilation +// unit that binds APIs using these types (see open3d_pybind.h). namespace pybind11 { namespace detail { template <> @@ -29,5 +30,15 @@ struct type_caster std::unique_ptr holder_; }; +template <> +struct type_caster + : public type_caster_base { +public: + bool load(handle src, bool convert); + +private: + std::unique_ptr holder_; +}; + } // namespace detail } // namespace pybind11 diff --git a/cpp/pybind/io/rpc.cpp b/cpp/pybind/io/rpc.cpp index 9871e4ec968..b2ee6e87ab0 100644 --- a/cpp/pybind/io/rpc.cpp +++ b/cpp/pybind/io/rpc.cpp @@ -11,7 +11,7 @@ #include "open3d/io/rpc/MessageUtils.h" #include "open3d/io/rpc/RemoteFunctions.h" #include "open3d/io/rpc/ZMQContext.h" -#include "pybind/core/tensor_type_caster.h" +#include "pybind/core/type_caster.h" #include "pybind/docstring.h" #include "pybind/open3d_pybind.h" diff --git a/cpp/pybind/make_python_package.cmake b/cpp/pybind/make_python_package.cmake index aa4171414b0..d1f3f94a56d 100644 --- a/cpp/pybind/make_python_package.cmake +++ b/cpp/pybind/make_python_package.cmake @@ -17,19 +17,11 @@ file(COPY ${PYTHON_PACKAGE_SRC_DIR}/ # 2) The compiled python-C++ module, i.e. open3d.so (or the equivalents) # Optionally other modules e.g. open3d_tf_ops.so may be included. -# Folder structure is base_dir/{cpu|cuda}/{pybind*.so|open3d_{torch|tf}_ops.so}, -# so copy base_dir directly to ${PYTHON_PACKAGE_DST_DIR}/open3d +# Copy each compiled extension directly into ${PYTHON_PACKAGE_DST_DIR}/open3d foreach(COMPILED_MODULE_PATH ${COMPILED_MODULE_PATH_LIST}) - get_filename_component(COMPILED_MODULE_NAME ${COMPILED_MODULE_PATH} NAME) - get_filename_component(COMPILED_MODULE_ARCH_DIR ${COMPILED_MODULE_PATH} DIRECTORY) - get_filename_component(COMPILED_MODULE_BASE_DIR ${COMPILED_MODULE_ARCH_DIR} DIRECTORY) - foreach(ARCH cpu cuda) - if(IS_DIRECTORY "${COMPILED_MODULE_BASE_DIR}/${ARCH}") - file(INSTALL "${COMPILED_MODULE_BASE_DIR}/${ARCH}/" DESTINATION - "${PYTHON_PACKAGE_DST_DIR}/open3d/${ARCH}" - FILES_MATCHING PATTERN "${COMPILED_MODULE_NAME}") - endif() - endforeach() + file(COPY ${COMPILED_MODULE_PATH} + DESTINATION ${PYTHON_PACKAGE_DST_DIR}/open3d/ + FOLLOW_SYMLINK_CHAIN) endforeach() # Include additional libraries that may be absent from the user system # eg: libc++.so and libc++abi.so (needed by filament) diff --git a/cpp/pybind/open3d_pybind.h b/cpp/pybind/open3d_pybind.h index 1204dc53c74..0d6ec27209b 100644 --- a/cpp/pybind/open3d_pybind.h +++ b/cpp/pybind/open3d_pybind.h @@ -39,9 +39,9 @@ #include "open3d/pipelines/registration/PoseGraph.h" #include "open3d/utility/Eigen.h" -// We include the type caster for tensor here because it must be included in -// every compilation unit. -#include "pybind/core/tensor_type_caster.h" +// Type casters must be included in every compilation unit that binds these +// types. +#include "pybind/core/type_caster.h" namespace py = pybind11; using namespace py::literals; diff --git a/cpp/pybind/t/geometry/raycasting_scene.cpp b/cpp/pybind/t/geometry/raycasting_scene.cpp index 0d924fc749b..98ac8d22e3e 100644 --- a/cpp/pybind/t/geometry/raycasting_scene.cpp +++ b/cpp/pybind/t/geometry/raycasting_scene.cpp @@ -6,7 +6,7 @@ // ---------------------------------------------------------------------------- #include "open3d/t/geometry/RaycastingScene.h" -#include "pybind/core/tensor_type_caster.h" +#include "pybind/core/type_caster.h" #include "pybind/t/geometry/geometry.h" namespace open3d { diff --git a/cpp/tests/core/Device.cpp b/cpp/tests/core/Device.cpp index 0f54a962d4e..710347c797d 100644 --- a/cpp/tests/core/Device.cpp +++ b/cpp/tests/core/Device.cpp @@ -7,6 +7,7 @@ #include "open3d/core/Device.h" +#include "open3d/core/Tensor.h" #include "tests/Tests.h" namespace open3d { @@ -41,6 +42,23 @@ TEST(Device, StringConstructorLower) { EXPECT_EQ(device.GetID(), 1); } +TEST(Device, BareTypeStringDefaultsToDevice0) { + core::Device cuda_device("cuda"); + EXPECT_EQ(cuda_device.GetType(), core::Device::DeviceType::CUDA); + EXPECT_EQ(cuda_device.GetID(), 0); + core::Device cpu_device("cpu"); + EXPECT_EQ(cpu_device.GetType(), core::Device::DeviceType::CPU); + EXPECT_EQ(cpu_device.GetID(), 0); + EXPECT_EQ(core::Device("CUDA", 0).GetID(), 0); +} + +TEST(Device, TensorToAcceptsDeviceString) { + core::Tensor t = + core::Tensor::Ones({2}, core::Float32, core::Device("CPU:0")); + EXPECT_EQ(t.To(core::Device("CPU:0")).GetDevice(), core::Device("CPU:0")); + EXPECT_EQ(t.To(core::Device("cpu")).GetDevice(), core::Device("CPU:0")); +} + TEST(Device, PrintAvailableDevices) { core::Device::PrintAvailableDevices(); } } // namespace tests diff --git a/docker/Dockerfile.openblas b/docker/Dockerfile.openblas index 2689b3e14da..3e344548376 100644 --- a/docker/Dockerfile.openblas +++ b/docker/Dockerfile.openblas @@ -10,6 +10,7 @@ ARG CONDA_SUFFIX ARG CMAKE_VERSION ARG PYTHON_VERSION ARG DEVELOPER_BUILD +ARG BUILD_PYTHON_MODULE=ON RUN if [ -z "${CONDA_SUFFIX}" ]; then echo "Error: ARG CONDA_SUFFIX not specified."; exit 1; fi \ && if [ -z "${CMAKE_VERSION}" ]; then echo "Error: ARG CMAKE_VERSION not specified."; exit 1; fi \ && if [ -z "${PYTHON_VERSION}" ]; then echo "Error: ARG PYTHON_VERSION not specified."; exit 1; fi \ @@ -105,9 +106,10 @@ RUN mkdir build \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX=~/open3d_install \ -DDEVELOPER_BUILD=${DEVELOPER_BUILD} \ + -DBUILD_PYTHON_MODULE=${BUILD_PYTHON_MODULE} \ .. \ && export NPROC=$(($(nproc)+2)) \ && make -j$NPROC \ - && make install-pip-package -j$NPROC \ + && if [ "${BUILD_PYTHON_MODULE}" = "ON" ]; then make install-pip-package -j$NPROC; fi \ && make install -j$NPROC -RUN cp build/lib/python_package/pip_package/*.whl / +RUN if [ "${BUILD_PYTHON_MODULE}" = "ON" ]; then cp build/lib/python_package/pip_package/*.whl /; fi diff --git a/docker/Dockerfile.wheel b/docker/Dockerfile.wheel index afbc45be5cb..883c42295ac 100644 --- a/docker/Dockerfile.wheel +++ b/docker/Dockerfile.wheel @@ -9,6 +9,7 @@ ARG CMAKE_VERSION ARG PYTHON_VERSION ARG BUILD_TENSORFLOW_OPS ARG BUILD_PYTORCH_OPS +ARG BUILD_PYTHON_MODULE ARG CI # Forward all ARG to ENV @@ -19,6 +20,7 @@ ENV CMAKE_VERSION=${CMAKE_VERSION} ENV PYTHON_VERSION=${PYTHON_VERSION} ENV BUILD_PYTORCH_OPS=${BUILD_PYTORCH_OPS} ENV BUILD_TENSORFLOW_OPS=${BUILD_TENSORFLOW_OPS} +ENV BUILD_PYTHON_MODULE=${BUILD_PYTHON_MODULE} # Prevent interactive inputs when installing packages ENV DEBIAN_FRONTEND=noninteractive @@ -27,18 +29,10 @@ ENV SUDO=command SHELL ["/bin/bash", "-c"] -# Fix Nvidia repo key rotation issue -# https://forums.developer.nvidia.com/t/notice-cuda-linux-repository-key-rotation/212771 -# https://forums.developer.nvidia.com/t/18-04-cuda-docker-image-is-broken/212892/10 -# https://code.visualstudio.com/remote/advancedcontainers/reduce-docker-warnings#:~:text=Warning%3A%20apt%2Dkey%20output%20should,not%20running%20from%20a%20terminal. -RUN export APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=DontWarn \ - && apt-key del 7fa2af80 \ - && apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/3bf863cc.pub \ - && apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub - # Dependencies: basic RUN apt-get update && apt-get install -y \ wget \ + ca-certificates \ ccache \ build-essential \ libssl-dev \ @@ -57,6 +51,11 @@ RUN apt-get update && apt-get install -y \ liblzma-dev \ && rm -rf /var/lib/apt/lists/* +# Install NVIDIA repository keyring package +RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb \ + && dpkg -i cuda-keyring_1.1-1_all.deb \ + && rm cuda-keyring_1.1-1_all.deb + # Dependencies: cmake RUN CMAKE_VERSION_NUMBERS=$(echo "${CMAKE_VERSION}" | cut -d"-" -f2) \ && wget -nv https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION_NUMBERS}/${CMAKE_VERSION}.tar.gz \ @@ -137,11 +136,11 @@ COPY . /root/Open3D WORKDIR /root/Open3D # Build python wheel +# Note: BUILD_SHARED_LIBS defaults to ON, which is required for Python wheels RUN export NPROC=$(($(nproc)+2)) \ - && export BUILD_SHARED_LIBS=OFF \ && source /root/Open3D/util/ci_utils.sh \ && build_pip_package build_azure_kinect build_jupyter \ - && if [ ${CI:-}a != a ]; then find /root/Open3D/build -mindepth 1 -maxdepth 1 ! -name 'lib' -exec rm -rf {} + ; fi + && if [ ${CI:-}a != a ]; then rm -rf build_cpu build_cuda && if [ -d build ]; then find build -mindepth 1 -maxdepth 1 -not -name lib -exec rm -rf {} +; fi; fi # remove build folder (except lib) to save CI space on Github # Compress ccache folder, move to / directory diff --git a/docker/docker_build.sh b/docker/docker_build.sh index 3eb2c188c3a..cb4683b95c3 100755 --- a/docker/docker_build.sh +++ b/docker/docker_build.sh @@ -57,10 +57,9 @@ OPTION: sycl-static : SYCL (oneAPI) with static lib # ML CIs (Dockerfile.ci) - 2-jammy : CUDA CI, 2-jammy, developer mode - 3-ml-shared-jammy-release : CUDA CI, 3-ml-shared-jammy (cxx11_abi), release mode - 3-ml-shared-jammy : CUDA CI, 3-ml-shared-jammy (cxx11_abi), developer mode - 5-ml-noble : CUDA CI, 5-ml-noble, developer mode + 2-noble : CUDA CI, 2-noble, developer mode + 3-ml-shared-noble-release : CUDA CI, 3-ml-shared-noble (cxx11_abi), release mode + 3-ml-shared-noble : CUDA CI, 3-ml-shared-noble (cxx11_abi), developer mode # CUDA wheels (Dockerfile.wheel) cuda_wheel_py310_dev : CUDA Python 3.10 wheel, developer mode @@ -105,13 +104,13 @@ openblas_export_env() { if [[ "amd64" =~ ^($options)$ ]]; then echo "[openblas_export_env()] platform AMD64" export DOCKER_TAG=open3d-ci:openblas-amd64 - export BASE_IMAGE=ubuntu:22.04 + export BASE_IMAGE=${BASE_IMAGE:-ubuntu:22.04} export CONDA_SUFFIX=x86_64 export CMAKE_VERSION=${CMAKE_VERSION} elif [[ "arm64" =~ ^($options)$ ]]; then echo "[openblas_export_env()] platform ARM64" export DOCKER_TAG=open3d-ci:openblas-arm64 - export BASE_IMAGE=arm64v8/ubuntu:22.04 + export BASE_IMAGE=${BASE_IMAGE:-arm64v8/ubuntu:22.04} export CONDA_SUFFIX=aarch64 export CMAKE_VERSION=${CMAKE_VERSION} else @@ -153,6 +152,12 @@ openblas_export_env() { export BUILD_PYTORCH_OPS=OFF export BUILD_TENSORFLOW_OPS=OFF export BUILD_SYCL_MODULE=OFF + + if [[ "core" =~ ^($options)$ ]]; then + export BUILD_PYTHON_MODULE=OFF + else + export BUILD_PYTHON_MODULE=ON + fi } openblas_build() { @@ -165,18 +170,22 @@ openblas_build() { --build-arg CMAKE_VERSION="${CMAKE_VERSION}" \ --build-arg PYTHON_VERSION="${PYTHON_VERSION}" \ --build-arg DEVELOPER_BUILD="${DEVELOPER_BUILD}" \ + --build-arg BUILD_PYTHON_MODULE="${BUILD_PYTHON_MODULE}" \ -t "${DOCKER_TAG}" \ -f docker/Dockerfile.openblas . popd - docker run -v "${PWD}:/opt/mount" --rm "${DOCKER_TAG}" \ - bash -c "cp /*.whl /opt/mount \ - && chown $(id -u):$(id -g) /opt/mount/*.whl" + if [ "$BUILD_PYTHON_MODULE" != "OFF" ]; then + docker run -v "${PWD}:/opt/mount" --rm "${DOCKER_TAG}" \ + bash -c "cp /*.whl /opt/mount \ + && chown $(id -u):$(id -g) /opt/mount/*.whl" + fi } cuda_wheel_build() { - BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu22.04 + BASE_IMAGE="${BASE_IMAGE:-nvidia/cuda:${CUDA_VERSION}-devel-ubuntu22.04}" CCACHE_TAR_NAME=open3d-ubuntu-2204-cuda-ci-ccache + DOCKER_TAG="open3d-ci:wheel" options="$(echo "$@" | tr ' ' '|')" echo "[cuda_wheel_build()] options: ${options}" @@ -199,10 +208,12 @@ cuda_wheel_build() { else DEVELOPER_BUILD=OFF fi - echo "[cuda_wheel_build()] PYTHON_VERSION: ${PYTHON_VERSION}" - echo "[cuda_wheel_build()] DEVELOPER_BUILD: ${DEVELOPER_BUILD}" - echo "[cuda_wheel_build()] BUILD_TENSORFLOW_OPS=${BUILD_TENSORFLOW_OPS:?'env var must be set.'}" - echo "[cuda_wheel_build()] BUILD_PYTORCH_OPS=${BUILD_PYTORCH_OPS:?'env var must be set.'}" + + if [[ "build-lib" =~ ^($options)$ ]]; then + BUILD_PYTHON_MODULE=OFF + else + BUILD_PYTHON_MODULE=ON + fi pushd "${HOST_OPEN3D_ROOT}" docker build \ @@ -213,17 +224,20 @@ cuda_wheel_build() { --build-arg PYTHON_VERSION="${PYTHON_VERSION}" \ --build-arg BUILD_TENSORFLOW_OPS="${BUILD_TENSORFLOW_OPS}" \ --build-arg BUILD_PYTORCH_OPS="${BUILD_PYTORCH_OPS}" \ + --build-arg BUILD_PYTHON_MODULE="${BUILD_PYTHON_MODULE}" \ --build-arg CI="${CI:-}" \ - -t open3d-ci:wheel \ + -t "${DOCKER_TAG}" \ -f docker/Dockerfile.wheel . popd - python_package_dir=/root/Open3D/build/lib/python_package - docker run -v "${PWD}:/opt/mount" --rm open3d-ci:wheel \ - bash -c "cp ${python_package_dir}/pip_package/open3d*.whl /opt/mount \ - && cp /${CCACHE_TAR_NAME}.tar.xz /opt/mount \ - && chown $(id -u):$(id -g) /opt/mount/open3d*.whl \ - && chown $(id -u):$(id -g) /opt/mount/${CCACHE_TAR_NAME}.tar.xz" + if [ "$BUILD_PYTHON_MODULE" != "OFF" ]; then + python_package_dir=/root/Open3D/build/lib/python_package + docker run -v "${PWD}:/opt/mount" --rm open3d-ci:wheel \ + bash -c "cp ${python_package_dir}/pip_package/open3d*.whl /opt/mount \ + && cp /${CCACHE_TAR_NAME}.tar.xz /opt/mount \ + && chown $(id -u):$(id -g) /opt/mount/open3d*.whl \ + && chown $(id -u):$(id -g) /opt/mount/${CCACHE_TAR_NAME}.tar.xz" + fi } ci_build() { @@ -263,12 +277,12 @@ ci_build() { && chown $(id -u):$(id -g) /opt/mount/open3d*" } -2-jammy_export_env() { - export DOCKER_TAG=open3d-ci:2-jammy +2-noble_export_env() { + export DOCKER_TAG=open3d-ci:2-noble export BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu22.04 export DEVELOPER_BUILD=ON - export CCACHE_TAR_NAME=open3d-ci-2-jammy + export CCACHE_TAR_NAME=open3d-ci-2-noble export PYTHON_VERSION=3.10 export BUILD_SHARED_LIBS=OFF export BUILD_CUDA_MODULE=ON @@ -278,12 +292,12 @@ ci_build() { export BUILD_SYCL_MODULE=OFF } -3-ml-shared-jammy_export_env() { - export DOCKER_TAG=open3d-ci:3-ml-shared-jammy +3-ml-shared-noble_export_env() { + export DOCKER_TAG=open3d-ci:3-ml-shared-noble export BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu22.04 export DEVELOPER_BUILD=ON - export CCACHE_TAR_NAME=open3d-ci-3-ml-shared-jammy + export CCACHE_TAR_NAME=open3d-ci-3-ml-shared-noble export PYTHON_VERSION=3.10 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=ON @@ -293,12 +307,12 @@ ci_build() { export BUILD_SYCL_MODULE=OFF } -3-ml-shared-jammy-release_export_env() { - export DOCKER_TAG=open3d-ci:3-ml-shared-jammy +3-ml-shared-noble-release_export_env() { + export DOCKER_TAG=open3d-ci:3-ml-shared-noble export BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu22.04 export DEVELOPER_BUILD=OFF - export CCACHE_TAR_NAME=open3d-ci-3-ml-shared-jammy + export CCACHE_TAR_NAME=open3d-ci-3-ml-shared-noble export PYTHON_VERSION=3.10 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=ON @@ -308,21 +322,6 @@ ci_build() { export BUILD_SYCL_MODULE=OFF } -5-ml-noble_export_env() { - export DOCKER_TAG=open3d-ci:5-ml-noble - - export BASE_IMAGE=nvidia/cuda:${CUDA_VERSION_LATEST}-devel-ubuntu22.04 - export DEVELOPER_BUILD=ON - export CCACHE_TAR_NAME=open3d-ci-5-ml-noble - export PYTHON_VERSION=3.12 - export BUILD_SHARED_LIBS=OFF - export BUILD_CUDA_MODULE=ON - export BUILD_TENSORFLOW_OPS=ON - export BUILD_PYTORCH_OPS=ON - export PACKAGE=OFF - export BUILD_SYCL_MODULE=OFF -} - cpu-static_export_env() { export DOCKER_TAG=open3d-ci:cpu-static @@ -386,17 +385,32 @@ cpu-shared-ml-release_export_env() { sycl-shared_export_env() { export DOCKER_TAG=open3d-ci:sycl-shared + options="$(echo "$@" | tr ' ' '|')" + + if [[ "$options" =~ py3(7|8|9|10|11|12|13) ]]; then + PYTHON_VERSION=${BASH_REMATCH[0]#py} + PYTHON_VERSION="${PYTHON_VERSION:0:1}.${PYTHON_VERSION:1}" + export BUILD_PYTHON_MODULE=ON + fi + # https://hub.docker.com/r/intel/oneapi-basekit # https://github.com/intel/oneapi-containers/blob/master/images/docker/basekit/Dockerfile.ubuntu-22.04 - export BASE_IMAGE=intel/cpp-essentials:2025.3.1-0-devel-ubuntu22.04 + export BASE_IMAGE=${BASE_IMAGE:-intel/cpp-essentials:2025.3.1-0-devel-ubuntu22.04} export DEVELOPER_BUILD=${DEVELOPER_BUILD:-ON} export CCACHE_TAR_NAME=open3d-ci-sycl export PYTHON_VERSION=${PYTHON_VERSION:-3.12} export BUILD_SHARED_LIBS=ON + export BUILD_CUDA_MODULE=OFF export BUILD_TENSORFLOW_OPS=ON export BUILD_PYTORCH_OPS=ON - export PACKAGE=ON + + if [[ "build-lib" =~ ^($options)$ ]]; then + export PACKAGE=ON + else + export PACKAGE=OFF + fi + export BUILD_SYCL_MODULE=ON export IGC_EnableDPEmulation=1 # Enable float64 emulation during compilation @@ -426,7 +440,7 @@ sycl-static_export_env() { } function main() { - if [[ "$#" -ne 1 ]]; then + if [[ "$#" -lt 1 ]]; then echo "Error: invalid number of arguments: $#." >&2 print_usage_and_exit_docker_build fi @@ -536,7 +550,8 @@ function main() { # SYCL CI sycl-shared) - sycl-shared_export_env + shift + sycl-shared_export_env "$@" ci_build ;; sycl-static) @@ -577,20 +592,16 @@ function main() { ;; # ML CIs - 2-jammy) - 2-jammy_export_env - ci_build - ;; - 3-ml-shared-jammy-release) - 3-ml-shared-jammy-release_export_env + 2-noble) + 2-noble_export_env ci_build ;; - 3-ml-shared-jammy) - 3-ml-shared-jammy_export_env + 3-ml-shared-noble-release) + 3-ml-shared-noble-release_export_env ci_build ;; - 5-ml-noble) - 5-ml-noble_export_env + 3-ml-shared-noble) + 3-ml-shared-noble_export_env ci_build ;; *) diff --git a/docker/docker_test.sh b/docker/docker_test.sh index c454f9a96c7..767d2a447ab 100755 --- a/docker/docker_test.sh +++ b/docker/docker_test.sh @@ -50,10 +50,9 @@ OPTION: sycl-static : SYCL (oneAPI) with static lib # ML CIs (Dockerfile.ci) - 2-jammy : CUDA CI, 2-jammy, developer mode - 3-ml-shared-jammy-release : CUDA CI, 3-ml-shared-jammy (cxx11_abi), release mode - 3-ml-shared-jammy : CUDA CI, 3-ml-shared-jammy (cxx11_abi), developer mode - 5-ml-noble : CUDA CI, 5-ml-noble, developer mode + 2-noble : CUDA CI, 2-noble, developer mode + 3-ml-shared-noble-release : CUDA CI, 3-ml-shared-noble (cxx11_abi), release mode + 3-ml-shared-noble : CUDA CI, 3-ml-shared-noble (cxx11_abi), developer mode " HOST_OPEN3D_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null 2>&1 && pwd)" @@ -348,23 +347,18 @@ sycl-static) ;; # ML CIs -2-jammy) - 2-jammy_export_env +2-noble) + 2-noble_export_env ci_print_env cpp_python_linking_uninstall_test ;; -3-ml-shared-jammy-release) - 3-ml-shared-jammy-release_export_env +3-ml-shared-noble-release) + 3-ml-shared-noble-release_export_env ci_print_env cpp_python_linking_uninstall_test ;; -3-ml-shared-jammy) - 3-ml-shared-jammy_export_env - ci_print_env - cpp_python_linking_uninstall_test - ;; -5-ml-noble) - 5-ml-noble_export_env +3-ml-shared-noble) + 3-ml-shared-noble_export_env ci_print_env cpp_python_linking_uninstall_test ;; diff --git a/docs/compilation.rst b/docs/compilation.rst index fa663c0d72d..a443804a617 100644 --- a/docs/compilation.rst +++ b/docs/compilation.rst @@ -25,12 +25,20 @@ System requirements * macOS: Install with Homebrew: ``brew install cmake`` * Windows: Download from: `CMake download page `_ -* CUDA 11.5+ (optional): Open3D supports GPU acceleration of an increasing number - of operations through CUDA on Linux. We recommend using CUDA 12+ for the - best compatibility with recent GPUs and optional external dependencies such - as Tensorflow or PyTorch. Please see the `official documentation - `_ to - install the CUDA toolkit from Nvidia. +* CUDA 11.5+ (optional): Open3D supports GPU acceleration through CUDA on Linux. + Prebuilt wheels statically link the CUDA runtime libraries directly into + ``libOpen3D``, so no separate NVIDIA runtime pip packages are required at + import time. For building from source, install the CUDA toolkit + (``nvidia-smi``, ``nvcc -V``) and configure with ``-DBUILD_CUDA_MODULE=ON`` + (``-DBUILD_WITH_CUDA_STATIC=ON`` by default). We recommend using CUDA 12+ + for the best compatibility with recent GPUs and optional external + dependencies such as Tensorflow or PyTorch. + +* Shared libraries: Open3D builds with ``BUILD_SHARED_LIBS=ON`` by default, + producing a single ``libOpen3D.so`` / ``Open3D.dll`` that contains CPU, + CUDA, and SYCL code together. Python wheels ship one ``pybind`` extension + module that links against this single library and picks the available + device (CPU/CUDA/SYCL) at runtime. * Ccache 4.0+ (optional, recommended): ccache is a compiler cache that can speed up the compilation process by avoiding recompilation of the same @@ -313,10 +321,14 @@ for all supported ML frameworks and bundling the high level Open3D-ML code. .. code-block:: bash - cmake -DBUILD_CUDA_MODULE=ON -DCMAKE_INSTALL_PREFIX= .. + cmake -DBUILD_CUDA_MODULE=ON -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_INSTALL_PREFIX= .. - Please note that CUDA support is work in progress and experimental. For building - Open3D with CUDA support, ensure that CUDA is properly installed by running following commands: + CUDA runtime libraries are statically linked into ``libOpen3D`` by default + (``-DBUILD_WITH_CUDA_STATIC=ON``), so CUDA wheels do not need any NVIDIA + redistributable pip packages at import time. Pass + ``-DBUILD_WITH_CUDA_STATIC=OFF`` to dynamically link the CUDA runtime + instead. For development, ensure the CUDA toolkit is available: .. code-block:: bash @@ -327,6 +339,81 @@ for all supported ML frameworks and bundling the high level Open3D-ML code. by following the `official documentation. `_ +.. _abi_dependency_compatibility: + +ABI dependency and compatibility +--------------------------------- + +When compiling Open3D from source or using prebuilt wheels with machine learning (ML) framework bindings (such as PyTorch or TensorFlow) and hardware acceleration (such as CUDA or SYCL), maintaining Application Binary Interface (ABI) compatibility across all dependencies is critical. + +An ABI mismatch between Open3D, the ML frameworks, and the underlying runtime libraries can lead to compilation failures, linker errors, or runtime crashes (such as segmentation faults). + +ABI Dependency Tree +```````````````````` + +The diagram below illustrates how Open3D and its custom operators depend on the underlying runtime ABI libraries: + +.. code-block:: text + + +-----------------------------------------------------+ + | Runtime ABI Library | + | (glibc, nvidia-rt, sycl-rt) | + +-----------+--------------+--------------+-----------+ + ^ ^ ^ + | | | + | +-----+-----+ +-----+-----+ + | | PyTorch | |TensorFlow | + | +-----+-----+ +-----+-----+ + | ^ ^ + | | | + | +-----+-----+ +-----+-----+ + | | torch-ops | | tf-ops | + | +-----+-----+ +-----+-----+ + | ^ ^ + | | | + +-----+-----+--------+--------------+-----+ + | Open3D | + +-----------------------------------------+ + +As shown in the diagram: + +* **Open3D** directly links against the core runtime ABI libraries (such as ``glibc``, ``nvidia-rt``, or ``sycl-rt``). CPU, CUDA, and SYCL code all live in the same ``libOpen3D`` shared library. +* When custom ML operators are enabled (``BUILD_PYTORCH_OPS=ON`` or ``BUILD_TENSORFLOW_OPS=ON``), Open3D compiles custom operator libraries (``torch-ops`` and ``tf-ops``). +* These custom operators depend directly on the installed ML frameworks (``pytorch`` and ``tensorflow``). +* Both the ML frameworks and the custom operators must link against the exact same runtime ABI libraries. + +Consequently, the dependency versions must be compatible. Typically, the major and minor versions of the toolchains and runtime libraries used at build time must match those used by the installed ML frameworks, and the runtime environment must satisfy the compatibility guarantees of each library. + +Runtime ABI Libraries and Compatibility Guarantees +````````````````````````````````````````````````````` + +glibc (GNU C Library) +""""""""""""""""""""" + +* **Backward Compatibility**: ``glibc`` guarantees strict backward compatibility. A binary compiled against an older version of ``glibc`` (e.g., ``glibc 2.31`` on Ubuntu 20.04) will run without issues on a system with a newer version of ``glibc`` (e.g., ``glibc 2.35`` on Ubuntu 22.04). It does **not** guarantee forward compatibility. +* **C++ Standard Library ABI (libstdc++)**: + * Open3D compiles with ``GLIBCXX_USE_CXX11_ABI=ON`` by default. + * Modern PyTorch and TensorFlow Linux wheel releases are also built with CXX11 ABI enabled (``_GLIBCXX_USE_CXX11_ABI=1``) by default. + * Open3D's CMake configuration automatically queries and verifies that the ABI configuration of the installed PyTorch and TensorFlow matches Open3D's configuration to prevent linker and runtime errors. + +nvidia-rt (NVIDIA CUDA Runtime and Driver) +"""""""""""""""""""""""""""""""""""""""""" + +* **Static Linking for Safety**: + * To prevent runtime version mismatch issues between different CUDA runtimes, Open3D builds with ``BUILD_WITH_CUDA_STATIC=ON`` by default. + * This statically links CUDA toolkit libraries (such as ``cudart_static``, ``cublas_static``, ``cusolver_static``, ``cusparse_static``, and ``npp*_static``) directly into ``libOpen3D``, isolating Open3D from external CUDA runtime version mismatches. +* **Compatibility Guarantees**: + * **Driver Backward Compatibility**: Newer NVIDIA drivers support older CUDA Toolkit and CUDA Runtime versions. + * **CUDA Minor Version Compatibility**: Starting with CUDA 11, NVIDIA guarantees binary compatibility within the same major version. An application compiled with any CUDA 11.x SDK can run on any driver that supports CUDA 11.0 or later. The same applies to CUDA 12.x and CUDA 13.x. + +sycl-rt (Intel oneAPI DPC++ Runtime) +"""""""""""""""""""""""""""""""""""" + +* **Compatibility Guarantees**: + * **oneAPI Runtime Compatibility**: Intel oneAPI guarantees backward compatibility for the DPC++ runtime (``dpcpp-cpp-rt``). A newer runtime can execute binaries compiled with an older oneAPI compiler. Forward compatibility is not supported. + * **Strict Version Matching**: Due to rapid development in SYCL standards and implementations, it is highly recommended to use matching major and minor versions of the DPC++ compiler and runtime. + * Open3D's SYCL wheels pin the runtime dependency in ``python/requirements_sycl.txt`` (e.g., ``dpcpp-cpp-rt==2025.3.1``). When compiling Open3D with SYCL support, ensure your Intel oneAPI Base Toolkit version is compatible with this runtime. + WebRTC remote visualization ``````````````````````````` diff --git a/python/open3d/__init__.py b/python/open3d/__init__.py index 87608701d5c..d199697cc7c 100644 --- a/python/open3d/__init__.py +++ b/python/open3d/__init__.py @@ -15,99 +15,47 @@ # https://github.com/dmlc/xgboost/issues/1715 import os import sys -import re +import site +import warnings os.environ["KMP_DUPLICATE_LIB_OK"] = "True" -# Enable thread composability manager to coordinate Intel OpenMP and TBB threads. Only works with Intel OpenMP. -# TBB must not be already loaded. +# Enable thread composability manager to coordinate Intel OpenMP and TBB +# threads. Only works with Intel OpenMP. TBB must not be already loaded. os.environ["TCM_ENABLE"] = "1" -from ctypes import CDLL -from ctypes.util import find_library from pathlib import Path -import warnings + from open3d._build_config import _build_config -if sys.platform == "win32": # Unix: Use rpath to find libraries - _win32_dll_dir = os.add_dll_directory(str(Path(__file__).parent)) +if sys.platform == "win32": + # Required for CPU wheel (bundled TBB) and SYCL wheel (SYCL runtime, which + # intel-sycl-rt will install in: + # - /Library/bin # (for standard/virtualenv/conda installs) + # - /Library/bin # (for user-level --user installs) + # CUDA runtime is statically linked. + _win32_dll_dirs = [os.add_dll_directory(str(Path(__file__).parent))] + _site_dirs = [*site.PREFIXES, *site.getsitepackages(), site.USER_BASE] + for _site_dir in _site_dirs: + if os.path.isdir(os.path.join(_site_dir, "Library", "bin")): + _win32_dll_dirs.append( + os.add_dll_directory(os.path.join(_site_dir, "Library", "bin"))) + +from open3d.pybind import ( + core, + camera, + data, + geometry, + io, + pipelines, + utility, + t, +) +from open3d import pybind __DEVICE_API__ = "cpu" -if _build_config["BUILD_CUDA_MODULE"]: - # Load CPU pybind dll gracefully without introducing new python variable. - # Do this before loading the CUDA pybind dll to correctly resolve symbols - try: # StopIteration if cpu version not available - CDLL(str(next((Path(__file__).parent / "cpu").glob("pybind*")))) - except StopIteration: - warnings.warn( - "Open3D was built with CUDA support, but Open3D CPU Python " - "bindings were not found. Open3D will not work on systems without" - " CUDA devices.", - ImportWarning, - ) - try: - if sys.platform == "win32" and sys.version_info >= (3, 8): - # Since Python 3.8, the PATH environment variable is not used to find DLLs anymore. - # To allow Windows users to use Open3D with CUDA without running into dependency-problems, - # look for the CUDA bin directory in PATH and explicitly add it to the DLL search path. - cuda_bin_path = None - for path in os.environ['PATH'].split(';'): - # search heuristic: look for a path containing "cuda" and "bin" in this order. - if re.search(r'cuda.*bin', path, re.IGNORECASE): - cuda_bin_path = path - break - - if cuda_bin_path: - os.add_dll_directory(cuda_bin_path) - - # Check CUDA availability without importing CUDA pybind symbols to - # prevent "symbol already registered" errors if first import fails. - _pybind_cuda = CDLL( - str(next((Path(__file__).parent / "cuda").glob("pybind*")))) - if _pybind_cuda.open3d_core_cuda_device_count() > 0: - from open3d.cuda.pybind import ( - core, - camera, - data, - geometry, - io, - pipelines, - utility, - t, - ) - from open3d.cuda import pybind - - __DEVICE_API__ = "cuda" - else: - warnings.warn( - "Open3D was built with CUDA support, but no suitable CUDA " - "devices found. If your system has CUDA devices, check your " - "CUDA drivers and runtime.", - ImportWarning, - ) - except OSError as os_error: - warnings.warn( - f"Open3D was built with CUDA support, but an error ocurred while loading the Open3D CUDA Python bindings. This is usually because the CUDA libraries could not be found. Check your CUDA installation. Falling back to the CPU pybind library. Reported error: {os_error}.", - ImportWarning, - ) - except StopIteration: - warnings.warn( - "Open3D was built with CUDA support, but Open3D CUDA Python " - "binding library not found! Falling back to the CPU Python " - "binding library.", - ImportWarning, - ) - -if __DEVICE_API__ == "cpu": - from open3d.cpu.pybind import ( - core, - camera, - data, - geometry, - io, - pipelines, - utility, - t, - ) - from open3d.cpu import pybind +if core.cuda.is_available(): + __DEVICE_API__ = "cuda" +elif core.sycl.is_available(): + __DEVICE_API__ = "xpu" def _insert_pybind_names(skip_names=()): @@ -115,10 +63,10 @@ def _insert_pybind_names(skip_names=()): python subpackages, since they have a different import mechanism.""" submodules = {} for modname in sys.modules: - if "open3d." + __DEVICE_API__ + ".pybind" in modname: + if "open3d.pybind" in modname: if any("." + skip_name in modname for skip_name in skip_names): continue - subname = modname.replace(__DEVICE_API__ + ".pybind.", "") + subname = modname.replace("open3d.pybind.", "") if subname not in sys.modules: submodules[subname] = sys.modules[modname] sys.modules.update(submodules) @@ -131,7 +79,7 @@ def _insert_pybind_names(skip_names=()): __version__ = "@PROJECT_VERSION@" if int(sys.version_info[0]) < 3: - raise Exception("Open3D only supports Python 3.") + raise RuntimeError("Open3D only supports Python 3.") if (_build_config["BUILD_JUPYTER_EXTENSION"] and os.environ.get( "OPEN3D_DISABLE_WEB_VISUALIZER", "False").lower() != "true"): @@ -210,5 +158,6 @@ def _jupyter_nbextension_paths(): if sys.platform == "win32": - _win32_dll_dir.close() -del os, sys, CDLL, find_library, Path, warnings, _insert_pybind_names + for dll_dir in _win32_dll_dirs: + dll_dir.close() +del os, sys, Path, warnings, _insert_pybind_names diff --git a/python/open3d/core/__init__.py b/python/open3d/core/__init__.py new file mode 100644 index 00000000000..078e436aa30 --- /dev/null +++ b/python/open3d/core/__init__.py @@ -0,0 +1,9 @@ +# ---------------------------------------------------------------------------- +# - Open3D: www.open3d.org - +# ---------------------------------------------------------------------------- +# Copyright (c) 2018-2024 www.open3d.org +# SPDX-License-Identifier: MIT +# ---------------------------------------------------------------------------- + +# Tensor / device API (implemented in the pybind extension). +from open3d.pybind.core import * # noqa: F403 diff --git a/python/open3d/ml/__init__.py b/python/open3d/ml/__init__.py index cf7342e0a27..5834889b534 100644 --- a/python/open3d/ml/__init__.py +++ b/python/open3d/ml/__init__.py @@ -5,12 +5,7 @@ # SPDX-License-Identifier: MIT # ---------------------------------------------------------------------------- -import os as _os -import open3d as _open3d -if _open3d.__DEVICE_API__ == 'cuda': - from open3d.cuda.pybind.ml import * -else: - from open3d.cpu.pybind.ml import * +from open3d.pybind.ml import * from . import configs from . import datasets diff --git a/python/open3d/ml/contrib/__init__.py b/python/open3d/ml/contrib/__init__.py index b883e8024fd..a431923712b 100644 --- a/python/open3d/ml/contrib/__init__.py +++ b/python/open3d/ml/contrib/__init__.py @@ -5,8 +5,4 @@ # SPDX-License-Identifier: MIT # ---------------------------------------------------------------------------- -import open3d as _open3d -if _open3d.__DEVICE_API__ == 'cuda': - from open3d.cuda.pybind.ml.contrib import * -else: - from open3d.cpu.pybind.ml.contrib import * +from open3d.pybind.ml.contrib import * diff --git a/python/open3d/ml/tf/python/ops/lib.py b/python/open3d/ml/tf/python/ops/lib.py index 4e55e26382d..ad5b994f82e 100644 --- a/python/open3d/ml/tf/python/ops/lib.py +++ b/python/open3d/ml/tf/python/ops/lib.py @@ -20,11 +20,10 @@ _package_root = _os.path.join(_this_dir, '..', '..', '..', '..') _lib_ext = {'linux': '.so', 'darwin': '.dylib', 'win32': '.dll'}[_sys.platform] _lib_suffix = '_debug' if _build_config['CMAKE_BUILD_TYPE'] == 'Debug' else '' -_lib_arch = ('cuda', 'cpu') if _build_config["BUILD_CUDA_MODULE"] else ('cpu',) -_lib_path.extend([ - _os.path.join(_package_root, la, 'open3d_tf_ops' + _lib_suffix + _lib_ext) - for la in _lib_arch -]) +# The op library is built directly into the package root (no cpu/cuda +# subfolder); CUDA availability is resolved at runtime by Open3D itself. +_lib_path.append( + _os.path.join(_package_root, 'open3d_tf_ops' + _lib_suffix + _lib_ext)) _load_except = None _loaded = False diff --git a/python/open3d/ml/torch/__init__.py b/python/open3d/ml/torch/__init__.py index 161420fbcb5..5f027067375 100644 --- a/python/open3d/ml/torch/__init__.py +++ b/python/open3d/ml/torch/__init__.py @@ -61,21 +61,17 @@ _package_root = _os.path.join(_this_dir, '..', '..') _lib_ext = {'linux': '.so', 'darwin': '.dylib', 'win32': '.dll'}[_sys.platform] _lib_suffix = '_debug' if _build_config['CMAKE_BUILD_TYPE'] == 'Debug' else '' -_lib_arch = ('cpu',) -if _build_config["BUILD_CUDA_MODULE"] and _torch.cuda.is_available(): - if _torch.version.cuda == _build_config["CUDA_VERSION"]: - _lib_arch = ('cuda', 'cpu') - else: - print("Warning: Open3D was built with CUDA {} but" - "PyTorch was built with CUDA {}. Falling back to CPU for now." - "Otherwise, install PyTorch with CUDA {}.".format( - _build_config["CUDA_VERSION"], _torch.version.cuda, - _build_config["CUDA_VERSION"])) -_lib_path.extend([ - _os.path.join(_package_root, la, - 'open3d_torch_ops' + _lib_suffix + _lib_ext) - for la in _lib_arch -]) +if (_build_config["BUILD_CUDA_MODULE"] and _torch.cuda.is_available() and + _torch.version.cuda != _build_config["CUDA_VERSION"]): + print("Warning: Open3D was built with CUDA {} but" + "PyTorch was built with CUDA {}. Falling back to CPU for now." + "Otherwise, install PyTorch with CUDA {}.".format( + _build_config["CUDA_VERSION"], _torch.version.cuda, + _build_config["CUDA_VERSION"])) +# The op library is built directly into the package root (no cpu/cuda +# subfolder); CUDA availability is resolved at runtime by Open3D itself. +_lib_path.append( + _os.path.join(_package_root, 'open3d_torch_ops' + _lib_suffix + _lib_ext)) _load_except = None _loaded = False diff --git a/python/open3d/visualization/__init__.py b/python/open3d/visualization/__init__.py index cab0cad14ad..90ed21413a1 100644 --- a/python/open3d/visualization/__init__.py +++ b/python/open3d/visualization/__init__.py @@ -6,14 +6,9 @@ # ---------------------------------------------------------------------------- import open3d -if open3d.__DEVICE_API__ == "cuda": - if open3d._build_config["BUILD_GUI"]: - from open3d.cuda.pybind.visualization import gui - from open3d.cuda.pybind.visualization import * -else: - if open3d._build_config["BUILD_GUI"]: - from open3d.cpu.pybind.visualization import gui - from open3d.cpu.pybind.visualization import * +if open3d._build_config["BUILD_GUI"]: + from open3d.pybind.visualization import gui +from open3d.pybind.visualization import * from ._external_visualizer import * from .draw_plotly import get_plotly_fig diff --git a/python/test/core/test_core.py b/python/test/core/test_core.py index 5910a5e012c..51a8d828e0c 100644 --- a/python/test/core/test_core.py +++ b/python/test/core/test_core.py @@ -127,6 +127,20 @@ def test_device(): assert o3c.Device("CUDA", 1).__str__() == "CUDA:1" + device = o3c.Device("cuda") + assert device.get_type() == o3c.Device.DeviceType.CUDA + assert device.get_id() == 0 + + +def test_tensor_to_device_string(): + t = o3c.Tensor([1, 2, 3]) + assert t.to("cpu:0").device == o3c.Device("cpu:0") + assert t.to("cpu").device == o3c.Device("cpu:0") + if o3c.cuda.is_available(): + t_gpu = t.to("cuda:0") + assert t_gpu.device.get_type() == o3c.Device.DeviceType.CUDA + assert t_gpu.to("cpu").device == o3c.Device("cpu:0") + @pytest.mark.parametrize("dtype", list_dtypes()) @pytest.mark.parametrize("device", list_devices(enable_sycl=True)) diff --git a/util/ci_utils.sh b/util/ci_utils.sh old mode 100644 new mode 100755 index 092ae3e103a..d95dfd86e04 --- a/util/ci_utils.sh +++ b/util/ci_utils.sh @@ -23,6 +23,7 @@ BUILD_TENSORFLOW_OPS=${BUILD_TENSORFLOW_OPS:-ON} BUILD_PYTORCH_OPS=${BUILD_PYTORCH_OPS:-ON} LOW_MEM_USAGE=${LOW_MEM_USAGE:-OFF} BUILD_SYCL_MODULE=${BUILD_SYCL_MODULE:-OFF} +BUILD_PYTHON_MODULE=${BUILD_PYTHON_MODULE:-ON} # Dependency versions: # CUDA: see docker/docker_build.sh @@ -120,6 +121,7 @@ build_all() { -DBUILD_UNIT_TESTS=ON -DBUILD_BENCHMARKS=ON -DBUILD_EXAMPLES=OFF + -DBUILD_PYTHON_MODULE="$BUILD_PYTHON_MODULE" ) echo @@ -132,7 +134,9 @@ build_all() { if [[ "$BUILD_SHARED_LIBS" == "ON" ]]; then make package fi - make VERBOSE=1 install-pip-package -j"$NPROC" + if [[ "$BUILD_PYTHON_MODULE" == "ON" ]]; then + make VERBOSE=1 install-pip-package -j"$NPROC" + fi echo } @@ -181,11 +185,9 @@ build_pip_package() { set -u echo - echo Building with CPU only... mkdir -p build - pushd build # PWD=Open3D/build - cmakeOptions=("-DBUILD_SHARED_LIBS=OFF" - "-DDEVELOPER_BUILD=$DEVELOPER_BUILD" + pushd build + cmakeOptions=("-DDEVELOPER_BUILD=$DEVELOPER_BUILD" "-DBUILD_COMMON_ISPC_ISAS=ON" "-DBUILD_AZURE_KINECT=$BUILD_AZURE_KINECT" "-DBUILD_LIBREALSENSE=ON" @@ -199,38 +201,44 @@ build_pip_package() { "-DBUILD_UNIT_TESTS=OFF" "-DBUILD_BENCHMARKS=OFF" "-DBUNDLE_OPEN3D_ML=$BUNDLE_OPEN3D_ML" + "-DBUILD_SHARED_LIBS=ON" ) - set -x # Echo commands on - cmake -DBUILD_CUDA_MODULE=OFF "${cmakeOptions[@]}" .. - set +x # Echo commands off - echo - - echo "Packaging Open3D CPU pip package..." - make VERBOSE=1 -j"$NPROC" pip-package - mv lib/python_package/pip_package/open3d*.whl . # save CPU wheel - if [ "$BUILD_CUDA_MODULE" == ON ]; then - echo - echo Installing CUDA versions of TensorFlow and PyTorch... install_python_dependencies with-cuda purge-cache + cmakeOptions+=("-DBUILD_CUDA_MODULE=ON" "-DBUILD_COMMON_CUDA_ARCHS=ON") + else + cmakeOptions+=("-DBUILD_CUDA_MODULE=OFF") + fi + set -x + if [ ! -f CMakeCache.txt ]; then + cmake -DBUILD_PYTHON_MODULE="${BUILD_PYTHON_MODULE}" "${cmakeOptions[@]}" .. + fi + set +x + if [ "$BUILD_PYTHON_MODULE" == "OFF" ]; then + echo "Building Open3D C++ Core only..." + make VERBOSE=1 -j"$NPROC" + else + echo "Packaging Open3D pip wheel (single configure)..." + make VERBOSE=1 -j"$NPROC" pip-package + fi + popd + + if [ "$BUILD_PYTHON_MODULE" != "OFF" ] && [ "$BUILD_CUDA_MODULE" == OFF ]; then echo - echo Building with CUDA... - rebuild_list=(bin lib/Release/*.a lib/_build_config.py cpp lib/ml) - echo - echo Removing CPU compiled files / folders: "${rebuild_list[@]}" - rm -r "${rebuild_list[@]}" || true - set -x # Echo commands on - cmake -DBUILD_CUDA_MODULE=ON \ - -DBUILD_COMMON_CUDA_ARCHS=ON \ - "${cmakeOptions[@]}" .. - set +x # Echo commands off + echo "Building open3d-cpu wheel..." + mkdir -p build_cpu + pushd build_cpu + if [ ! -f CMakeCache.txt ]; then + cmake -DBUILD_CUDA_MODULE=OFF -DBUILD_PYTHON_MODULE=ON \ + "${cmakeOptions[@]}" .. + fi + make VERBOSE=1 -j"$NPROC" pip-package + popd + mkdir -p build/lib/python_package/pip_package + cp build_cpu/lib/python_package/pip_package/open3d_cpu*.whl \ + build/lib/python_package/pip_package/ || true fi echo - - echo "Packaging Open3D full pip package..." - make VERBOSE=1 -j"$NPROC" pip-package - mv open3d*.whl lib/python_package/pip_package/ # restore CPU wheel - popd # PWD=Open3D } # Test wheel in blank virtual environment diff --git a/util/run_ci.sh b/util/run_ci.sh index be1e4a1a662..b04e758652b 100755 --- a/util/run_ci.sh +++ b/util/run_ci.sh @@ -18,7 +18,9 @@ echo df -h echo "Running Open3D C++ unit tests..." -run_cpp_unit_tests +if [ "${SKIP_CPP_TESTS:-OFF}" != "ON" ]; then + run_cpp_unit_tests +fi # Run on GPU only. CPU versions run on Github already if nvidia-smi >/dev/null 2>&1; then From 447c814e9c843592d4bb36716d939be6b8e9b82b Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Mon, 6 Jul 2026 21:25:13 -0700 Subject: [PATCH 2/3] Fix CI regressions from ss/dll port (PR #7516) Several bugs slipped through when cherry-picking CI/Docker changes and the simplified single-pybind-module python packaging from copilot/separate-pybind-and-libopen3d: - python/open3d/__init__.py: _insert_pybind_names() stripped the "open3d." prefix (not just "pybind."), so e.g. "open3d.pybind.t" was registered as bare "t" instead of "open3d.t", breaking `import open3d.t` and any `from open3d.t... import ...`. This failed pytest collection on the Ubuntu and OpenBLAS CI jobs. - .github/workflows/windows.yml: BUILD_WEBRTC was hardcoded to 'OFF', but main enables it for the STATIC_RUNTIME=ON + BUILD_SHARED_LIBS=OFF matrix cells, since our own prebuilt BoringSSL is built with the dynamic CRT and is only ABI-compatible when WebRTC (built with matching static CRT) supplies crypto/ssl symbols instead. Restored the conditional; this fixes LNK2001 "unresolved external symbol __imp_bsearch" failures in the three static-runtime build-lib jobs. - .github/workflows/ubuntu-cuda.yml: GCE image family was bumped to "ubuntu-os-docker-gpu-2204-lts", which does not exist in the GCP project ("was not found" at instance-create time). Reverted to main's working "ubuntu-os-docker-gpu-2004-lts" (the GCE host image is independent of the Ubuntu version used inside the CI Docker container). - .github/workflows/ubuntu-wheel.yml: the new split-out "build-lib" job calls docker_build.sh, which requires BUILD_TENSORFLOW_OPS/ BUILD_PYTORCH_OPS to be set (bash `set -u`); these were set on the build-wheel job but missing on build-lib, causing "unbound variable". Not fixed (pre-existing/flaky, reproduced identically on unrelated branches, unrelated to this port): macOS gfortran symlink lookup failing on macos-14 runners, and transient pyenv-installer network fetch failures on ARM64/py313+ wheel builds. Verified: reproduced the "open3d.t" ModuleNotFoundError from the CI logs using a local minimal CPU-only build (BUILD_PYTHON_MODULE=ON, no GUI/tests/ ISPC/WebRTC) and confirmed `import open3d.t`, `from open3d.t.geometry import Image`, and pytest collection of python/test/t/geometry/test_image.py all work after the fix; python/test/core/test_core.py (279 tests) still passes. Co-authored-by: Cursor --- .github/workflows/ubuntu-cuda.yml | 2 +- .github/workflows/ubuntu-wheel.yml | 4 ++++ .github/workflows/windows.yml | 4 +++- python/open3d/__init__.py | 5 ++++- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ubuntu-cuda.yml b/.github/workflows/ubuntu-cuda.yml index c1080d8635e..9b6579922dc 100644 --- a/.github/workflows/ubuntu-cuda.yml +++ b/.github/workflows/ubuntu-cuda.yml @@ -118,7 +118,7 @@ jobs: --machine-type=n1-standard-8 \ --boot-disk-size="128GB" \ --boot-disk-type="pd-ssd" \ - --image-family="ubuntu-os-docker-gpu-2204-lts" \ + --image-family="ubuntu-os-docker-gpu-2004-lts" \ --metadata-from-file=startup-script=./util/gcloud_auto_clean.sh \ --scopes="storage-full,compute-rw" \ --service-account="$GCE_GPU_CI_SA"; do diff --git a/.github/workflows/ubuntu-wheel.yml b/.github/workflows/ubuntu-wheel.yml index 607dbf77464..3cadfd0095d 100644 --- a/.github/workflows/ubuntu-wheel.yml +++ b/.github/workflows/ubuntu-wheel.yml @@ -30,6 +30,10 @@ jobs: DEVELOPER_BUILD: 'ON' CCACHE_TAR_NAME: open3d-ubuntu-2204-cuda-ci-ccache OPEN3D_CPU_RENDERING: true + # Required by cuda_wheel_build() in docker/docker_build.sh (py310 is + # used as a dummy Python version to build the shared lib layer only). + BUILD_PYTORCH_OPS: 'ON' + BUILD_TENSORFLOW_OPS: 'ON' steps: - name: Checkout source code uses: actions/checkout@v4 diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index a7180bf8cd4..fafb0592718 100755 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -56,7 +56,9 @@ jobs: - BUILD_CUDA_MODULE: ON # FIXME CONFIG: Debug env: - BUILD_WEBRTC: 'OFF' # TODO: WebRTC DLL not available for Windows; re-enable when fixed + # WebRTC is only built (as a static lib) for the static-runtime, + # static-library config; its DLL is not available for Windows yet. + BUILD_WEBRTC: ${{ ( matrix.BUILD_SHARED_LIBS == 'OFF' && matrix.STATIC_RUNTIME == 'ON' ) && 'ON' || 'OFF' }} BUILD_PYTORCH_OPS: ${{ ( matrix.BUILD_CUDA_MODULE == 'ON' || matrix.CONFIG == 'Debug' ) && 'OFF' || 'ON' }} # FIXME steps: diff --git a/python/open3d/__init__.py b/python/open3d/__init__.py index d199697cc7c..5b9e23454ac 100644 --- a/python/open3d/__init__.py +++ b/python/open3d/__init__.py @@ -66,7 +66,10 @@ def _insert_pybind_names(skip_names=()): if "open3d.pybind" in modname: if any("." + skip_name in modname for skip_name in skip_names): continue - subname = modname.replace("open3d.pybind.", "") + # Keep the leading "open3d." so submodules are registered under + # e.g. "open3d.t" rather than a bare "t" (which is not importable + # via `import open3d.t`). + subname = modname.replace("pybind.", "") if subname not in sys.modules: submodules[subname] = sys.modules[modname] sys.modules.update(submodules) From b4ec5d8ca235a5e18d77bc14bc7c37fa73131413 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey <41028320+ssheorey@users.noreply.github.com> Date: Mon, 6 Jul 2026 23:41:57 -0700 Subject: [PATCH 3/3] Always install gcc to find gfortran --- .github/workflows/macos.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 4957a6302f5..52ae0adb05d 100755 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -68,6 +68,7 @@ jobs: run: | brew install ccache glslang spirv-cross # Fix gfortran not found issue + brew install gcc ln -s $(brew --prefix gcc)/bin/gfortran-* /usr/local/bin/gfortran ccache -M 2G # See .github/workflows/readme.md for ccache strategy. @@ -221,8 +222,9 @@ jobs: source util/ci_utils.sh install_python_dependencies - # Fix macos-14 arm64 runner image issues, see comments in MacOS job. - ln -s $(which gfortran-13) /usr/local/bin/gfortran + # Fix gfortran not found issue + brew install gcc + ln -s $(brew --prefix gcc)/bin/gfortran-* /usr/local/bin/gfortran brew install ccache glslang spirv-cross ccache -M 2G # See .github/workflows/readme.md for ccache strategy.