feat: slim down zvec dynamic libraries

.github/workflows/04-android-build.yml

@@ -71,33 +71,8 @@ jobs:
           sdkmanager --install "system-images;android-${{ matrix.api }};google_apis_playstore;x86_64" 2>/dev/null || \
           sdkmanager --install "system-images;android-${{ matrix.api }};default;x86_64"
 
-      # ── Step 1: build host protoc (using HOST compiler, NOT NDK) ───────
-      - name: Cache host protoc
-        uses: actions/cache@v6
-        with:
-          path: build_host
-          key: ${{ runner.os }}-host-protoc-${{ hashFiles('thirdparty/protobuf/**', 'CMakeLists.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-host-protoc-
-
-      - name: 'Step 1: Build host protoc'
-        shell: bash
-        run: |
-          if [ ! -f "build_host/bin/protoc" ]; then
-            git submodule foreach --recursive 'git stash --include-untracked' 2>/dev/null || true
-            cmake -S . -B build_host \
-              -DCMAKE_BUILD_TYPE=Release \
-              -DCMAKE_TOOLCHAIN_FILE="" \
-              -DCMAKE_C_COMPILER_LAUNCHER=ccache \
-              -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-              -G Ninja
-            cmake --build build_host --target protoc --parallel
-          else
-            echo "Using cached host protoc"
-          fi
-
-      # ── Step 2: cross-compile zvec + tests for Android ─────────────────
-      - name: 'Step 2: Cross-compile zvec and tests'
+      # ── Step 1: cross-compile zvec + tests for Android ─────────────────────────
+      - name: 'Step 1: Cross-compile zvec and tests'
         shell: bash
         env:
           BUILD_DIR: build_android_${{ matrix.abi }}
@@ -123,7 +98,6 @@ jobs:
             -DAUTO_DETECT_ARCH=OFF \
             -DENABLE_WERROR=ON \
             -DCMAKE_INSTALL_PREFIX="$BUILD_DIR/install" \
-            -DGLOBAL_CC_PROTOBUF_PROTOC="$GITHUB_WORKSPACE/build_host/bin/protoc" \
             -DCMAKE_C_COMPILER_LAUNCHER=ccache \
             -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
 
@@ -150,8 +124,8 @@ jobs:
           echo "Building ${#TEST_NAMES[@]} test executables..."
           ninja -C "$BUILD_DIR" -j$(nproc) "${TEST_NAMES[@]}"
 
-      # ── Step 3: start emulator ─────────────────────────────────────────
-      - name: 'Step 3: Start Android emulator'
+      # ── Step 2: start emulator ─────────────────────────────────────────
+      - name: 'Step 2: Start Android emulator'
         shell: bash
         run: |
           AVD_NAME="zvec_test_avd"
@@ -244,8 +218,8 @@ jobs:
           echo "Device ABI: $(adb shell getprop ro.product.cpu.abi | tr -d '\r')"
           echo "ABI list  : $(adb shell getprop ro.product.cpu.abilist | tr -d '\r')"
 
-      # ── Step 4: run unit tests on emulator ─────────────────────────────
-      - name: 'Step 4: Run unit tests on emulator'
+      # ── Step 3: run unit tests on emulator ─────────────────────────────
+      - name: 'Step 3: Run unit tests on emulator'
         shell: bash
         env:
           BUILD_DIR: build_android_${{ matrix.abi }}
@@ -385,8 +359,8 @@ jobs:
           fi
           echo "All tests passed!"
 
-      # ── Step 5: build and run examples ─────────────────────────────────
-      - name: 'Step 5: Build and run examples'
+      # ── Step 4: build and run examples ─────────────────────────────────
+      - name: 'Step 4: Build and run examples'
         shell: bash
         env:
           BUILD_DIR: build_android_${{ matrix.abi }}
@@ -405,8 +379,8 @@ jobs:
             -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
           cmake --build "$EXAMPLES_BUILD" --parallel
 
-          # Reuse the shared-library directory from Step 4; push again in
-          # case Step 4 was skipped or the directory was cleaned.
+          # Reuse the shared-library directory from Step 3; push again in
+          # case Step 3 was skipped or the directory was cleaned.
           DEVICE_LIB_DIR="/data/local/tmp/zvec_tests/lib"
           adb shell "mkdir -p $DEVICE_LIB_DIR" 2>/dev/null || true
           SO_COUNT=0

.github/workflows/06-ios-build.yml

@@ -37,25 +37,6 @@ jobs:
           key: ios-${{ matrix.platform }}
           max-size: 150M
 
-      - name: Cache host protoc build
-        uses: actions/cache@v6
-        with:
-          path: build_host
-          key: macos-host-protoc-${{ hashFiles('thirdparty/protobuf/**', 'CMakeLists.txt') }}
-          restore-keys: |
-            macos-host-protoc-
-
-      - name: Build host protoc
-        run: |
-          if [ ! -f "build_host/bin/protoc" ]; then
-            cmake -S . -B build_host -DCMAKE_BUILD_TYPE=Release -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \
-              -DCMAKE_C_COMPILER_LAUNCHER=ccache \
-              -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
-            cmake --build build_host --target protoc --parallel $(sysctl -n hw.ncpu)
-          else
-            echo "Using cached host protoc"
-          fi
-
       - name: Configure and Build
         run: |
           git submodule foreach --recursive 'git stash --include-untracked' || true
@@ -73,7 +54,6 @@ jobs:
             -DBUILD_TOOLS=OFF \
             -DENABLE_WERROR=ON \
             -DCMAKE_INSTALL_PREFIX="./install" \
-            -DGLOBAL_CC_PROTOBUF_PROTOC="$GITHUB_WORKSPACE/build_host/bin/protoc" \
             -DIOS=ON \
             -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \
             -DCMAKE_C_COMPILER_LAUNCHER=ccache \

.github/workflows/clang_tidy.yml

@@ -134,8 +134,7 @@ jobs:
           path: |
             build/external
             build/thirdparty
-            build/src/db/proto
-          key: clang-tidy-headers-${{ runner.os }}-${{ hashFiles('thirdparty/**/*.cmake', 'thirdparty/**/CMakeLists.txt', 'thirdparty/**/*.patch', 'src/db/proto/*.proto') }}-${{ steps.submodule_hash.outputs.value }}
+          key: clang-tidy-headers-${{ runner.os }}-${{ hashFiles('thirdparty/**/*.cmake', 'thirdparty/**/CMakeLists.txt', 'thirdparty/**/*.patch') }}-${{ steps.submodule_hash.outputs.value }}
 
       - name: Build generated headers only
         if: steps.tidy_files.outputs.any_tidy_files == 'true' && steps.cache_headers.outputs.cache-hit != 'true'
@@ -149,8 +148,7 @@ jobs:
           path: |
             build/external
             build/thirdparty
-            build/src/db/proto
-          key: clang-tidy-headers-${{ runner.os }}-${{ hashFiles('thirdparty/**/*.cmake', 'thirdparty/**/CMakeLists.txt', 'thirdparty/**/*.patch', 'src/db/proto/*.proto') }}-${{ steps.submodule_hash.outputs.value }}
+          key: clang-tidy-headers-${{ runner.os }}-${{ hashFiles('thirdparty/**/*.cmake', 'thirdparty/**/CMakeLists.txt', 'thirdparty/**/*.patch') }}-${{ steps.submodule_hash.outputs.value }}
 
       - name: Run clang-tidy on changed files (parallel)
         if: steps.tidy_files.outputs.any_tidy_files == 'true'

.gitmodules

@@ -24,9 +24,6 @@
 	path = thirdparty/glog/glog-0.5.0
 	url = https://github.com/google/glog.git
 	ignore = dirty
-[submodule "thirdparty/protobuf/protobuf-3.21.12"]
-	path = thirdparty/protobuf/protobuf-3.21.12
-	url = https://github.com/protocolbuffers/protobuf.git
 [submodule "thirdparty/lz4/lz4-1.9.4"]
 	path = thirdparty/lz4/lz4-1.9.4
 	url = https://github.com/lz4/lz4.git

CMakeLists.txt

@@ -38,6 +38,9 @@ if(MSVC)
 else()
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror=return-type")
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror=return-type")
+    # Fine-grained dead code elimination: place each function/data in its own section
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ffunction-sections -fdata-sections")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ffunction-sections -fdata-sections")
 endif()
 
 if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND NOT IOS)
@@ -49,6 +52,21 @@ if(NOT DEFINED PROJECT_ROOT_DIR OR NOT PROJECT_ROOT_DIR)
     set(PROJECT_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR} CACHE PATH "Root directory of the project" FORCE)
 endif()
 
+# LTO (Link-Time Optimization) — helps on Linux with --gc-sections but may
+# increase size on macOS due to aggressive inlining.  Default OFF; enable
+# explicitly with -DZVEC_ENABLE_LTO=ON when targeting Linux.
+option(ZVEC_ENABLE_LTO "Enable Link-Time Optimization for Release builds" OFF)
+if(ZVEC_ENABLE_LTO AND CMAKE_BUILD_TYPE STREQUAL "Release")
+    include(CheckIPOSupported)
+    check_ipo_supported(RESULT IPO_SUPPORTED OUTPUT IPO_OUTPUT)
+    if(IPO_SUPPORTED)
+        set(CMAKE_INTERPROCEDURAL_OPTIMIZATION ON)
+        message(STATUS "LTO enabled for Release build")
+    else()
+        message(STATUS "LTO not supported: ${IPO_OUTPUT}")
+    endif()
+endif()
+
 message(STATUS "PROJECT_ROOT_DIR = ${PROJECT_ROOT_DIR}")
 
 include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
@@ -144,7 +162,7 @@ cc_directories(src)
 
 cc_directories(tests)
 
-add_custom_target(clang_tidy_deps DEPENDS zvec_proto ARROW.BUILD glog gflags Lz4.BUILD)
+add_custom_target(clang_tidy_deps DEPENDS ARROW.BUILD glog gflags Lz4.BUILD)
 
 if(BUILD_TOOLS)
     cc_directories(tools)

cmake/bazel.cmake

@@ -1502,7 +1502,7 @@ function(_find_protobuf _VERSION)
       "${protoc_INCLUDE_DIR}" CACHE STRING "Protobuf includes"
     )
   set(
-      CC_PROTOBUF_LIBS_${_VERSION} libprotobuf CACHE STRING "Protobuf libraries"
+      CC_PROTOBUF_LIBS_${_VERSION} libprotobuf-lite CACHE STRING "Protobuf libraries"
     )
 endfunction()
 

scripts/build_android.sh

@@ -19,7 +19,7 @@ API_LEVEL=${1:-35}
 BUILD_TYPE=${2:-"Release"}
 CORE_COUNT=$(sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 4)
 
-# ── Android SDK paths (set later, after host protoc build) ───────────
+# ── Android SDK paths ────────────────────────────────────────────────
 ANDROID_SDK_ROOT=${ANDROID_SDK_ROOT:-$HOME/Library/Android/sdk}
 ANDROID_NDK_HOME=${ANDROID_NDK_HOME:-$(ls -d "$ANDROID_SDK_ROOT/ndk/"* 2>/dev/null | sort -V | tail -1)}
 
@@ -39,40 +39,9 @@ if [ ! -d "$ANDROID_NDK_HOME" ]; then
     exit 1
 fi
 
-# ── Step 1: build host protoc (using HOST compiler, NOT NDK) ─────────
+# ── Step 1: cross-compile zvec + tests for Android ───────────────────
 echo ""
-echo ">>> Step 1: Building protoc for host..."
-HOST_BUILD_DIR="build_host"
-
-git submodule foreach --recursive 'git stash --include-untracked' 2>/dev/null || true
-
-if [ ! -f "$CURRENT_DIR/$HOST_BUILD_DIR/bin/protoc" ]; then
-    # Explicitly avoid NDK toolchain for host build
-    cmake -S . -B "$HOST_BUILD_DIR" \
-        -DCMAKE_BUILD_TYPE="$BUILD_TYPE" \
-        -DCMAKE_TOOLCHAIN_FILE="" \
-        -G Ninja
-    cmake --build "$HOST_BUILD_DIR" --target protoc -j"$CORE_COUNT"
-else
-    echo "  (cached — skipping)"
-fi
-PROTOC_EXECUTABLE=$CURRENT_DIR/$HOST_BUILD_DIR/bin/protoc
-echo ">>> Step 1: Done (protoc=$PROTOC_EXECUTABLE)"
-
-# ── Now export Android env vars for cross-compilation ────────────────
-export ANDROID_SDK_ROOT
-export ANDROID_HOME=$ANDROID_SDK_ROOT
-export ANDROID_NDK_HOME
-export CMAKE_TOOLCHAIN_FILE=$ANDROID_NDK_HOME/build/cmake/android.toolchain.cmake
-
-export PATH=$PATH:$ANDROID_SDK_ROOT/cmdline-tools/latest/bin
-export PATH=$PATH:$ANDROID_SDK_ROOT/platform-tools
-export PATH=$PATH:$ANDROID_SDK_ROOT/emulator
-export PATH=$PATH:$ANDROID_NDK_HOME
-
-# ── Step 2: cross-compile zvec + tests for Android ───────────────────
-echo ""
-echo ">>> Step 2: Cross-compiling zvec for Android ($ABI, API $API_LEVEL)..."
+echo ">>> Step 1: Cross-compiling zvec for Android ($ABI, API $API_LEVEL)..."
 
 # reset thirdparty so the cross toolchain can patch cleanly
 git submodule foreach --recursive 'git stash --include-untracked' 2>/dev/null || true
@@ -94,7 +63,6 @@ cmake -S . -B "$BUILD_DIR" -G Ninja \
     -DENABLE_NATIVE=OFF \
     -DAUTO_DETECT_ARCH=OFF \
     -DCMAKE_INSTALL_PREFIX="$BUILD_DIR/install" \
-    -DGLOBAL_CC_PROTOBUF_PROTOC="$PROTOC_EXECUTABLE"
 
 echo "  Building library..."
 cmake --build "$BUILD_DIR" -j"$CORE_COUNT"
@@ -130,11 +98,11 @@ fi
 echo "  Building ${#TEST_NAMES[@]} test executables..."
 ninja -C "$BUILD_DIR" -j"$CORE_COUNT" "${TEST_NAMES[@]}"
 
-echo ">>> Step 2: Done"
+echo ">>> Step 1: Done"
 
-# ── Step 3: collect test binaries ────────────────────────────────────
+# ── Step 2: collect test binaries ────────────────────────────────────
 echo ""
-echo ">>> Step 3: Collecting test binaries..."
+echo ">>> Step 2: Collecting test binaries..."
 
 TEST_BINS=()
 for name in "${TEST_NAMES[@]}"; do

scripts/build_ios.sh

@@ -34,24 +34,8 @@ echo "  Architecture: $ARCH"
 echo "  Build Type: $BUILD_TYPE"
 echo "  iOS Deployment Target: $IOS_DEPLOYMENT_TARGET"
 
-# step1: use host env to compile protoc
-echo "step1: building protoc for host..."
-
-git submodule foreach --recursive 'git stash --include-untracked'
-
-HOST_BUILD_DIR="build_host"
-mkdir -p $HOST_BUILD_DIR
-cd $HOST_BUILD_DIR
-
-cmake -DCMAKE_BUILD_TYPE="$BUILD_TYPE" ..
-make -j protoc
-PROTOC_EXECUTABLE=$CURRENT_DIR/$HOST_BUILD_DIR/bin/protoc
-cd $CURRENT_DIR
-
-echo "step1: Done!!!"
-
-# step2: cross build zvec for iOS
-echo "step2: building zvec for iOS..."
+# step1: cross build zvec for iOS
+echo "step1: building zvec for iOS..."
 
 # reset thirdparty directory
 git submodule foreach --recursive 'git stash --include-untracked'
@@ -79,19 +63,18 @@ cmake \
     -DBUILD_PYTHON_BINDINGS=OFF \
     -DBUILD_TOOLS=OFF \
     -DCMAKE_INSTALL_PREFIX="./install" \
-    -DGLOBAL_CC_PROTOBUF_PROTOC=$PROTOC_EXECUTABLE \
     -DIOS=ON \
     ../
 
 echo "building..."
 CORE_COUNT=$(sysctl -n hw.ncpu)
 make -j$CORE_COUNT
 
-echo "step2: Done!!!"
+echo "step1: Done!!!"
 
-# step3: build and run all unit tests on simulator
+# step2: build and run all unit tests on simulator
 if [ "$PLATFORM" != "OS" ]; then
-  echo "step3: building and running unit tests on simulator..."
+  echo "step2: building and running unit tests on simulator..."
 
   make -j$CORE_COUNT unittest
 
@@ -163,7 +146,7 @@ sys.exit(1)
     exit 1
   fi
 
-  echo "step3: Done!!!"
+  echo "step2: Done!!!"
 else
   echo "Skipping tests (device build cannot run on simulator)"
 fi

src/CMakeLists.txt

@@ -146,6 +146,30 @@ function(zvec_add_all_in_one_shared TARGET_NAME OUTPUT_NAME)
         endif()
     endif()
 
+    # Restrict exported symbols to zvec's own public C++ API (namespace zvec,
+    # which includes nested zvec::ailego / zvec::core / zvec::turbo /
+    # zvec::reranker). All third-party symbols (arrow, rocksdb, parquet,
+    # protobuf, antlr4) are hidden, and unreferenced code is dead-stripped to
+    # reduce the shared library size. The public C++ headers never expose
+    # third-party types, so hiding them is safe for external C++ consumers.
+    if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
+        if(APPLE)
+            set(_ZVEC_ALLIN_EXPORTS ${CMAKE_CURRENT_SOURCE_DIR}/exported_symbols_cpp.txt)
+            target_link_options(${TARGET_NAME} PRIVATE
+                "-Wl,-exported_symbols_list,${_ZVEC_ALLIN_EXPORTS}"
+                "-Wl,-dead_strip")
+            set_property(TARGET ${TARGET_NAME} APPEND PROPERTY
+                LINK_DEPENDS ${_ZVEC_ALLIN_EXPORTS})
+        elseif(UNIX)
+            set(_ZVEC_ALLIN_EXPORTS ${CMAKE_CURRENT_SOURCE_DIR}/exported_symbols_cpp.lds)
+            target_link_options(${TARGET_NAME} PRIVATE
+                "-Wl,--version-script,${_ZVEC_ALLIN_EXPORTS}"
+                "-Wl,--gc-sections")
+            set_property(TARGET ${TARGET_NAME} APPEND PROPERTY
+                LINK_DEPENDS ${_ZVEC_ALLIN_EXPORTS})
+        endif()
+    endif()
+
     target_include_directories(${TARGET_NAME}
         PUBLIC
             $<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/src/generated>

src/binding/c/CMakeLists.txt

@@ -222,6 +222,21 @@ target_compile_options(zvec_c_api PRIVATE
     $<$<CXX_COMPILER_ID:Clang>:-Wall -Wextra -Wpedantic>
 )
 
+# Linker-level dead code elimination with explicit export list.
+# Only symbols in the export list are kept as global; the linker can then
+# aggressively strip all code unreachable from those roots.
+if(APPLE)
+    target_link_options(zvec_c_api PRIVATE
+        -Wl,-dead_strip
+        -Wl,-exported_symbols_list,${CMAKE_CURRENT_SOURCE_DIR}/exported_symbols.txt
+    )
+elseif(UNIX)
+    target_link_options(zvec_c_api PRIVATE
+        -Wl,--gc-sections
+        -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/exported_symbols.lds
+    )
+endif()
+
 # Strip symbols in release builds to reduce library size
 if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
     if(UNIX AND NOT APPLE)