diff --git a/.github/workflows/build-libseekdb.yml b/.github/workflows/build-libseekdb.yml
new file mode 100644
index 000000000..afd9e60d7
--- /dev/null
+++ b/.github/workflows/build-libseekdb.yml
@@ -0,0 +1,901 @@
+# Build, pack and upload libseekdb for multiple platforms (linux x64/arm64, macos arm64, windows x64, android arm64-v8a) to S3
+#
+# Reference build environments (use these systems/env as the standard):
+# linux-x64: runner ubuntu-22.04, container quay.io/pypa/manylinux2014_x86_64 (glibc 2.17, CentOS 7+), zip libseekdb-linux-x64.zip
+# linux-arm64: runner ubuntu-22.04-arm, container quay.io/pypa/manylinux2014_aarch64 (glibc 2.17), zip libseekdb-linux-arm64.zip
+# darwin-arm64: runner macos-14, native, zip libseekdb-darwin-arm64.zip (min macOS 11.0)
+# windows-x64: runner windows-2022, .\build.ps1 + libseekdb-build.ps1, zip libseekdb-windows-x64.zip
+# android-arm64-v8a: runner macos-14, NDK + ./build.sh --android, zip libseekdb-android-arm64-v8a.zip
+#
+# macOS builds use runner macos-14 and set CMAKE_OSX_DEPLOYMENT_TARGET=11.0 so the dylib runs on macOS 11+ (Big Sur and later).
+# On macOS, dylibs are signed in libseekdb-build.sh: ad-hoc when no cert; when repo is oceanbase/seekdb and secrets are set,
+# use Developer ID (secrets: OSX_CODESIGN_BUILD_CERTIFICATE_BASE64, OSX_CODESIGN_P12_PASSWORD, OSX_CODESIGN_KEYCHAIN_PASSWORD, OSX_CODESIGN_IDENTITY).
+# Optional: add notarization step and APPLE_ID/PASSWORD/TEAM_ID secrets to notarize the zip.
+# Artifacts: platform zips including libseekdb-android-arm64-v8a.zip; combined artifact libseekdb-all-platforms; optional S3 upload when DESTINATION_TARGET_PATH or AWS_S3_BUCKET and credentials are set.
+#
+# Job step order: Checkout → caches → compile → setup Node/Rust/Go/Java → FFI binding tests (continue-on-error per language
+# so all languages run; final summary step fails the job if any binding test failed) → pack → upload artifact → save caches (always).
+# Android: compile → setup toolchains → host-skip FFI notice → pack → verify zip → upload → save caches → summary (see job comment).
+name: Build libseekdb
+run-name: Build libseekdb for ${{ github.event_name == 'workflow_dispatch' && inputs.ref != '' && inputs.ref || (github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha) }}
+
+# PRs: pull_request. Direct pushes: only long-lived / release lines — avoids duplicate runs when the same commit
+# is both "push" to a feature branch and "pull_request" sync (open PR to upstream from that branch).
+# S3 upload still only on main, master, develop, *.*.x, integration/* (UPLOAD_S3; see release-artifacts job).
+on:
+ push:
+ branches:
+ - main
+ - master
+ - develop
+ - "integration/**"
+ - "release/**"
+ # e.g. 1.0.x; aligns with S3 / UPLOAD_S3 for dot-x lines
+ - "*.*.x"
+ paths-ignore:
+ - "*.md"
+ - "LICENSE"
+ - "CODEOWNERS"
+ - "docs/**"
+ workflow_dispatch:
+ inputs:
+ ref:
+ description: "Branch, tag or commit SHA to build (empty = use default branch)"
+ required: false
+ type: string
+ default: ""
+ pull_request:
+ paths-ignore:
+ - "*.md"
+ - "LICENSE"
+ - "CODEOWNERS"
+ - "docs/**"
+
+env:
+ COMMIT_SHA: ${{ github.event_name == 'workflow_dispatch' && inputs.ref != '' && inputs.ref || (github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha) }}
+ AWS_REGION: ${{ vars.AWS_REGION || 'ap-southeast-1' }}
+ UPLOAD_S3: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/heads/integration/') || contains(github.ref, '.x'))) }}
+ BUCKET_NAME: ${{ vars.AWS_S3_BUCKET || 'oceanbase-seekdb-builds' }}
+ DESTINATION_TARGET_PATH: ${{ vars.DESTINATION_TARGET_PATH || format('s3://oceanbase-seekdb-builds/libseekdb/all_commits/{0}', github.event_name == 'workflow_dispatch' && inputs.ref != '' && inputs.ref || (github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha)) }}
+ S3_BUCKET: ${{ vars.AWS_S3_BUCKET || 'oceanbase-seekdb-builds' }}
+ S3_PREFIX: libseekdb/all_commits/${{ github.event_name == 'workflow_dispatch' && inputs.ref != '' && inputs.ref || (github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha) }}
+
+jobs:
+ # ---------- Build libseekdb on Linux / macOS ----------
+ build:
+ name: Build libseekdb (${{ matrix.platform }})
+ runs-on: ${{ matrix.runner }}
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - platform: linux-x64
+ runner: ubuntu-22.04
+ artifact_name: libseekdb-linux-x64
+ container_image: quay.io/pypa/manylinux2014_x86_64
+ deps_file: oceanbase.el7.x86_64.deps
+ - platform: linux-arm64
+ runner: ubuntu-22.04-arm
+ artifact_name: libseekdb-linux-arm64
+ container_image: quay.io/pypa/manylinux2014_aarch64
+ deps_file: oceanbase.el7.aarch64.deps
+
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+ with:
+ # workflow_dispatch: use inputs.ref if set; PR: head sha; push: event sha
+ ref: ${{ github.event_name == 'workflow_dispatch' && inputs.ref != '' && inputs.ref || (github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha) }}
+
+ - name: Cache deps (Linux, el7)
+ uses: actions/cache@v4
+ with:
+ path: deps/3rd
+ key: ${{ runner.os }}-libseekdb-deps-${{ matrix.platform }}-el7-${{ hashFiles(format('deps/init/{0}', matrix.deps_file)) }}
+ restore-keys: |
+ ${{ runner.os }}-libseekdb-deps-${{ matrix.platform }}-el7-
+
+ - name: Cache ccache (Linux)
+ uses: actions/cache@v4
+ with:
+ path: .ccache
+ # Static key per platform so second run always hits; ccache content is keyed by source hash internally
+ key: ${{ runner.os }}-ccache-libseekdb-${{ matrix.platform }}
+ restore-keys: |
+ ${{ runner.os }}-ccache-libseekdb-${{ matrix.platform }}
+
+ # Run build inside manylinux2014 so Node/actions run on host (glibc 2.28+), build runs in CentOS 7 (glibc 2.17) for compatibility
+ - name: Build libseekdb (Linux, manylinux2014)
+ env:
+ BUILD_TYPE: release
+ run: |
+ docker run --rm -u root \
+ -v "$GITHUB_WORKSPACE:$GITHUB_WORKSPACE" -w "$GITHUB_WORKSPACE" \
+ -e BUILD_TYPE -e GITHUB_WORKSPACE \
+ ${{ matrix.container_image }} \
+ bash -c '
+ set -e
+ # Install deps in two steps: base first (wget required for dep_create.sh), then ccache if available
+ yum install -y git wget rpm cpio make glibc-devel glibc-headers binutils m4 python3 python3-devel libtool libaio ncurses-devel which zlib-devel
+ yum install -y epel-release 2>/dev/null || true
+ yum install -y ccache 2>/dev/null || true
+ git config --global --add safe.directory "$GITHUB_WORKSPACE"
+ export CCACHE_DIR="$GITHUB_WORKSPACE/.ccache"
+ export CCACHE_COMPILERCHECK=content
+ export CCACHE_NOHASHDIR=1
+ mkdir -p deps/3rd/usr/local/oceanbase/devtools/bin
+ CCACHE_SRC=$(command -v ccache 2>/dev/null || true)
+ if [ -n "$CCACHE_SRC" ] && [ -x "$CCACHE_SRC" ]; then
+ ln -sf "$CCACHE_SRC" deps/3rd/usr/local/oceanbase/devtools/bin/ccache
+ USE_CCACHE="-DOB_USE_CCACHE=ON"
+ else
+ USE_CCACHE="-DOB_USE_CCACHE=OFF"
+ fi
+ export PATH="$GITHUB_WORKSPACE/deps/3rd/usr/local/oceanbase/devtools/bin:$PATH"
+ bash build.sh init
+ if [ -x /opt/python/cp39-cp39/bin/python3.9 ]; then
+ export PATH="/opt/python/cp39-cp39/bin:$PATH"
+ PYVER=3.9
+ else
+ PYVER=$(python3 -c "import sys; print(f\"{sys.version_info.major}.{sys.version_info.minor}\")")
+ fi
+ bash build.sh release --init $USE_CCACHE -DBUILD_EMBED_MODE=ON -DPYTHON_VERSION=$PYVER --make libseekdb
+ [ -n "$USE_CCACHE" ] && ccache -s || true
+ '
+ - name: Fix ownership (container writes as root)
+ run: sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE"
+
+ - name: Setup Node.js (Linux)
+ uses: actions/setup-node@v4
+ with:
+ node-version: "18"
+
+ - name: Setup Rust (Linux)
+ uses: dtolnay/rust-toolchain@stable
+ with:
+ toolchain: stable
+
+ - name: Setup Go (Linux)
+ uses: actions/setup-go@v5
+ with:
+ go-version: "1.21"
+
+ - name: Setup Java (Linux)
+ uses: actions/setup-java@v4
+ with:
+ distribution: "temurin"
+ java-version: "17"
+
+ - name: Test Node.js FFI binding (Linux)
+ id: binding_node_ffi
+ continue-on-error: true
+ run: |
+ cd unittest/include/nodejs
+ npm install
+ bash test.sh
+
+ - name: Test Node.js N-API binding (Linux)
+ id: binding_node_napi
+ continue-on-error: true
+ run: |
+ PATH=$(echo "$PATH" | tr ':' '\n' | grep -v 'deps/3rd' | tr '\n' ':' | sed 's/:$//')
+ cd unittest/include/nodejs_napi
+ npm install
+ bash test.sh
+
+ - name: Test Python binding (Linux)
+ id: binding_python
+ continue-on-error: true
+ run: |
+ cd unittest/include/python
+ bash test.sh
+
+ - name: Test Rust binding (Linux)
+ id: binding_rust
+ continue-on-error: true
+ run: |
+ cd unittest/include/rust
+ bash test.sh
+
+ - name: Test Go binding (Linux)
+ id: binding_go
+ continue-on-error: true
+ run: |
+ PATH=$(echo "$PATH" | tr ':' '\n' | grep -v 'deps/3rd' | tr '\n' ':' | sed 's/:$//')
+ cd unittest/include/go
+ bash test.sh
+
+ - name: Test Java binding (Linux)
+ id: binding_java
+ continue-on-error: true
+ run: |
+ PATH=$(echo "$PATH" | tr ':' '\n' | grep -v 'deps/3rd' | tr '\n' ':' | sed 's/:$//')
+ cd unittest/include/java
+ bash test.sh
+
+ - name: Pack libseekdb (Linux)
+ run: |
+ chmod +x package/libseekdb/libseekdb-build.sh
+ cd package/libseekdb && bash libseekdb-build.sh "${GITHUB_WORKSPACE}/build_release/src/include"
+
+ - name: Upload artifact
+ uses: actions/upload-artifact@v4
+ with:
+ name: ${{ matrix.artifact_name }}
+ path: package/libseekdb/libseekdb-*.zip
+
+ - name: Save Cache deps (Linux)
+ if: always()
+ uses: actions/cache/save@v4
+ with:
+ path: deps/3rd
+ key: ${{ runner.os }}-libseekdb-deps-${{ matrix.platform }}-el7-${{ hashFiles(format('deps/init/{0}', matrix.deps_file)) }}
+ - name: Save Cache ccache (Linux)
+ if: always()
+ uses: actions/cache/save@v4
+ with:
+ path: .ccache
+ key: ${{ runner.os }}-ccache-libseekdb-${{ matrix.platform }}
+
+ - name: Binding tests outcome (Linux)
+ if: always()
+ run: |
+ failed=0
+ for o in \
+ "${{ steps.binding_node_ffi.outcome }}" \
+ "${{ steps.binding_node_napi.outcome }}" \
+ "${{ steps.binding_python.outcome }}" \
+ "${{ steps.binding_rust.outcome }}" \
+ "${{ steps.binding_go.outcome }}" \
+ "${{ steps.binding_java.outcome }}"; do
+ [ "$o" = "failure" ] && failed=1
+ done
+ if [ "$failed" -ne 0 ]; then
+ echo "::error::One or more libseekdb binding tests failed on Linux"
+ exit 1
+ fi
+ echo "All binding test steps succeeded (or were skipped)."
+
+ # ---------- Build on macOS (no container) ----------
+ build-macos:
+ name: Build libseekdb (${{ matrix.platform }})
+ runs-on: ${{ matrix.runner }}
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - platform: darwin-arm64
+ runner: macos-14
+ artifact_name: libseekdb-darwin-arm64
+ arch: arm64
+ cmake_arch: arm64
+
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ github.event_name == 'workflow_dispatch' && inputs.ref != '' && inputs.ref || (github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha) }}
+
+ - name: Install macOS dependencies
+ run: brew install cmake dylibbundler googletest ccache pybind11 utf8proc thrift re2 brotli bzip2 || true
+
+ - name: Cache deps (macOS)
+ uses: actions/cache@v4
+ with:
+ path: deps/3rd
+ key: ${{ runner.os }}-libseekdb-deps-${{ matrix.platform }}-${{ hashFiles('deps/init/oceanbase.macos.arm64.deps') }}
+ restore-keys: |
+ ${{ runner.os }}-libseekdb-deps-${{ matrix.platform }}-
+
+ - name: Cache ccache (macOS)
+ uses: actions/cache@v4
+ with:
+ path: .ccache
+ # Static key per platform so second run always hits
+ key: ${{ runner.os }}-ccache-libseekdb-${{ matrix.platform }}
+ restore-keys: |
+ ${{ runner.os }}-ccache-libseekdb-${{ matrix.platform }}
+
+ - name: Build init (macOS)
+ run: bash build.sh init
+
+ - name: Build libseekdb (macOS)
+ env:
+ BUILD_TYPE: release
+ ARCH: ${{ matrix.arch }}
+ CMAKE_OSX_ARCHITECTURES: ${{ matrix.cmake_arch }}
+ CCACHE_DIR: ${{ github.workspace }}/.ccache
+ CCACHE_COMPILERCHECK: content
+ CCACHE_NOHASHDIR: 1
+ run: |
+ # Env.cmake looks for ccache in deps/3rd/.../devtools/bin; put it there so no Env.cmake change is needed
+ mkdir -p deps/3rd/usr/local/oceanbase/devtools/bin
+ ln -sf "$(which ccache)" deps/3rd/usr/local/oceanbase/devtools/bin/ccache
+ # Use runner's Python (macOS has 3.x, not 3.8 by default) so embed CMake finds it
+ PYVER=$(python3 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')")
+ bash build.sh release --init -DOB_USE_CCACHE=ON -DBUILD_EMBED_MODE=ON -DPYTHON_VERSION=$PYVER -DCMAKE_OSX_ARCHITECTURES=${{ matrix.cmake_arch }} -DCMAKE_OSX_DEPLOYMENT_TARGET=11.0 --make libseekdb
+ ccache -s
+
+ - name: Setup Node.js (macOS)
+ uses: actions/setup-node@v4
+ with:
+ node-version: "18"
+
+ - name: Setup Rust (macOS)
+ uses: dtolnay/rust-toolchain@stable
+ with:
+ toolchain: stable
+
+ - name: Setup Go (macOS)
+ uses: actions/setup-go@v5
+ with:
+ go-version: "1.21"
+
+ - name: Setup Java (macOS)
+ uses: actions/setup-java@v4
+ with:
+ distribution: "temurin"
+ java-version: "17"
+
+ - name: Test Node.js FFI binding (macOS)
+ id: binding_node_ffi
+ continue-on-error: true
+ run: |
+ cd unittest/include/nodejs
+ npm install
+ bash test.sh
+
+ - name: Test Node.js N-API binding (macOS)
+ id: binding_node_napi
+ continue-on-error: true
+ run: |
+ PATH=$(echo "$PATH" | tr ':' '\n' | grep -v 'deps/3rd' | tr '\n' ':' | sed 's/:$//')
+ cd unittest/include/nodejs_napi
+ npm install
+ bash test.sh
+
+ - name: Test Python binding (macOS)
+ id: binding_python
+ continue-on-error: true
+ run: |
+ cd unittest/include/python
+ bash test.sh
+
+ - name: Test Rust binding (macOS)
+ id: binding_rust
+ continue-on-error: true
+ run: |
+ cd unittest/include/rust
+ bash test.sh
+
+ - name: Test Go binding (macOS)
+ id: binding_go
+ continue-on-error: true
+ run: |
+ PATH=$(echo "$PATH" | tr ':' '\n' | grep -v 'deps/3rd' | tr '\n' ':' | sed 's/:$//')
+ cd unittest/include/go
+ bash test.sh
+
+ - name: Test Java binding (macOS)
+ id: binding_java
+ continue-on-error: true
+ run: |
+ PATH=$(echo "$PATH" | tr ':' '\n' | grep -v 'deps/3rd' | tr '\n' ':' | sed 's/:$//')
+ cd unittest/include/java
+ bash test.sh
+
+ # macOS code signing. Entitlements allow loading embedded dylib.
+ - name: Create entitlements (macOS)
+ run: |
+ echo -e '\n\n\n\n com.apple.security.cs.disable-library-validation\n \n\n' > package/libseekdb/entitlements.plist
+
+ # Only import certificate on main repo; script skips when cert empty (ad-hoc signing used).
+ - name: Import certificate (macOS)
+ if: github.repository == 'oceanbase/seekdb'
+ env:
+ BUILD_CERTIFICATE_BASE64: ${{ secrets.OSX_CODESIGN_BUILD_CERTIFICATE_BASE64 }}
+ P12_PASSWORD: ${{ secrets.OSX_CODESIGN_P12_PASSWORD }}
+ KEYCHAIN_PASSWORD: ${{ secrets.OSX_CODESIGN_KEYCHAIN_PASSWORD }}
+ run: . package/libseekdb/osx_import_codesign_certificate.sh
+
+ - name: Pack libseekdb (macOS)
+ env:
+ ARCH: ${{ matrix.arch }}
+ CODESIGN_IDENTITY: ${{ secrets.OSX_CODESIGN_IDENTITY }}
+ CODESIGN_ENTITLEMENTS: ${{ github.workspace }}/package/libseekdb/entitlements.plist
+ run: cd package/libseekdb && bash libseekdb-build.sh
+
+ - name: Upload artifact
+ uses: actions/upload-artifact@v4
+ with:
+ name: ${{ matrix.artifact_name }}
+ path: package/libseekdb/libseekdb-*.zip
+
+ # Save caches even on failure so next run can resume (deps/ccache)
+ - name: Save Cache deps (macOS)
+ if: always()
+ uses: actions/cache/save@v4
+ with:
+ path: deps/3rd
+ key: ${{ runner.os }}-libseekdb-deps-${{ matrix.platform }}-${{ hashFiles('deps/init/oceanbase.macos.arm64.deps') }}
+ - name: Save Cache ccache (macOS)
+ if: always()
+ uses: actions/cache/save@v4
+ with:
+ path: .ccache
+ key: ${{ runner.os }}-ccache-libseekdb-${{ matrix.platform }}
+
+ - name: Binding tests outcome (macOS)
+ if: always()
+ run: |
+ failed=0
+ for o in \
+ "${{ steps.binding_node_ffi.outcome }}" \
+ "${{ steps.binding_node_napi.outcome }}" \
+ "${{ steps.binding_python.outcome }}" \
+ "${{ steps.binding_rust.outcome }}" \
+ "${{ steps.binding_go.outcome }}" \
+ "${{ steps.binding_java.outcome }}"; do
+ [ "$o" = "failure" ] && failed=1
+ done
+ if [ "$failed" -ne 0 ]; then
+ echo "::error::One or more libseekdb binding tests failed on macOS"
+ exit 1
+ fi
+ echo "All binding test steps succeeded (or were skipped)."
+
+ # ---------- Android NDK cross-compile (macOS arm64 host; target arm64-v8a) ----------
+ # ubuntu-* runners are x86_64 by default. macOS-14 uses Apple silicon + NDK darwin-arm64 prebuilts (see docs).
+ # Alternative: runs-on: ubuntu-24.04-arm for Linux ARM64.
+ #
+ # Why no FFI binding tests here: libseekdb.so is Android arm64-v8a / ELF for Bionic — the dynamic linker,
+ # libc, and JNI/NDK ABI differ from macOS dyld. CI cannot dlopen/load that .so on the host like Linux/macOS/Windows jobs.
+ # Running the same unittest/include/*/test.sh would require an Android emulator, rooted device, or adb push + adb shell test.
+ build-android:
+ name: Build libseekdb (android-arm64-v8a)
+ runs-on: macos-14
+ timeout-minutes: 180
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ github.event_name == 'workflow_dispatch' && inputs.ref != '' && inputs.ref || (github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha) }}
+
+ - name: Install macOS packages
+ run: |
+ brew install cmake ccache wget || true
+
+ - name: Install Android NDK
+ uses: nttld/setup-ndk@v1
+ with:
+ # Use rYYx form (e.g. r27d); full build numbers (27.3.13750724) 404 on the action download URL
+ ndk-version: r27d
+
+ - name: Cache deps (Android)
+ uses: actions/cache@v4
+ with:
+ path: deps/3rd
+ key: ${{ runner.os }}-libseekdb-deps-android-arm64-v8a-${{ hashFiles('deps/init/oceanbase.android.arm64.deps') }}
+ restore-keys: |
+ ${{ runner.os }}-libseekdb-deps-android-arm64-v8a-
+
+ - name: Cache ccache (Android)
+ uses: actions/cache@v4
+ with:
+ path: .ccache
+ key: ${{ runner.os }}-ccache-libseekdb-android-arm64-v8a
+ restore-keys: |
+ ${{ runner.os }}-ccache-libseekdb-android-arm64-v8a-
+
+ - name: Build libseekdb (Android)
+ env:
+ BUILD_TYPE: release
+ CCACHE_DIR: ${{ github.workspace }}/.ccache
+ CCACHE_COMPILERCHECK: content
+ CCACHE_NOHASHDIR: 1
+ run: |
+ set -e
+ # Do not symlink ccache into deps/3rd before --init: dep_create.sh rm -rf deps/3rd.
+ # CMake finds ccache on PATH when devtools has none (Android deps omit obdevtools-ccache).
+ # ANDROID_NDK_HOME is set by the NDK install step; build.sh fixes ANDROID_ABI=arm64-v8a.
+ bash build.sh release --android -DOB_USE_CCACHE=ON -DBUILD_EMBED_MODE=ON --init --make libseekdb
+ ccache -s
+
+ # Same toolchain install order as other jobs (no host FFI below).
+ - name: Setup Node.js (Android)
+ uses: actions/setup-node@v4
+ with:
+ node-version: "18"
+
+ - name: Setup Rust (Android)
+ uses: dtolnay/rust-toolchain@stable
+ with:
+ toolchain: stable
+
+ - name: Setup Go (Android)
+ uses: actions/setup-go@v5
+ with:
+ go-version: "1.21"
+
+ - name: Setup Java (Android)
+ uses: actions/setup-java@v4
+ with:
+ distribution: "temurin"
+ java-version: "17"
+
+ - name: Binding tests (Android)
+ id: android_binding_notice
+ run: |
+ echo "::notice::Skipping native FFI binding tests on this runner: libseekdb.so targets Android arm64-v8a / Bionic and cannot be dlopen()'d on the macOS host."
+ echo "Zip verification runs after pack; device or emulator CI would be required for full FFI tests."
+
+ - name: Pack libseekdb (Android)
+ run: cd package/libseekdb && bash libseekdb-build.sh --android
+
+ - name: Verify packaged zip (Android)
+ id: android_verify_zip
+ continue-on-error: true
+ run: unzip -t package/libseekdb/libseekdb-android-arm64-v8a.zip
+
+ - name: Upload artifact
+ uses: actions/upload-artifact@v4
+ with:
+ name: libseekdb-android-arm64-v8a
+ path: package/libseekdb/libseekdb-android-arm64-v8a.zip
+
+ - name: Save Cache deps (Android)
+ if: always()
+ uses: actions/cache/save@v4
+ with:
+ path: deps/3rd
+ key: ${{ runner.os }}-libseekdb-deps-android-arm64-v8a-${{ hashFiles('deps/init/oceanbase.android.arm64.deps') }}
+
+ - name: Save Cache ccache (Android)
+ if: always()
+ uses: actions/cache/save@v4
+ with:
+ path: .ccache
+ key: ${{ runner.os }}-ccache-libseekdb-android-arm64-v8a
+
+ - name: Pack / zip verification outcome (Android)
+ if: always()
+ run: |
+ if [ "${{ steps.android_verify_zip.outcome }}" = "failure" ]; then
+ echo "::error::Packaged zip verification failed (unzip -t)"
+ exit 1
+ fi
+ echo "Android pack and zip verification OK (or verify step skipped)."
+
+ # ---------- Build on Windows x64 (native, embed DLL) ----------
+ build-windows:
+ name: Build libseekdb (windows-x64)
+ runs-on: windows-2022
+ timeout-minutes: 360
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ github.event_name == 'workflow_dispatch' && inputs.ref != '' && inputs.ref || (github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha) }}
+
+ - name: Setup Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.12"
+
+ - name: Cache deps (Windows)
+ uses: actions/cache@v4
+ with:
+ path: deps/3rd
+ key: ${{ runner.os }}-libseekdb-deps-windows-x64-${{ hashFiles('deps/init/oceanbase.windows.x86_64.deps') }}
+ restore-keys: |
+ ${{ runner.os }}-libseekdb-deps-windows-x64-
+
+ - name: Cache ccache (Windows)
+ uses: actions/cache@v4
+ with:
+ path: .ccache
+ key: ${{ runner.os }}-ccache-libseekdb-windows-x64
+ restore-keys: |
+ ${{ runner.os }}-ccache-libseekdb-windows-x64-
+
+ - name: Install ccache (Windows)
+ shell: pwsh
+ run: |
+ Set-StrictMode -Version Latest
+ $ErrorActionPreference = "Stop"
+ function Refresh-Path {
+ $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
+ }
+ Refresh-Path
+ if (Get-Command ccache -ErrorAction SilentlyContinue) {
+ Write-Host "ccache already on PATH"
+ Get-Command ccache | Out-Host
+ exit 0
+ }
+ choco install ccache -y --no-progress 2>&1 | Out-Host
+ Refresh-Path
+ if ($LASTEXITCODE -ne 0 -or -not (Get-Command ccache -ErrorAction SilentlyContinue)) {
+ Write-Host "::notice::choco install ccache failed or ccache not on PATH; installing from ccache GitHub releases."
+ $ver = "4.13.2"
+ $zipName = "ccache-$ver-windows-x86_64.zip"
+ $url = "https://github.com/ccache/ccache/releases/download/v$ver/$zipName"
+ $tools = Join-Path $env:GITHUB_WORKSPACE ".tools"
+ $dest = Join-Path $tools "ccache-$ver-win64"
+ New-Item -ItemType Directory -Force -Path $dest | Out-Null
+ $zipPath = Join-Path $dest $zipName
+ Invoke-WebRequest -Uri $url -OutFile $zipPath -UseBasicParsing
+ Expand-Archive -Path $zipPath -DestinationPath $dest -Force
+ $exe = Get-ChildItem -Path $dest -Filter ccache.exe -Recurse | Select-Object -First 1
+ if (-not $exe) { throw "ccache.exe not found after extracting $zipName under $dest" }
+ $binDir = $exe.Directory.FullName
+ Add-Content -Path $env:GITHUB_PATH -Value $binDir -Encoding utf8
+ $env:PATH = "$binDir;$env:PATH"
+ }
+ Get-Command ccache | Out-Host
+
+ - name: Build libseekdb (Windows)
+ shell: pwsh
+ env:
+ BUILD_TYPE: release
+ CCACHE_DIR: ${{ github.workspace }}/.ccache
+ CCACHE_COMPILERCHECK: content
+ CCACHE_NOHASHDIR: 1
+ run: |
+ Set-StrictMode -Version Latest
+ $ErrorActionPreference = "Stop"
+ # Pick up Chocolatey / GITHUB_PATH / fallback-installed ccache
+ $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
+ $py = python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')"
+ if (-not (Test-Path "deps/3rd/DONE")) {
+ .\build.ps1 init
+ }
+ # Prepend deps LLVM to PATH; pass lld-link so CMake does not pick GNU ld (MinGW) for clang-cl.
+ $ws = if ($env:GITHUB_WORKSPACE) { $env:GITHUB_WORKSPACE } else { (Get-Location).Path }
+ # CMake -G Ninja requires ninja on PATH at configure time (otherwise CMake may fall back or leave a stale non-Ninja cache).
+ foreach ($rel in @("deps/3rd/tools/ninja", "deps/3rd/tools/cmake/bin")) {
+ $tp = Join-Path $ws $rel
+ if (Test-Path -LiteralPath $tp) { $env:PATH = "$tp;$env:PATH" }
+ }
+ $llvmRoot = if ($env:OB_LLVM_DIR) { $env:OB_LLVM_DIR } else { Join-Path $ws "deps/3rd/tools/llvm18" }
+ $llvmBin = Join-Path $llvmRoot "bin"
+ $lldLink = Join-Path $llvmBin "lld-link.exe"
+ if (-not (Test-Path $lldLink)) { throw "lld-link.exe not found: $lldLink" }
+ $lldFwd = $lldLink.Replace("\", "/")
+ $env:PATH = "$llvmBin;$env:PATH"
+ $ccacheOpt = "-DOB_USE_CCACHE=OFF"
+ if (Get-Command ccache -ErrorAction SilentlyContinue) {
+ $ccacheOpt = "-DOB_USE_CCACHE=ON"
+ } else {
+ Write-Host "::warning::ccache not found; building without compiler cache."
+ }
+ # Map RelWithDebInfo/Debug/MinSizeRel -> Release for IMPORTED targets (e.g. Python3::Module .lib/.dll on Windows).
+ # Ensures cache has these on first cmake run; complements cmake/Env.cmake for CI runners where Python stubs omit RelWithDebInfo.
+ .\build.ps1 release --ninja --target libseekdb "-DBUILD_EMBED_MODE=ON" "-DPYTHON_VERSION=$py" "-DCMAKE_LINKER=$lldFwd" "-DCMAKE_MAP_IMPORTED_CONFIG_RELWITHDEBINFO=Release" "-DCMAKE_MAP_IMPORTED_CONFIG_DEBUG=Release" "-DCMAKE_MAP_IMPORTED_CONFIG_MINSIZEREL=Release" $ccacheOpt
+ if (Get-Command ccache -ErrorAction SilentlyContinue) { ccache -s }
+
+ - name: Debug facts — libseekdb Windows (deterministic)
+ shell: pwsh
+ env:
+ BUILD_TYPE: release
+ run: |
+ $ErrorActionPreference = "Continue"
+ $ws = if ($env:GITHUB_WORKSPACE) { $env:GITHUB_WORKSPACE } else { (Get-Location).Path }
+ foreach ($rel in @("deps/3rd/tools/ninja", "deps/3rd/tools/cmake/bin")) {
+ $tp = Join-Path $ws $rel
+ if (Test-Path -LiteralPath $tp) { $env:PATH = "$tp;$env:PATH" }
+ }
+ . ./unittest/include/seekdb-windows-dll-resolve.ps1
+ . ./unittest/include/debug-libseekdb-windows.ps1
+ Write-LibseekdbWindowsBuildFacts -RepoRoot "$PWD"
+
+ - name: Verify libseekdb DLL (Windows)
+ shell: pwsh
+ env:
+ BUILD_TYPE: release
+ run: |
+ Set-StrictMode -Version Latest
+ $ErrorActionPreference = "Stop"
+ . ./unittest/include/seekdb-windows-dll-resolve.ps1
+ $bdn = Get-SeekDbWindowsBuildDirNameFromEnv
+ $r = Find-SeekDbWindowsDll -RepoRoot "$PWD" -BuildDirName $bdn
+ if (-not $r) {
+ Write-SeekDbWindowsDllDiagnostics -RepoRoot "$PWD" -BuildDirName $bdn
+ throw "libseekdb build did not produce seekdb.dll under build_$bdn (see diagnostics above)."
+ }
+ Write-Host "libseekdb DLL: $($r.DllPath)"
+ "SEEKDB_LIB_PATH=$($r.DllPath)" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+
+ - name: Setup Node.js (Windows)
+ uses: actions/setup-node@v4
+ with:
+ node-version: "18"
+
+ - name: Setup Rust (Windows)
+ uses: dtolnay/rust-toolchain@stable
+ with:
+ toolchain: stable
+
+ - name: Setup Go (Windows)
+ uses: actions/setup-go@v5
+ with:
+ go-version: "1.21"
+
+ - name: Setup Java (Windows)
+ uses: actions/setup-java@v4
+ with:
+ distribution: "temurin"
+ java-version: "17"
+
+ - name: Install MinGW and Maven (Go CGO + Java)
+ shell: pwsh
+ run: choco install mingw maven -y --no-progress
+
+ # One job step per language: if something hangs, the in-progress step name in the Actions UI shows
+ # whether the stall is in Python, npm, node-gyp, cargo, go, or Java (not “whole binding” vs global timeout).
+ - name: Binding tests — Python (Windows)
+ id: bind_win_python
+ continue-on-error: true
+ shell: pwsh
+ env:
+ SEEKDB_BINDING_SECTION: Python
+ run: |
+ Set-StrictMode -Version Latest
+ $ErrorActionPreference = "Stop"
+ $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
+ Get-Command gcc | Out-Host
+ Get-Command mvn | Out-Host
+ .\unittest\include\run-libseekdb-binding-tests.ps1 -RepoRoot "$PWD" -ContinueOnError
+
+ - name: Binding tests — Node FFI / koffi (Windows)
+ id: bind_win_node_ffi
+ continue-on-error: true
+ shell: pwsh
+ env:
+ SEEKDB_BINDING_SECTION: NodeFfi
+ run: |
+ Set-StrictMode -Version Latest
+ $ErrorActionPreference = "Stop"
+ $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
+ .\unittest\include\run-libseekdb-binding-tests.ps1 -RepoRoot "$PWD" -ContinueOnError
+
+ - name: Binding tests — Node N-API (Windows)
+ id: bind_win_node_napi
+ continue-on-error: true
+ shell: pwsh
+ env:
+ SEEKDB_BINDING_SECTION: NodeNapi
+ # VECTOR + DBMS_HYBRID_SEARCH cases can stall native code on Windows runners; core N-API still covered.
+ SEEKDB_NODE_NAPI_SKIP_HEAVY: "1"
+ run: |
+ Set-StrictMode -Version Latest
+ $ErrorActionPreference = "Stop"
+ $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
+ .\unittest\include\run-libseekdb-binding-tests.ps1 -RepoRoot "$PWD" -ContinueOnError
+
+ - name: Binding tests — Rust (Windows)
+ id: bind_win_rust
+ continue-on-error: true
+ shell: pwsh
+ env:
+ SEEKDB_BINDING_SECTION: Rust
+ run: |
+ Set-StrictMode -Version Latest
+ $ErrorActionPreference = "Stop"
+ $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
+ .\unittest\include\run-libseekdb-binding-tests.ps1 -RepoRoot "$PWD" -ContinueOnError
+
+ - name: Binding tests — Go (Windows)
+ id: bind_win_go
+ continue-on-error: true
+ shell: pwsh
+ env:
+ SEEKDB_BINDING_SECTION: Go
+ run: |
+ Set-StrictMode -Version Latest
+ $ErrorActionPreference = "Stop"
+ $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
+ .\unittest\include\run-libseekdb-binding-tests.ps1 -RepoRoot "$PWD" -ContinueOnError
+
+ - name: Binding tests — Java (Windows)
+ id: bind_win_java
+ continue-on-error: true
+ shell: pwsh
+ env:
+ SEEKDB_BINDING_SECTION: Java
+ run: |
+ Set-StrictMode -Version Latest
+ $ErrorActionPreference = "Stop"
+ $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
+ .\unittest\include\run-libseekdb-binding-tests.ps1 -RepoRoot "$PWD" -ContinueOnError
+
+ - name: Pack libseekdb zip (Windows)
+ shell: pwsh
+ env:
+ BUILD_TYPE: release
+ run: .\package\libseekdb\libseekdb-build.ps1
+
+ - name: Upload artifact
+ uses: actions/upload-artifact@v4
+ with:
+ name: libseekdb-windows-x64
+ path: package/libseekdb/libseekdb-windows-x64.zip
+
+ - name: Save Cache deps (Windows)
+ if: always()
+ uses: actions/cache/save@v4
+ with:
+ path: deps/3rd
+ key: ${{ runner.os }}-libseekdb-deps-windows-x64-${{ hashFiles('deps/init/oceanbase.windows.x86_64.deps') }}
+
+ - name: Save Cache ccache (Windows)
+ if: always()
+ uses: actions/cache/save@v4
+ with:
+ path: .ccache
+ key: ${{ runner.os }}-ccache-libseekdb-windows-x64
+
+ - name: Binding tests outcome (Windows)
+ if: always()
+ shell: pwsh
+ run: |
+ $failed = $false
+ if ("${{ steps.bind_win_python.outcome }}" -eq "failure") { Write-Host "::error::Binding section failed: Python"; $failed = $true }
+ if ("${{ steps.bind_win_node_ffi.outcome }}" -eq "failure") { Write-Host "::error::Binding section failed: Node FFI (koffi)"; $failed = $true }
+ if ("${{ steps.bind_win_node_napi.outcome }}" -eq "failure") { Write-Host "::error::Binding section failed: Node N-API"; $failed = $true }
+ if ("${{ steps.bind_win_rust.outcome }}" -eq "failure") { Write-Host "::error::Binding section failed: Rust"; $failed = $true }
+ if ("${{ steps.bind_win_go.outcome }}" -eq "failure") { Write-Host "::error::Binding section failed: Go"; $failed = $true }
+ if ("${{ steps.bind_win_java.outcome }}" -eq "failure") { Write-Host "::error::Binding section failed: Java"; $failed = $true }
+ if ($failed) { exit 1 }
+ Write-Host "All Windows binding test sections succeeded."
+
+ # ---------- Collect libseekdb artifacts and upload to S3 (runs only when all needed build jobs succeed, including binding tests) ----------
+ release-artifacts:
+ name: Collect artifacts and upload to S3
+ runs-on: ubuntu-22.04
+ needs:
+ - build
+ - build-macos
+ - build-android
+ - build-windows
+ steps:
+ - name: Download all artifacts
+ uses: actions/download-artifact@v4
+ with:
+ path: release-artifacts
+ merge-multiple: true
+
+ - name: List all artifacts
+ run: |
+ echo "=== All artifacts ==="
+ ls -la release-artifacts/
+
+ - name: Upload combined artifact (for workflow download)
+ uses: actions/upload-artifact@v4
+ with:
+ name: libseekdb-all-platforms
+ path: release-artifacts/
+
+ - name: Configure AWS credentials
+ if: env.UPLOAD_S3 == 'true'
+ uses: aws-actions/configure-aws-credentials@v4
+ with:
+ aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ aws-region: ${{ env.AWS_REGION }}
+
+ - name: Upload to S3
+ if: env.UPLOAD_S3 == 'true'
+ run: |
+ set -e
+ if [ -n "${{ env.DESTINATION_TARGET_PATH }}" ]; then
+ S3_TARGET="${{ env.DESTINATION_TARGET_PATH }}"
+ else
+ S3_TARGET="s3://${{ env.S3_BUCKET }}/${{ env.S3_PREFIX }}/"
+ fi
+ [ "${S3_TARGET: -1}" != "/" ] && S3_TARGET="${S3_TARGET}/"
+ echo "Uploading to $S3_TARGET"
+ aws s3 cp release-artifacts/ "$S3_TARGET" --recursive --exclude "*" --include "*.zip" --no-progress
+ echo "Uploaded:"
+ aws s3 ls "$S3_TARGET" --recursive
+ echo "Done."
+ continue-on-error: true
diff --git a/.gitignore b/.gitignore
index 50b0751c9..11d8bbd34 100644
--- a/.gitignore
+++ b/.gitignore
@@ -381,6 +381,9 @@ tools/ob-configserver/bin/*
tools/ob-configserver/tests/*.log
tools/ob-configserver/tests/*.out
+############# package/libseekdb #############
+package/libseekdb/libseekdb-*.zip
+
## .NET Configurator build output (rebuilt by dotnet publish)
tools/windows/seekdbConfigurator/bin/
tools/windows/seekdbConfigurator/obj/
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9aa447dad..6454b9f5a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -9,6 +9,13 @@ project("OceanBase"
HOMEPAGE_URL "https://www.oceanbase.ai"
LANGUAGES CXX C ASM)
+# Android NDK toolchain / defaults may pin C++17 after project(); enforce C++20 for language features (e.g. consteval).
+if(ANDROID)
+ set(CMAKE_CXX_STANDARD 20)
+ set(CMAKE_CXX_STANDARD_REQUIRED ON)
+ set(CMAKE_CXX_EXTENSIONS ON)
+endif()
+
if(WIN32)
find_program(LLVM_LIB_PROGRAM llvm-lib)
if(LLVM_LIB_PROGRAM)
@@ -130,6 +137,7 @@ endif()
add_subdirectory(deps/oblib)
add_subdirectory(src/objit)
+include(${CMAKE_SOURCE_DIR}/cmake/EnsureWindowsDiaGuids.cmake)
add_subdirectory(src)
add_subdirectory(sqlite-benchmark)
diff --git a/build.ps1 b/build.ps1
index ec43864d9..7995b6ea7 100644
--- a/build.ps1
+++ b/build.ps1
@@ -18,7 +18,9 @@ $Action = "debug"
$Ninja = $false
$Init = $false
$Jobs = 0
-$h = $false
+$h = $false
+$NinjaTarget = "observer"
+$ExtraCmake = @()
$i = 0
while ($i -lt $args.Count) {
@@ -27,12 +29,17 @@ while ($i -lt $args.Count) {
{ $_ -in "-h", "--help", "-help" } { $h = $true }
{ $_ -in "--ninja", "-ninja" } { $Ninja = $true }
{ $_ -in "--init", "-init" } { $Init = $true }
+ { $_ -in "--target" } {
+ $i++
+ if ($i -lt $args.Count) { $NinjaTarget = "$($args[$i])" }
+ }
{ $_ -in "-j", "--jobs" } {
$i++
if ($i -lt $args.Count) { $Jobs = [int]$args[$i] }
}
default {
if (-not $a.StartsWith("-")) { $Action = $a }
+ elseif ($a.StartsWith("-D")) { $ExtraCmake += $a }
else { Write-Host "[build.ps1][WARN] Unknown flag: $a" -ForegroundColor Yellow }
}
}
@@ -181,6 +188,7 @@ Usage:
.\build.ps1 [BuildType] --ninja Configure + compile (ninja)
.\build.ps1 [BuildType] --ninja -j 16 Compile with 16 jobs
.\build.ps1 [BuildType] --ninja --init Init deps, then build
+ .\build.ps1 release --ninja --target libseekdb Build embed DLL (example)
.\build.ps1 package Build release + MSI/ZIP installer
BuildType:
@@ -191,6 +199,8 @@ BuildType:
Flags:
--ninja Configure + compile with Ninja
--init Run dependency init before building (like build.sh --init)
+ --target NAME Ninja target (default: observer); e.g. libseekdb
+ -DVAR=VALUE Extra CMake cache entries (repeatable); e.g. -DBUILD_EMBED_MODE=ON
Environment variables (override dependency paths):
OB_VCPKG_DIR vcpkg install root (default: deps/3rd or C:/VcpkgInstalled)
@@ -286,7 +296,13 @@ function Do-Build {
[string[]]$ExtraCMakeArgs = @()
)
- $buildDir = "$TOPDIR\build_$($BuildType.ToLower())"
+ # Align directory names with build.sh: release -> build_release, debug -> build_debug
+ $folderName = switch ($BuildType) {
+ "RelWithDebInfo" { "release" }
+ "Debug" { "debug" }
+ default { $BuildType.ToLower() }
+ }
+ $buildDir = "$TOPDIR\build_$folderName"
if (-not (Test-Path $buildDir)) {
New-Item -ItemType Directory -Path $buildDir | Out-Null
}
@@ -300,9 +316,9 @@ function Do-Build {
"-DOB_VCPKG_DIR=$DefaultVcpkgDir",
"-DOB_OPENSSL_DIR=$DefaultOpenSSLDir",
"-DOB_LLVM_DIR=$DefaultLLVMDir"
- ) + $ExtraCMakeArgs
+ ) + $ExtraCMakeArgs + $ExtraCmake
- Write-Log "CMake configure: build_$($BuildType.ToLower())"
+ Write-Log "CMake configure: build_$folderName"
Write-Log " Build type : $BuildType"
Write-Log " VcpkgDir : $DefaultVcpkgDir"
Write-Log " OpenSSLDir : $DefaultOpenSSLDir"
@@ -311,13 +327,54 @@ function Do-Build {
Push-Location $buildDir
try {
- & cmake @cmakeArgs | Out-Host
- if ($LASTEXITCODE -ne 0) {
- Write-Err "CMake configure failed (exit code $LASTEXITCODE)"
- exit $LASTEXITCODE
+ function Invoke-CMakeConfigure {
+ & cmake @cmakeArgs | Out-Host
+ return $LASTEXITCODE
+ }
+
+ $exit = Invoke-CMakeConfigure
+ if ($exit -ne 0) {
+ Write-Err "CMake configure failed (exit code $exit)"
+ exit $exit
}
Write-Log "CMake configure succeeded."
+ # Facts from CI: CMakeCache can say CMAKE_GENERATOR=Ninja while no *.ninja exists (incomplete/stale tree).
+ # Recover once by wiping cache + CMakeFiles and re-running cmake.
+ function Test-HasNinjaBuildFiles {
+ param([string]$Dir)
+ if (Test-Path (Join-Path $Dir "build.ninja")) { return $true }
+ try {
+ $nf = [System.IO.Directory]::GetFiles($Dir, "*.ninja", [System.IO.SearchOption]::TopDirectoryOnly)
+ return ($nf.Length -gt 0)
+ } catch {
+ return $false
+ }
+ }
+
+ if (-not (Test-HasNinjaBuildFiles -Dir $buildDir)) {
+ Write-Log "[build.ps1] No Ninja build files after configure; clearing CMakeCache + CMakeFiles and re-configuring once."
+ Remove-Item (Join-Path $buildDir "CMakeCache.txt") -Force -ErrorAction SilentlyContinue
+ Remove-Item (Join-Path $buildDir "CMakeFiles") -Recurse -Force -ErrorAction SilentlyContinue
+ $exit2 = Invoke-CMakeConfigure
+ if ($exit2 -ne 0) {
+ Write-Err "CMake re-configure failed (exit code $exit2)"
+ exit $exit2
+ }
+ Write-Log "CMake re-configure finished."
+ }
+
+ # Fail fast if Ninja was requested but the build tree still has no Ninja backend files.
+ if (-not (Test-HasNinjaBuildFiles -Dir $buildDir)) {
+ Write-Err "CMake did not create build.ninja (or any *.ninja) under $buildDir after configure/retry (expected -G Ninja)."
+ $cache = Join-Path $buildDir "CMakeCache.txt"
+ if (Test-Path $cache) {
+ Select-String -Path $cache -Pattern "^CMAKE_GENERATOR:" | ForEach-Object { Write-Err $_.Line }
+ }
+ Write-Err "Fix: delete $buildDir completely and re-run, or ensure Ninja is on PATH when cmake runs (e.g. deps\3rd\tools\ninja)."
+ exit 1
+ }
+
# Copy compile_commands.json to project root for IDE support
$ccJson = "$buildDir\compile_commands.json"
if (Test-Path $ccJson) {
@@ -334,17 +391,41 @@ function Do-Build {
# -- ninja build -----------------------------------------------------
function Do-Ninja {
- param([string]$BuildDir)
+ param(
+ [string]$BuildDir,
+ [string]$Target = "observer"
+ )
- Write-Log "Building with Ninja (-j $Jobs) in $BuildDir ..."
+ Write-Log "Building with Ninja (-j $Jobs) target=$Target in $BuildDir ..."
Push-Location $BuildDir
try {
- & ninja -j $Jobs observer | Out-Host
+ & ninja -j $Jobs $Target | Out-Host
if ($LASTEXITCODE -ne 0) {
Write-Err "Build failed (exit code $LASTEXITCODE)"
exit $LASTEXITCODE
}
Write-Log "Build succeeded!"
+
+ # Deterministic post-condition for libseekdb: link must emit a DLL somewhere under the build dir.
+ if ($Target -eq "libseekdb") {
+ $foundDll = $false
+ foreach ($leaf in @("seekdb.dll", "libseekdb.dll")) {
+ try {
+ $arr = [System.IO.Directory]::GetFiles($BuildDir, $leaf, [System.IO.SearchOption]::AllDirectories)
+ if ($arr -and $arr.Length -gt 0) {
+ $foundDll = $true
+ Write-Log "Found ${leaf} at $($arr[0])"
+ break
+ }
+ } catch {
+ # ignore enumeration errors; treat as not found
+ }
+ }
+ if (-not $foundDll) {
+ Write-Err "ninja libseekdb succeeded but no seekdb.dll / libseekdb.dll under $BuildDir — link step did not produce a DLL (check ninja/link output above)."
+ exit 1
+ }
+ }
}
finally {
Pop-Location
@@ -404,7 +485,7 @@ function Do-Package {
}
$buildDir = Do-Build -BuildType "RelWithDebInfo" -ExtraCMakeArgs @("-DOB_BUILD_PACKAGE=ON")
- Do-Ninja -BuildDir $buildDir
+ Do-Ninja -BuildDir $buildDir -Target observer
# Sign binaries before they are packaged into the MSI
$exesToSign = @(
@@ -476,12 +557,12 @@ switch ($Action.ToLower()) {
{ $_ -in "release", "relwithdebinfo" } {
if ($Init) { Do-Init }
$buildDir = Do-Build -BuildType "RelWithDebInfo"
- if ($Ninja) { Do-Ninja -BuildDir $buildDir }
+ if ($Ninja) { Do-Ninja -BuildDir $buildDir -Target $NinjaTarget }
}
{ $_ -in "debug", "" } {
if ($Init) { Do-Init }
$buildDir = Do-Build -BuildType "Debug"
- if ($Ninja) { Do-Ninja -BuildDir $buildDir }
+ if ($Ninja) { Do-Ninja -BuildDir $buildDir -Target $NinjaTarget }
}
"-h" {
Show-Usage
diff --git a/build.sh b/build.sh
index a647f3169..bfe1b8b1c 100755
--- a/build.sh
+++ b/build.sh
@@ -6,13 +6,11 @@ BUILD_SH=$TOPDIR/build.sh
DEP_DIR=${TOPDIR}/deps/3rd/usr/local/oceanbase/deps/devel
TOOLS_DIR=${TOPDIR}/deps/3rd/usr/local/oceanbase/devtools
-# Get CPU cores and CMAKE command, compatible with macOS and Linux
+# Get CPU cores; cmake path is resolved in do_build() (Linux may use host cmake before deps devtools exist)
if [[ "$(uname -s)" == "Darwin" ]]; then
- CMAKE_COMMAND="cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=1"
CPU_CORES=$(sysctl -n hw.ncpu)
KERNEL_RELEASE=""
else
- CMAKE_COMMAND="${TOOLS_DIR}/bin/cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=1"
CPU_CORES=$(grep -c ^processor /proc/cpuinfo)
KERNEL_RELEASE=$(grep -Po 'release [0-9]{1}' /etc/issue 2>/dev/null)
fi
@@ -166,22 +164,25 @@ function do_init
# make build directory && cmake && make (if need)
function do_build
{
- # Check if cmake exists, compatible with macOS and Linux
+ # Resolve cmake: prefer OceanBase devtools on Linux; else Homebrew/common paths; else PATH (e.g. apt cmake on CI)
CMAKE_PATH=""
if [[ "$(uname -s)" == "Darwin" ]]; then
- # macOS: cmake may be at /opt/homebrew/bin/cmake or /usr/local/bin/cmake
if [ -f /opt/homebrew/bin/cmake ]; then
CMAKE_PATH="/opt/homebrew/bin/cmake"
elif [ -f /usr/local/bin/cmake ]; then
CMAKE_PATH="/usr/local/bin/cmake"
fi
else
- # Linux
- CMAKE_PATH="${TOOLS_DIR}/bin/cmake"
+ if [[ -x "${TOOLS_DIR}/bin/cmake" ]]; then
+ CMAKE_PATH="${TOOLS_DIR}/bin/cmake"
+ fi
+ fi
+ if [[ -z "$CMAKE_PATH" ]] && command -v cmake >/dev/null 2>&1; then
+ CMAKE_PATH="$(command -v cmake)"
fi
if [ -z "$CMAKE_PATH" ]; then
- echo_log "[NOTICE] Your workspace has not initialized dependencies, please append '--init' args to initialize dependencies"
+ echo_log "[NOTICE] cmake not found. On Linux install cmake (e.g. apt install cmake) or run --init so ${TOOLS_DIR}/bin/cmake exists"
exit 1
fi
@@ -196,11 +197,11 @@ function do_build
echo_err "Set ANDROID_NDK_HOME or install the NDK"
exit 1
fi
- ANDROID_CMAKE_ARGS="-DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK_HOME/build/cmake/android.toolchain.cmake -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=android-28"
+ ANDROID_CMAKE_ARGS="-DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK_HOME/build/cmake/android.toolchain.cmake -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=android-28 -DCMAKE_CXX_STANDARD=20 -DCMAKE_CXX_EXTENSIONS=ON"
echo_log "Android NDK: $ANDROID_NDK_HOME"
fi
- ${CMAKE_COMMAND} ${TOPDIR} ${ANDROID_CMAKE_ARGS} "$@"
+ "${CMAKE_PATH}" -DCMAKE_EXPORT_COMPILE_COMMANDS=1 ${TOPDIR} ${ANDROID_CMAKE_ARGS} "$@"
if [ $? -ne 0 ]; then
echo_err "Failed to generate Makefile"
exit 1
diff --git a/cmake/EnsureWindowsDiaGuids.cmake b/cmake/EnsureWindowsDiaGuids.cmake
new file mode 100644
index 000000000..91397e936
--- /dev/null
+++ b/cmake/EnsureWindowsDiaGuids.cmake
@@ -0,0 +1,66 @@
+# LLVM imports may reference BuildTools\DIA SDK\...\diaguids.lib; CI/home often only have Community/etc.
+# Copy one existing VS 2022 diaguids.lib there before linking (must run before add_subdirectory(src)).
+
+if(NOT WIN32)
+ return()
+endif()
+
+# LLVM/seekdb Windows CI is x64 -> amd64 DIA libs; ARM64 host uses arm64.
+if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^(ARM64|aarch64)$")
+ set(_a "arm64")
+else()
+ set(_a "amd64")
+endif()
+
+set(_dst "C:/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/DIA SDK/lib/${_a}/diaguids.lib")
+if(EXISTS "${_dst}")
+ return()
+endif()
+
+set(_cand "")
+if(DEFINED ENV{VSINSTALLDIR})
+ file(TO_CMAKE_PATH "$ENV{VSINSTALLDIR}" _r)
+ string(REGEX REPLACE "/+$" "" _r "${_r}")
+ list(APPEND _cand "${_r}/DIA SDK/lib/${_a}/diaguids.lib")
+endif()
+
+set(_vw "C:/Program Files (x86)/Microsoft Visual Studio/Installer/vswhere.exe")
+if(EXISTS "${_vw}")
+ execute_process(
+ COMMAND "${_vw}" -latest -products * -utf8 -property installationPath
+ OUTPUT_VARIABLE _vp OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET RESULT_VARIABLE _vr
+ )
+ if(_vr EQUAL 0 AND _vp)
+ string(STRIP "${_vp}" _vp)
+ list(APPEND _cand "${_vp}/DIA SDK/lib/${_a}/diaguids.lib")
+ endif()
+endif()
+
+foreach(_root "C:/Program Files/Microsoft Visual Studio/2022" "C:/Program Files (x86)/Microsoft Visual Studio/2022")
+ if(EXISTS "${_root}")
+ file(GLOB _g "${_root}/*/DIA SDK/lib/${_a}/diaguids.lib")
+ list(APPEND _cand ${_g})
+ endif()
+endforeach()
+
+set(_src "")
+foreach(_i IN LISTS _cand)
+ if(EXISTS "${_i}")
+ set(_src "${_i}")
+ break()
+ endif()
+endforeach()
+
+if(NOT _src)
+ message(WARNING "EnsureWindowsDiaGuids: diaguids.lib (${_a}) not found. LLVM/lld link may fail.")
+ return()
+endif()
+
+get_filename_component(_dd "${_dst}" DIRECTORY)
+file(MAKE_DIRECTORY "${_dd}")
+execute_process(COMMAND "${CMAKE_COMMAND}" -E copy "${_src}" "${_dst}" RESULT_VARIABLE _ec)
+if(NOT _ec EQUAL 0)
+ message(WARNING "EnsureWindowsDiaGuids: copy failed (${_ec}); try elevated cmake or install DIA SDK.")
+else()
+ message(STATUS "EnsureWindowsDiaGuids: ${_src} -> ${_dst}")
+endif()
diff --git a/cmake/Env.cmake b/cmake/Env.cmake
index acb66e6ea..43516c0d5 100644
--- a/cmake/Env.cmake
+++ b/cmake/Env.cmake
@@ -147,6 +147,17 @@ else()
set(CMAKE_CXX_FLAGS "-std=gnu++20")
endif()
+
+# Before the first project(), WIN32 may be unset, so the elseif(WIN32) block below can skip
+# CMAKE_MAP_IMPORTED_CONFIG_*. That breaks Windows RelWithDebInfo with FindPython3 (Python3::Module
+# has no IMPORTED_IMPLIB for that config). CMAKE_HOST_WIN32 is set when cmake runs on Windows; skip
+# for Android NDK cross-builds (OB_ANDROID) so we do not force host mapping onto the NDK tree.
+if(CMAKE_HOST_WIN32 AND NOT OB_ANDROID)
+ set(CMAKE_MAP_IMPORTED_CONFIG_DEBUG Release CACHE STRING "imported: map Debug -> Release" FORCE)
+ set(CMAKE_MAP_IMPORTED_CONFIG_RELWITHDEBINFO Release CACHE STRING "imported: map RelWithDebInfo -> Release" FORCE)
+ set(CMAKE_MAP_IMPORTED_CONFIG_MINSIZEREL Release CACHE STRING "imported: map MinSizeRel -> Release" FORCE)
+endif()
+
if(OB_DISABLE_PIE)
message(STATUS "build without pie")
set(PIE_OPT "-no-pie")
@@ -213,9 +224,15 @@ if(OB_ANDROID)
# and Env.cmake runs before project() which would set ANDROID.
set(OB_CLANG_BIN "clang")
set(OB_CLANGXX_BIN "clang++")
- # NDK toolchain bin dir (derive from ANDROID_NDK_HOME or CMAKE_TOOLCHAIN_FILE)
+ # NDK toolchain bin dir (derive from ANDROID_NDK_HOME or CMAKE_TOOLCHAIN_FILE).
+ # GLOB prebuilt/*/bin so Linux hosts (e.g. CI) resolve linux-x86_64, macOS resolves darwin-x86_64.
if(DEFINED ENV{ANDROID_NDK_HOME})
- set(_NDK_TOOLCHAIN_BIN "$ENV{ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/darwin-x86_64/bin")
+ file(GLOB _NDK_TOOLCHAIN_BIN "$ENV{ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/*/bin")
+ list(LENGTH _NDK_TOOLCHAIN_BIN _ndk_bin_n)
+ if(_ndk_bin_n LESS 1)
+ message(FATAL_ERROR "ANDROID_NDK_HOME: no toolchains/llvm/prebuilt/*/bin under $ENV{ANDROID_NDK_HOME}")
+ endif()
+ list(GET _NDK_TOOLCHAIN_BIN 0 _NDK_TOOLCHAIN_BIN)
else()
# Derive from toolchain file path: .../build/cmake/android.toolchain.cmake -> .../toolchains/llvm/prebuilt/*/bin
get_filename_component(_NDK_ROOT "${CMAKE_TOOLCHAIN_FILE}" DIRECTORY)
@@ -316,9 +333,14 @@ endif()
ob_define(OB_USE_CCACHE OFF)
if (OB_USE_CCACHE)
+ # Prefer devtools (from deps); dep_create may wipe deps/3rd before a symlink is recreated,
+ # and Android deps do not ship obdevtools-ccache — fall back to ccache on PATH.
find_program(OB_CCACHE ccache PATHS "${DEVTOOLS_DIR}/bin" NO_DEFAULT_PATH)
if (NOT OB_CCACHE)
- message(FATAL_ERROR "cannot find ccache.")
+ find_program(OB_CCACHE ccache)
+ endif()
+ if (NOT OB_CCACHE)
+ message(FATAL_ERROR "cannot find ccache. Install ccache (e.g. apt install ccache) or place it under ${DEVTOOLS_DIR}/bin.")
else()
set(CMAKE_C_COMPILER_LAUNCHER ${OB_CCACHE})
set(CMAKE_CXX_COMPILER_LAUNCHER ${OB_CCACHE})
diff --git a/deps/init/dep_create.ps1 b/deps/init/dep_create.ps1
index 118b82834..96095662e 100644
--- a/deps/init/dep_create.ps1
+++ b/deps/init/dep_create.ps1
@@ -9,7 +9,7 @@
After extraction the layout is:
deps/3rd/vcpkg/x64-windows/ (vcpkg installed packages)
deps/3rd/openssl/ (OpenSSL)
- deps/3rd/vsag/ (vsag vector search library)
+ deps/3rd/vsag/ (vsag vector search library)
deps/3rd/tools/cmake/ (CMake)
deps/3rd/tools/ninja/ (Ninja)
deps/3rd/tools/llvm18/ (LLVM 18)
@@ -123,20 +123,16 @@ foreach ($sect in $sections.Keys) {
Write-Log " cached"
} else {
Write-Log " downloading from $url ..."
+ # Schannel/CI: use --ssl-no-revoke (avoids curl 35 / revocation offline); retries for flaky links.
$tmpPath = "$pkgPath.tmp"
- try {
- & curl.exe -L -f -s --retry 3 --retry-delay 2 -o $tmpPath $url
- if ($LASTEXITCODE -ne 0) {
- throw "curl exit code $LASTEXITCODE"
- }
- Move-Item -Force $tmpPath $pkgPath
- }
- catch {
- if (Test-Path $tmpPath) { Remove-Item -Force $tmpPath }
- Write-Err "Failed to download: $url"
- Write-Err "$_"
+ if (Test-Path $tmpPath) { Remove-Item -Force $tmpPath -ErrorAction SilentlyContinue }
+ & curl.exe -L -f -sS --connect-timeout 120 --ssl-no-revoke --retry 3 --retry-delay 2 -o $tmpPath $url
+ if ($LASTEXITCODE -ne 0) {
+ if (Test-Path $tmpPath) { Remove-Item -Force $tmpPath -ErrorAction SilentlyContinue }
+ Write-Err "Failed to download: $url (curl exit $LASTEXITCODE)"
exit 4
}
+ Move-Item -Force $tmpPath $pkgPath
}
# -- Extract -------------------------------------------------
diff --git a/deps/init/dep_create.sh b/deps/init/dep_create.sh
index bb31682af..7232cf51e 100644
--- a/deps/init/dep_create.sh
+++ b/deps/init/dep_create.sh
@@ -72,6 +72,17 @@ function echo_err() {
echo -e "[dep_create.sh][ERROR] $@" 1>&2
}
+# GNU tar: archives built on macOS may contain PAX extended headers (e.g. LIBARCHIVE.xattr.com.apple.provenance);
+# extracting on Linux otherwise prints harmless "Ignoring unknown extended header keyword" noise.
+function extract_tar_gz_strip1() {
+ local dir="$1" archive="$2"
+ if tar --version 2>/dev/null | head -n1 | grep -q 'GNU tar'; then
+ (cd "$dir" && tar --warning=no-unknown-keyword -xzf "$archive" --strip-components=1)
+ else
+ (cd "$dir" && tar -xzf "$archive" --strip-components=1)
+ fi
+}
+
function get_os_release() {
if [[ "${ANDROID_BUILD}" == "true" ]]; then
OS_RELEASE="android"
@@ -461,7 +472,7 @@ do
fi
echo_log "unpack package <${pkg}>... \c"
if [[ "${IS_TAR_PLATFORM}" == "true" ]]; then
- (cd ${TARGET_DIR_3RD} && tar -xzf "${TARGET_DIR_3RD}/pkg/${pkg}" --strip-components=1)
+ extract_tar_gz_strip1 "${TARGET_DIR_3RD}" "${TARGET_DIR_3RD}/pkg/${pkg}"
elif [[ "$ID" = "arch" || "$ID" = "garuda" ]]; then
(cd ${TARGET_DIR_3RD} && rpmextract.sh "${TARGET_DIR_3RD}/pkg/${pkg}")
else
diff --git a/deps/init/oceanbase.al8.aarch64.deps b/deps/init/oceanbase.al8.aarch64.deps
index 411db28e4..be4ef414f 100644
--- a/deps/init/oceanbase.al8.aarch64.deps
+++ b/deps/init/oceanbase.al8.aarch64.deps
@@ -33,7 +33,6 @@ devdeps-grpc-abiv1-1.46.7-572026031815.al8.aarch64.rpm
devdeps-roaringbitmap-croaring-3.0.0-92024092815.al8.aarch64.rpm
devdeps-apache-arrow-20.0.0-452026020914.al8.aarch64.rpm
devdeps-vsag-abiv1-1.1.0-1752026041310.al8.aarch64.rpm
-devdeps-apache-orc-2.1.1-122026020912.al8.aarch64.rpm
devdeps-fast-float-6.1.3-42024112122.al8.aarch64.rpm
devdeps-sqlite-abiv1-3.38.1-72026031815.al8.aarch64.rpm
diff --git a/deps/init/oceanbase.al8.x86_64.deps b/deps/init/oceanbase.al8.x86_64.deps
index 7e580531e..00db36882 100644
--- a/deps/init/oceanbase.al8.x86_64.deps
+++ b/deps/init/oceanbase.al8.x86_64.deps
@@ -34,7 +34,6 @@ devdeps-grpc-abiv1-1.46.7-572026031815.al8.x86_64.rpm
devdeps-roaringbitmap-croaring-3.0.0-92024092815.al8.x86_64.rpm
devdeps-apache-arrow-20.0.0-452026020914.al8.x86_64.rpm
devdeps-vsag-abiv1-1.1.0-1752026041310.al8.x86_64.rpm
-devdeps-apache-orc-2.1.1-122026020912.al8.x86_64.rpm
devdeps-fast-float-6.1.3-42024112122.al8.x86_64.rpm
devdeps-sqlite-abiv1-3.38.1-72026031815.al8.x86_64.rpm
diff --git a/deps/init/oceanbase.android.arm64.deps b/deps/init/oceanbase.android.arm64.deps
index 57bf7a920..94f55016f 100644
--- a/deps/init/oceanbase.android.arm64.deps
+++ b/deps/init/oceanbase.android.arm64.deps
@@ -6,7 +6,6 @@ repo=https://mirrors.aliyun.com/oceanbase/development-kit/android/26/arm64
[deps]
devdeps-abseil-cpp-20211102.0-20260309.tar.gz
devdeps-apache-arrow-20.0.0-20260309.tar.gz
-devdeps-apache-orc-1.8.8-20260309.tar.gz
devdeps-aws-sdk-cpp-1.11.156-20260309.tar.gz
devdeps-boost-1.74.0-20260309.tar.gz
devdeps-fast-float-6.1.3-20260309.tar.gz
diff --git a/deps/init/oceanbase.el7.aarch64.deps b/deps/init/oceanbase.el7.aarch64.deps
index 5886b8199..986e02406 100644
--- a/deps/init/oceanbase.el7.aarch64.deps
+++ b/deps/init/oceanbase.el7.aarch64.deps
@@ -34,7 +34,6 @@ devdeps-grpc-abiv1-1.46.7-572026031815.el7.aarch64.rpm
devdeps-roaringbitmap-croaring-3.0.0-42024042816.el7.aarch64.rpm
devdeps-apache-arrow-20.0.0-452026020914.el7.aarch64.rpm
devdeps-vsag-abiv1-1.1.0-1752026041310.el7.aarch64.rpm
-devdeps-apache-orc-2.1.1-122026020912.el7.aarch64.rpm
devdeps-fast-float-6.1.3-42024112122.el7.aarch64.rpm
devdeps-sqlite-abiv1-3.38.1-72026031815.el7.aarch64.rpm
diff --git a/deps/init/oceanbase.el7.x86_64.deps b/deps/init/oceanbase.el7.x86_64.deps
index 73082fca4..614b01972 100644
--- a/deps/init/oceanbase.el7.x86_64.deps
+++ b/deps/init/oceanbase.el7.x86_64.deps
@@ -36,7 +36,6 @@ devdeps-grpc-abiv1-1.46.7-572026031815.el7.x86_64.rpm
devdeps-roaringbitmap-croaring-3.0.0-42024042816.el7.x86_64.rpm
devdeps-apache-arrow-20.0.0-452026020914.el7.x86_64.rpm
devdeps-vsag-abiv1-1.1.0-1752026041310.el7.x86_64.rpm
-devdeps-apache-orc-2.1.1-122026020912.el7.x86_64.rpm
devdeps-fast-float-6.1.3-42024112122.el7.x86_64.rpm
devdeps-sqlite-abiv1-3.38.1-72026031815.el7.x86_64.rpm
diff --git a/deps/init/oceanbase.el8.aarch64.deps b/deps/init/oceanbase.el8.aarch64.deps
index 114bdd544..0cf318ed7 100644
--- a/deps/init/oceanbase.el8.aarch64.deps
+++ b/deps/init/oceanbase.el8.aarch64.deps
@@ -34,7 +34,6 @@ devdeps-grpc-abiv1-1.46.7-572026031815.el8.aarch64.rpm
devdeps-roaringbitmap-croaring-3.0.0-42024042816.el8.aarch64.rpm
devdeps-apache-arrow-20.0.0-452026020914.el8.aarch64.rpm
devdeps-vsag-abiv1-1.1.0-1752026041310.el8.aarch64.rpm
-devdeps-apache-orc-2.1.1-122026020912.el8.aarch64.rpm
devdeps-fast-float-6.1.3-42024112122.el8.aarch64.rpm
devdeps-sqlite-abiv1-3.38.1-72026031815.el8.aarch64.rpm
diff --git a/deps/init/oceanbase.el8.x86_64.deps b/deps/init/oceanbase.el8.x86_64.deps
index 85d1c5ac5..e96735114 100644
--- a/deps/init/oceanbase.el8.x86_64.deps
+++ b/deps/init/oceanbase.el8.x86_64.deps
@@ -35,7 +35,6 @@ devdeps-grpc-abiv1-1.46.7-572026031815.el8.x86_64.rpm
devdeps-roaringbitmap-croaring-3.0.0-42024042816.el8.x86_64.rpm
devdeps-apache-arrow-20.0.0-452026020914.el8.x86_64.rpm
devdeps-vsag-abiv1-1.1.0-1752026041310.el8.x86_64.rpm
-devdeps-apache-orc-2.1.1-122026020912.el8.x86_64.rpm
devdeps-fast-float-6.1.3-42024112122.el8.x86_64.rpm
devdeps-sqlite-abiv1-3.38.1-72026031815.el8.x86_64.rpm
diff --git a/deps/init/oceanbase.el9.aarch64.deps b/deps/init/oceanbase.el9.aarch64.deps
index 507b6dbf2..1110f5fba 100644
--- a/deps/init/oceanbase.el9.aarch64.deps
+++ b/deps/init/oceanbase.el9.aarch64.deps
@@ -38,7 +38,6 @@ devdeps-grpc-abiv1-1.46.7-572026031815.el8.aarch64.rpm
devdeps-roaringbitmap-croaring-3.0.0-42024042816.el8.aarch64.rpm
devdeps-apache-arrow-20.0.0-452026020914.el8.aarch64.rpm
devdeps-vsag-abiv1-1.1.0-1752026041310.el8.aarch64.rpm
-devdeps-apache-orc-2.1.1-122026020912.el8.aarch64.rpm
devdeps-fast-float-6.1.3-42024112122.el8.aarch64.rpm
devdeps-sqlite-abiv1-3.38.1-72026031815.el8.aarch64.rpm
diff --git a/deps/init/oceanbase.el9.x86_64.deps b/deps/init/oceanbase.el9.x86_64.deps
index 0d764d4ba..d6f36741c 100644
--- a/deps/init/oceanbase.el9.x86_64.deps
+++ b/deps/init/oceanbase.el9.x86_64.deps
@@ -39,7 +39,6 @@ devdeps-grpc-abiv1-1.46.7-572026031815.el8.x86_64.rpm
devdeps-apache-arrow-20.0.0-452026020914.el8.x86_64.rpm
devdeps-roaringbitmap-croaring-3.0.0-42024042816.el8.x86_64.rpm
devdeps-vsag-abiv1-1.1.0-1752026041310.el8.x86_64.rpm
-devdeps-apache-orc-2.1.1-122026020912.el8.x86_64.rpm
devdeps-fast-float-6.1.3-42024112122.el8.x86_64.rpm
devdeps-sqlite-abiv1-3.38.1-72026031815.el8.x86_64.rpm
diff --git a/deps/init/oceanbase.macos13.arm64.deps b/deps/init/oceanbase.macos13.arm64.deps
index d6de9db04..e1198cede 100644
--- a/deps/init/oceanbase.macos13.arm64.deps
+++ b/deps/init/oceanbase.macos13.arm64.deps
@@ -11,7 +11,6 @@ repo=https://mirrors.aliyun.com/oceanbase/development-kit/darwin/13/arm64
[deps]
devdeps-abseil-cpp-20211102.0-20260126.tar.gz
devdeps-apache-arrow-20.0.0-20260205.tar.gz
-devdeps-apache-orc-1.8.8-20260126.tar.gz
devdeps-boost-1.74.0-20260126.tar.gz
devdeps-fast-float-6.1.3-20260126.tar.gz
devdeps-icu-69.1-20260126.tar.gz
diff --git a/deps/init/oceanbase.macos15.arm64.deps b/deps/init/oceanbase.macos15.arm64.deps
index cdd643f97..eabddfe80 100644
--- a/deps/init/oceanbase.macos15.arm64.deps
+++ b/deps/init/oceanbase.macos15.arm64.deps
@@ -11,7 +11,6 @@ repo=https://mirrors.aliyun.com/oceanbase/development-kit/darwin/15/arm64
[deps]
devdeps-abseil-cpp-20211102.0-20251205.tar.gz
devdeps-apache-arrow-20.0.0-20260204.tar.gz
-devdeps-apache-orc-1.8.8-20251208.tar.gz
devdeps-boost-1.74.0-20251205.tar.gz
devdeps-fast-float-6.1.3-20251205.tar.gz
devdeps-icu-69.1-20251205.tar.gz
diff --git a/deps/oblib/src/CMakeLists.txt b/deps/oblib/src/CMakeLists.txt
index ac193fce5..2661cb453 100644
--- a/deps/oblib/src/CMakeLists.txt
+++ b/deps/oblib/src/CMakeLists.txt
@@ -48,7 +48,7 @@ target_include_directories(
${DEP_DIR}/include/apr-1/
${DEP_DIR}/include/icu
${DEP_DIR}/include/icu/common
- ${DEP_DIR}/include/apache-arrow
+ $<$>:${DEP_DIR}/include/apache-arrow>
${DEP_DIR}/include/fast_float
${USSL_INCLUDE_DIRS}
)
@@ -100,7 +100,7 @@ if (OB_USE_CLANG)
-Wno-ambiguous-reversed-operator -Wno-invalid-partial-specialization
-Wno-string-concatenation
/EHsc -Wno-missing-noreturn -Wno-undefined-func-template
- -Wno-undef -Wno-sign-conversion -Wno-strict-prototypes -Wno-switch-default -Wno-language-extension-token
+ -Wno-undef -Wno-sign-conversion -Wno-strict-prototypes -Wno-switch-default -Wno-language-extension-token
-Wno-missing-prototypes -Wno-extra-semi-stmt -Wno-unused-macros -Wno-unsafe-buffer-usage -Wno-c++98-compat -Wno-c++98-compat-pedantic
-Wno-reserved-identifier -Wno-padded -Wno-variadic-macros -Wno-global-constructors -Wno-undefined-reinterpret-cast
-Wno-tautological-unsigned-zero-compare -Wno-integer-overflow -Wno-shift-count-overflow -Wno-cast-function-type -Wno-cast-function-type-strict
@@ -110,8 +110,8 @@ if (OB_USE_CLANG)
-Wno-exit-time-destructors -Wno-inconsistent-missing-destructor-override -Wno-shadow -Wno-unreachable-code
-Wno-switch-enum -Wno-atomic-implicit-seq-cst -Wno-documentation-unknown-command
-Wno-implicit-int-conversion -Wno-gnu-conditional-omitted-operand -Wno-cast-align -Wno-comma -Wno-tautological-type-limit-compare
- -Wno-disabled-macro-expansion -Wno-unreachable-code-break -Wno-missing-variable-declarations
- -Wno-nonportable-system-include-path -Wno-c99-extensions -Wno-flexible-array-extensions
+ -Wno-disabled-macro-expansion -Wno-unreachable-code-break -Wno-missing-variable-declarations
+ -Wno-nonportable-system-include-path -Wno-c99-extensions -Wno-flexible-array-extensions
-Wno-compound-token-split-by-space -Wno-newline-eof -Wno-shadow-field -Wno-microsoft-unqualified-friend -Wno-float-equal
-Wno-used-but-marked-unused -Wno-unreachable-code-return -Wno-float-conversion -Wno-conditional-uninitialized
-Wno-signed-enum-bitfield -Wno-bitfield-enum-conversion -Wno-redundant-parens -Wno-gnu-statement-expression
@@ -222,7 +222,7 @@ if(APPLE OR ANDROID)
set(_LIB_PATH "${DEP_DIR}/lib/")
# macOS and Android don't have libaio
set(LIBAIO_LINK_OPTION "")
- # On macOS, arrow/parquet are in lib/, but s2/orc/snappy/protoc/protobuf/lz4/zstd are in lib64/
+ # On macOS, arrow/parquet are in lib/, but s2/protoc/protobuf/lz4/zstd are in lib64/
if(APPLE)
set(_LIB_PATH_EXTRA "${DEP_DIR}/lib64/")
else()
@@ -322,12 +322,7 @@ if(WIN32)
${VCPKG_LIB_DIR}/brotlienc.lib
${VCPKG_LIB_DIR}/brotlidec.lib
${VCPKG_LIB_DIR}/bz2.lib
- ${VCPKG_LIB_DIR}/orc.lib
- ${VCPKG_LIB_DIR}/snappy.lib
- ${VCPKG_LIB_DIR}/libprotoc.lib
- ${VCPKG_LIB_DIR}/libprotobuf.lib
- ${VCPKG_LIB_DIR}/lz4.lib
- ${VCPKG_LIB_DIR}/zstd.lib
+
${VCPKG_LIB_DIR}/sqlite3.lib
${VCPKG_LIB_DIR}/pthreadVC3.lib
ws2_32 crypt32 bcrypt userenv wldap32 advapi32
@@ -351,16 +346,10 @@ else()
${DEP_DIR}/lib/libicustubdata.a
${DEP_DIR}/lib/libicuuc.a
${DEP_DIR}/lib/libprotobuf-c.a
- ${_LIB_PATH}/libarrow.a
- ${_LIB_PATH}/libparquet.a
- $<$>:${_LIB_PATH}/libarrow_bundled_dependencies.a>
- $<$:-L/opt/homebrew/lib -lutf8proc -lthrift -lre2 -lbrotlicommon -lbrotlienc -lbrotlidec -lbz2>
- ${_LIB_PATH_EXTRA}/liborc.a
- ${_LIB_PATH_EXTRA}/libsnappy.a
- $<$>:${_LIB_PATH_EXTRA}/libprotoc.a>
- ${_LIB_PATH_EXTRA}/libprotobuf.a
- ${_LIB_PATH_EXTRA}/liblz4.a
- ${_LIB_PATH_EXTRA}/libzstd.a
+ $<$>:${_LIB_PATH}/libarrow.a>
+ $<$>:${_LIB_PATH}/libparquet.a>
+ $<$>,$>>:${_LIB_PATH}/libarrow_bundled_dependencies.a>
+ $<$>,$>:-L/opt/homebrew/lib -lutf8proc -lthrift -lre2 -lbrotlicommon -lbrotlienc -lbrotlidec -lbz2 -llz4 -lzstd>
$<$:-L${DEP_DIR}/var/usr/lib64 -L${DEP_DIR}/var/usr/lib -L${DEP_3RD_DIR}/usr/lib -L${DEP_3RD_DIR}/usr/lib64>
${LIBAIO_LINK_OPTION} $<$>:-lpthread> -ldl
$<$:-llog>
diff --git a/deps/oblib/src/grpc/CMakeLists.txt b/deps/oblib/src/grpc/CMakeLists.txt
index ded6fdda9..919c898e5 100644
--- a/deps/oblib/src/grpc/CMakeLists.txt
+++ b/deps/oblib/src/grpc/CMakeLists.txt
@@ -50,28 +50,53 @@ foreach(PROTO_NAME ${PROTO_NAMES})
set(GRPC_CC "${PROTO_GEN_DIR}/${PROTO_NAME}.grpc.pb.cc")
set(GRPC_H "${PROTO_GEN_DIR}/${PROTO_NAME}.grpc.pb.h")
- add_custom_command(
- OUTPUT ${PB_CC} ${PB_H}
- COMMAND ${CMAKE_COMMAND} -E env "LD_LIBRARY_PATH=${_PROTOC_LD_PATH}:$ENV{LD_LIBRARY_PATH}"
- ${_PROTOC}
- --cpp_out=${PROTO_GEN_DIR}
- -I${PROTO_SRC_DIR}
- ${PROTO_FILE}
- DEPENDS ${PROTO_FILE}
- COMMENT "Generating protobuf C++ from ${PROTO_NAME}.proto"
- )
+ # Linux/macOS: protoc may need LD_LIBRARY_PATH for bundled libprotobuf.so.
+ # Windows (vcpkg): invoke protoc/grpc_cpp_plugin directly; LD_LIBRARY_PATH is meaningless.
+ if(WIN32)
+ add_custom_command(
+ OUTPUT ${PB_CC} ${PB_H}
+ COMMAND ${_PROTOC}
+ --cpp_out=${PROTO_GEN_DIR}
+ -I${PROTO_SRC_DIR}
+ ${PROTO_FILE}
+ DEPENDS ${PROTO_FILE}
+ COMMENT "Generating protobuf C++ from ${PROTO_NAME}.proto"
+ )
- add_custom_command(
- OUTPUT ${GRPC_CC} ${GRPC_H}
- COMMAND ${CMAKE_COMMAND} -E env "LD_LIBRARY_PATH=${_PROTOC_LD_PATH}:$ENV{LD_LIBRARY_PATH}"
- ${_PROTOC}
- --grpc_out=${PROTO_GEN_DIR}
- --plugin=protoc-gen-grpc=${_GRPC_CPP_PLUGIN}
- -I${PROTO_SRC_DIR}
- ${PROTO_FILE}
- DEPENDS ${PROTO_FILE}
- COMMENT "Generating gRPC C++ from ${PROTO_NAME}.proto"
- )
+ add_custom_command(
+ OUTPUT ${GRPC_CC} ${GRPC_H}
+ COMMAND ${_PROTOC}
+ --grpc_out=${PROTO_GEN_DIR}
+ --plugin=protoc-gen-grpc=${_GRPC_CPP_PLUGIN}
+ -I${PROTO_SRC_DIR}
+ ${PROTO_FILE}
+ DEPENDS ${PROTO_FILE}
+ COMMENT "Generating gRPC C++ from ${PROTO_NAME}.proto"
+ )
+ else()
+ add_custom_command(
+ OUTPUT ${PB_CC} ${PB_H}
+ COMMAND ${CMAKE_COMMAND} -E env "LD_LIBRARY_PATH=${_PROTOC_LD_PATH}:$ENV{LD_LIBRARY_PATH}"
+ ${_PROTOC}
+ --cpp_out=${PROTO_GEN_DIR}
+ -I${PROTO_SRC_DIR}
+ ${PROTO_FILE}
+ DEPENDS ${PROTO_FILE}
+ COMMENT "Generating protobuf C++ from ${PROTO_NAME}.proto"
+ )
+
+ add_custom_command(
+ OUTPUT ${GRPC_CC} ${GRPC_H}
+ COMMAND ${CMAKE_COMMAND} -E env "LD_LIBRARY_PATH=${_PROTOC_LD_PATH}:$ENV{LD_LIBRARY_PATH}"
+ ${_PROTOC}
+ --grpc_out=${PROTO_GEN_DIR}
+ --plugin=protoc-gen-grpc=${_GRPC_CPP_PLUGIN}
+ -I${PROTO_SRC_DIR}
+ ${PROTO_FILE}
+ DEPENDS ${PROTO_FILE}
+ COMMENT "Generating gRPC C++ from ${PROTO_NAME}.proto"
+ )
+ endif()
list(APPEND GENERATED_SRCS ${PB_CC} ${GRPC_CC})
endforeach()
diff --git a/deps/oblib/src/lib/alloc/alloc_func.cpp b/deps/oblib/src/lib/alloc/alloc_func.cpp
index 14d8a8fb3..ca2f2e2d5 100644
--- a/deps/oblib/src/lib/alloc/alloc_func.cpp
+++ b/deps/oblib/src/lib/alloc/alloc_func.cpp
@@ -216,14 +216,6 @@ int set_meta_obj_limit(uint64_t tenant_id, int64_t meta_obj_pct_lmt)
return set_ctx_limit(tenant_id, common::ObCtxIds::META_OBJ_CTX_ID, ctx_limit);
}
-int set_rpc_limit(uint64_t tenant_id, int64_t rpc_pct_lmt)
-{
- if (OB_SYS_TENANT_ID != tenant_id) return OB_SUCCESS;
- const int64_t tenant_limit = get_tenant_memory_limit(tenant_id);
- const int64_t rpc_lmt = (tenant_limit / 100) * rpc_pct_lmt;
- return set_ctx_limit(tenant_id, common::ObCtxIds::RPC_CTX_ID, rpc_lmt);
-}
-
bool errsim_alloc(const ObMemAttr &attr)
{
int en4_val = (int)EventTable::EN_4;
diff --git a/deps/oblib/src/lib/alloc/alloc_func.h b/deps/oblib/src/lib/alloc/alloc_func.h
index f8f638df2..b83ba6a3b 100644
--- a/deps/oblib/src/lib/alloc/alloc_func.h
+++ b/deps/oblib/src/lib/alloc/alloc_func.h
@@ -63,10 +63,6 @@ int set_wa_limit(uint64_t tenand_id, int64_t wa_pctg);
// - meta_obj_pct_lmt: percentage limitation of tenant memory can be used for meta object.
int set_meta_obj_limit(uint64_t tenant_id, int64_t meta_obj_pct_lmt);
-// set rpc memory limit for specified tenant.
-// - rpc_pct_lmt: percentage limitation of tenant rpc memory.
-int set_rpc_limit(uint64_t tenant_id, int64_t rpc_pct_lmt);
-
bool errsim_alloc(const ObMemAttr &attr);
int set_req_chunkmgr_parallel(uint64_t tenant_id, uint64_t ctx_id, int32_t parallel);
diff --git a/deps/oblib/src/lib/alloc/memory_dump.cpp b/deps/oblib/src/lib/alloc/memory_dump.cpp
index dcf1667fb..4d6d4b02c 100644
--- a/deps/oblib/src/lib/alloc/memory_dump.cpp
+++ b/deps/oblib/src/lib/alloc/memory_dump.cpp
@@ -23,6 +23,7 @@
#ifndef _WIN32
#include
#endif
+#include
#include "lib/signal/ob_signal_struct.h"
#include "lib/thread/thread_mgr.h"
#include "lib/container/ob_vector.h"
@@ -46,36 +47,105 @@ RLOCAL(sigjmp_buf, jmp);
static void dump_handler(int sig, siginfo_t *s, void *p)
{
- if (SIGSEGV == sig || SIGABRT == sig) {
- siglongjmp(jmp, 1);
- } else {
- ob_signal_handler(sig, s, p);
- }
+ siglongjmp(jmp, 1);
}
-#endif
-template
-void do_with_segv_catch(Function &&func, bool &has_segv, decltype(func()) &ret)
+class DumpSignalGuard final
{
- has_segv = false;
-#ifndef _WIN32
- signal_handler_t handler_bak = get_signal_handler();
- int js = sigsetjmp(jmp, 1);
- if (0 == js) {
- get_signal_handler() = dump_handler;
- ret = func();
- } else if (1 == js) {
- has_segv = true;
- } else {
- LOG_ERROR_RET(OB_ERR_UNEXPECTED, "unexpected error!!!", K(js));
- ob_abort();
+public:
+ DumpSignalGuard()
+ {
+ install_dump_signal_handler();
}
- get_signal_handler() = handler_bak;
-#else
- ret = func();
-#endif
-}
+ ~DumpSignalGuard()
+ {
+ restore_dump_signal_handler();
+ }
+
+public:
+
+ template
+ void do_with_segv_catch(Function &&func, bool &has_segv, decltype(func()) &ret)
+ {
+ has_segv = false;
+ if (installed_) {
+ has_segv = false;
+ int js = sigsetjmp(jmp, 1);
+ if (0 == js) {
+ ret = func();
+ } else if (1 == js) {
+ has_segv = true;
+ } else {
+ LOG_ERROR_RET(OB_ERR_UNEXPECTED, "unexpected error!!!", K(js));
+ ob_abort();
+ }
+ } else {
+ ret = func();
+ }
+ }
+
+private:
+ void install_dump_signal_handler()
+ {
+ int ret = OB_SUCCESS;
+ struct sigaction sa_new;
+ sa_new.sa_flags = SA_SIGINFO;
+ sa_new.sa_sigaction = dump_handler;
+ sigemptyset(&sa_new.sa_mask);
+ installed_ = true;
+ int i = 0;
+ for (i = 0; OB_SUCC(ret) && i < ARRAYSIZEOF(signals_); i++) {
+ if (sigaction(signals_[i], &sa_new, &sa_old_[i]) != 0) {
+ ret = OB_ERR_SYS;
+ LOG_WARN_RET(ret, "failed to install signal handler", K(errno));
+ }
+ }
+ if (OB_SUCC(ret)) {
+ installed_ = true;
+ } else {
+ installed_ = false;
+ for (int j = 0; j < i; j++) {
+ if (sigaction(signals_[j], &sa_old_[j], nullptr) != 0) {
+ LOG_WARN_RET(OB_ERR_SYS, "failed to restore signal handler", K(errno));
+ }
+ }
+ }
+ }
+
+ void restore_dump_signal_handler()
+ {
+ if (installed_) {
+ for (int i = 0; i < ARRAYSIZEOF(signals_); i++) {
+ if (sigaction(signals_[i], &sa_old_[i], nullptr) != 0) {
+ LOG_WARN_RET(OB_ERR_SYS, "failed to restore signal handler", K(errno));
+ }
+ }
+ installed_ = false;
+ }
+ }
+private:
+ static constexpr int signals_[] = {SIGSEGV, SIGABRT};
+ struct sigaction sa_old_[ARRAYSIZEOF(signals_)];
+ bool installed_ = false;
+};
+
+#else // _WIN32
+
+class DumpSignalGuard final
+{
+public:
+ DumpSignalGuard() = default;
+ ~DumpSignalGuard() = default;
+
+ template
+ void do_with_segv_catch(Function &&func, bool &has_segv, decltype(func()) &ret)
+ {
+ has_segv = false;
+ ret = func();
+ }
+};
+#endif // _WIN32
ObMemoryDump::ObMemoryDump()
: task_mutex_(ObLatchIds::ALLOC_MEM_DUMP_TASK_LOCK),
@@ -324,7 +394,8 @@ AChunk *ObMemoryDump::find_chunk(void *ptr)
return ret;
};
bool has_segv = false;
- do_with_segv_catch(func, has_segv, ret);
+ DumpSignalGuard guard;
+ guard.do_with_segv_catch(func, has_segv, ret);
if (has_segv) {
LOG_INFO("restore from sigsegv, let's goon~");
}
@@ -552,6 +623,7 @@ void ObMemoryDump::handle(void *task)
auto &w_stat = w_stat_;
auto &lmap = lmap_;
lmap.clear();
+ DumpSignalGuard guard;
for (int i = 0; OB_SUCC(ret) && i < chunk_cnt; i++) {
AChunk *chunk = chunks_[i];
auto func = [&, chunk] {
@@ -584,7 +656,7 @@ void ObMemoryDump::handle(void *task)
return ret;
};
bool has_segv = false;
- do_with_segv_catch(func, has_segv, ret);
+ guard.do_with_segv_catch(func, has_segv, ret);
if (has_segv) {
LOG_INFO("restore from sigsegv, let's goon~");
segv_cnt++;
@@ -714,6 +786,7 @@ void ObMemoryDump::handle(void *task)
// sort chunk
lib::ob_sort(chunks_, chunks_ + cnt);
// iter chunk
+ DumpSignalGuard guard;
for (int i = 0; OB_SUCC(ret) && i < cnt; i++) {
AChunk *chunk = chunks_[i];
char *print_buf = print_buf_; // for lambda capture
@@ -736,7 +809,7 @@ void ObMemoryDump::handle(void *task)
return OB_SUCCESS;
};
bool has_segv = false;
- do_with_segv_catch(func, has_segv, ret);
+ guard.do_with_segv_catch(func, has_segv, ret);
if (has_segv) {
LOG_INFO("restore from sigsegv, let's goon~");
continue;
diff --git a/deps/oblib/src/lib/alloc/ob_malloc_allocator.cpp b/deps/oblib/src/lib/alloc/ob_malloc_allocator.cpp
index b265e6d03..55a69a81f 100644
--- a/deps/oblib/src/lib/alloc/ob_malloc_allocator.cpp
+++ b/deps/oblib/src/lib/alloc/ob_malloc_allocator.cpp
@@ -133,8 +133,6 @@ int ObMallocAllocator::create_tenant_allocator(uint64_t tenant_id, void *buf,
ObTenantCtxAllocatorV2(tenant_id, ctx_id, &tmp_allocator[ctx_id]);
if (OB_FAIL(ctx_allocator[ctx_id].set_tenant_memory_mgr())) {
LOG_ERROR("set_tenant_memory_mgr failed", K(ret));
- } else if (ObCtxIds::DO_NOT_USE_ME == ctx_id) {
- ctx_allocator[ctx_id].set_limit(256L<<20);
}
new (ctx_allocator[ctx_id].get_allocator())
ObTenantCtxAllocator(ctx_allocator[ctx_id], tenant_id, ctx_id);
diff --git a/deps/oblib/src/lib/allocator/ob_ctx_define.h b/deps/oblib/src/lib/allocator/ob_ctx_define.h
index 261b554aa..592992b7e 100644
--- a/deps/oblib/src/lib/allocator/ob_ctx_define.h
+++ b/deps/oblib/src/lib/allocator/ob_ctx_define.h
@@ -41,23 +41,17 @@ struct ObCtxAttrCenter
{
#define PARALLEL_DEF(name, parallel) ctx_attr_[ObCtxIds::name].parallel_ = parallel;
PARALLEL_DEF(DEFAULT_CTX_ID, 32)
- PARALLEL_DEF(LIBEASY, 32)
PARALLEL_DEF(PLAN_CACHE_CTX_ID, 4)
PARALLEL_DEF(LOGGER_CTX_ID, 4)
#undef CTX_PARALLEL_DEF
#define ENABLE_DIRTY_LIST_DEF(name) ctx_attr_[ObCtxIds::name].enable_dirty_list_ = true;
- ENABLE_DIRTY_LIST_DEF(LIBEASY)
ENABLE_DIRTY_LIST_DEF(LOGGER_CTX_ID)
#undef ENABLE_DIRTY_LIST_DEF
#define ENABLE_NO_LOG_DEF(name) ctx_attr_[ObCtxIds::name].enable_no_log_ = true;
ENABLE_NO_LOG_DEF(LOGGER_CTX_ID)
#undef ENABLE_NO_LOG_DEF
-
-#define DISABLE_SYNC_WASH_DEF(name) ctx_attr_[ObCtxIds::name].disable_sync_wash_ = true;
- DISABLE_SYNC_WASH_DEF(MERGE_RESERVE_CTX_ID)
-#undef DISABLE_SYNC_WASH_DEF
}
static ObCtxAttrCenter &instance();
ObCtxAttr attr_of_ctx(int64_t ctx_id) const
diff --git a/deps/oblib/src/lib/allocator/ob_libeasy_mem_pool.cpp b/deps/oblib/src/lib/allocator/ob_libeasy_mem_pool.cpp
index b5f06d630..94ae4ffc0 100644
--- a/deps/oblib/src/lib/allocator/ob_libeasy_mem_pool.cpp
+++ b/deps/oblib/src/lib/allocator/ob_libeasy_mem_pool.cpp
@@ -35,7 +35,7 @@ void *common::ob_easy_realloc(void *ptr, size_t size)
}
auto &set = obrpc::ObRpcPacketSet::instance();
attr.label_ = set.name_of_idx(set.idx_of_pcode(pcode));
- attr.ctx_id_ = ObCtxIds::LIBEASY;
+ attr.ctx_id_ = ObCtxIds::DEFAULT_CTX_ID;
attr.tenant_id_ = OB_SERVER_TENANT_ID;
{
TP_SWITCH_GUARD(true);
diff --git a/deps/oblib/src/lib/allocator/ob_mod_define.h b/deps/oblib/src/lib/allocator/ob_mod_define.h
index fb9c85a57..5f0bafc10 100644
--- a/deps/oblib/src/lib/allocator/ob_mod_define.h
+++ b/deps/oblib/src/lib/allocator/ob_mod_define.h
@@ -16,29 +16,15 @@
#ifdef CTX_ITEM_DEF
CTX_ITEM_DEF(DEFAULT_CTX_ID)
-CTX_ITEM_DEF(DO_NOT_USE_ME)
CTX_ITEM_DEF(MEMSTORE_CTX_ID)
-CTX_ITEM_DEF(EXECUTE_CTX_ID)
-CTX_ITEM_DEF(TRANS_CTX_MGR_ID)
CTX_ITEM_DEF(PLAN_CACHE_CTX_ID)
CTX_ITEM_DEF(WORK_AREA)
CTX_ITEM_DEF(GLIBC)
CTX_ITEM_DEF(CO_STACK)
-CTX_ITEM_DEF(LIBEASY)
CTX_ITEM_DEF(LOGGER_CTX_ID)
CTX_ITEM_DEF(KVSTORE_CACHE_ID)
CTX_ITEM_DEF(META_OBJ_CTX_ID)
-CTX_ITEM_DEF(TX_CALLBACK_CTX_ID)
-CTX_ITEM_DEF(LOB_CTX_ID)
-CTX_ITEM_DEF(RPC_CTX_ID)
-CTX_ITEM_DEF(PKT_NIO)
-CTX_ITEM_DEF(TX_DATA_TABLE)
-CTX_ITEM_DEF(STORAGE_LONG_TERM_META_CTX_ID)
-CTX_ITEM_DEF(MDS_DATA_ID)
-CTX_ITEM_DEF(MDS_CTX_ID)
CTX_ITEM_DEF(UNEXPECTED_IN_500)
-CTX_ITEM_DEF(MERGE_RESERVE_CTX_ID)
-CTX_ITEM_DEF(MERGE_NORMAL_CTX_ID)
CTX_ITEM_DEF(VECTOR_CTX_ID)
CTX_ITEM_DEF(MAX_CTX_ID)
#endif
@@ -629,6 +615,15 @@ struct ObCtxIds
#undef CTX_ITEM_DEF
};
static constexpr int SCHEMA_SERVICE = DEFAULT_CTX_ID;
+ static constexpr int EXECUTE_CTX_ID = DEFAULT_CTX_ID;
+ static constexpr int TRANS_CTX_MGR_ID = DEFAULT_CTX_ID;
+ static constexpr int TX_CALLBACK_CTX_ID = DEFAULT_CTX_ID;
+ static constexpr int LOB_CTX_ID = DEFAULT_CTX_ID;
+ static constexpr int TX_DATA_TABLE = DEFAULT_CTX_ID;
+ static constexpr int MDS_DATA_ID = DEFAULT_CTX_ID;
+ static constexpr int MDS_CTX_ID = DEFAULT_CTX_ID;
+ static constexpr int MERGE_RESERVE_CTX_ID = DEFAULT_CTX_ID;
+ static constexpr int MERGE_NORMAL_CTX_ID = DEFAULT_CTX_ID;
};
class ObCtxInfo
diff --git a/deps/oblib/src/lib/allocator/ob_tc_malloc.cpp b/deps/oblib/src/lib/allocator/ob_tc_malloc.cpp
index 24ef906b9..2fa5e5264 100644
--- a/deps/oblib/src/lib/allocator/ob_tc_malloc.cpp
+++ b/deps/oblib/src/lib/allocator/ob_tc_malloc.cpp
@@ -115,9 +115,6 @@ void __attribute__((constructor(MALLOC_INIT_PRIORITY))) init_global_memory_pool
in_hook()= true;
global_default_allocator = ObMallocAllocator::get_instance();
in_hook()= false;
-#ifndef OB_USE_ASAN
- abort_unless(OB_SUCCESS == install_ob_signal_handler());
-#endif
init_proc_map_info();
#ifdef ENABLE_SANITY
abort_unless(init_sanity());
diff --git a/deps/oblib/src/lib/cpu/ob_cpu_topology.cpp b/deps/oblib/src/lib/cpu/ob_cpu_topology.cpp
index 061e07d15..2d1ab43ca 100644
--- a/deps/oblib/src/lib/cpu/ob_cpu_topology.cpp
+++ b/deps/oblib/src/lib/cpu/ob_cpu_topology.cpp
@@ -75,7 +75,12 @@ int CpuFlagSet::init_from_os(uint64_t& flags)
{
int ret = OB_SUCCESS;
flags = 0;
-#if defined(__linux__)
+#if defined(__ANDROID__)
+ // Android defines __linux__ as well, but app processes are sandboxed and shelling
+ // out to grep /proc/cpuinfo via system()/posix_spawn() is unsafe during startup.
+ // Rely on direct CPU probing instead.
+ init_from_cpu(flags);
+#elif defined(__linux__)
const char* const CPU_FLAG_CMDS[(int)CpuFlag::MAX] = {"grep -E ' sse4_2( |$)' /proc/cpuinfo > /dev/null 2>&1",
"grep -E ' avx( |$)' /proc/cpuinfo > /dev/null 2>&1",
"grep -E ' avx2( |$)' /proc/cpuinfo > /dev/null 2>&1",
@@ -95,8 +100,8 @@ int CpuFlagSet::init_from_os(uint64_t& flags)
flags |= (1 << i);
}
}
-#elif defined(__APPLE__) || defined(__ANDROID__)
- // On macOS/Android, /proc/cpuinfo doesn't exist or SSE/AVX features are irrelevant.
+#elif defined(__APPLE__)
+ // On macOS, /proc/cpuinfo doesn't exist or SSE/AVX features are irrelevant.
// We can use sysctl to check for features, but for now we rely on init_from_cpu
// and just return success here with flags set to a reasonable default or
// matched with cpu flags to avoid mismatch error in constructor.
diff --git a/deps/oblib/src/lib/file/ob_file.cpp b/deps/oblib/src/lib/file/ob_file.cpp
index 8183cb986..02ac732a0 100644
--- a/deps/oblib/src/lib/file/ob_file.cpp
+++ b/deps/oblib/src/lib/file/ob_file.cpp
@@ -1036,22 +1036,42 @@ int64_t pwrite_align(const int fd, const void *buf, const int64_t count, const i
int64_t get_file_size(const int fd)
{
int64_t ret = -1;
+#ifdef _WIN32
+ // MSVCRT's `struct stat::st_size` is 32-bit (_off_t = long); files larger
+ // than 2 GiB get a truncated size. Use _fstat64 to report the real size.
+ struct _stat64 st;
+ if (-1 != fd
+ && 0 == _fstat64(fd, &st)) {
+ ret = st.st_size;
+ }
+#else
struct stat st;
if (-1 != fd
&& 0 == fstat(fd, &st)) {
ret = st.st_size;
}
+#endif
return ret;
}
int64_t get_file_size(const char *fname)
{
int64_t ret = -1;
+#ifdef _WIN32
+ // MSVCRT's `struct stat::st_size` is 32-bit (_off_t = long); files larger
+ // than 2 GiB get a truncated size. Use _stat64 to report the real size.
+ struct _stat64 st;
+ if (NULL != fname
+ && 0 == _stat64(fname, &st)) {
+ ret = st.st_size;
+ }
+#else
struct stat st;
if (NULL != fname
&& 0 == stat(fname, &st)) {
ret = st.st_size;
}
+#endif
return ret;
}
@@ -1459,7 +1479,11 @@ void ObFileAsyncAppender::close()
// Fatal error
_OB_LOG_RET(ERROR, OB_ERR_SYS, "fsync fail fd=%d, will set fd=-1, and the fd will leek", fd_);
} else {
+#ifdef _WIN32
+ if (0 != ob_ftruncate(fd_, file_pos_)) {
+#else
if (0 != ftruncate(fd_, file_pos_)) {
+#endif
OB_LOG_RET(WARN, OB_ERR_SYS, "fail to truncate file ", K_(fd), K(errno), KERRMSG);
}
if (0 != ::close(fd_)) {
diff --git a/deps/oblib/src/lib/lock/ob_scond.h b/deps/oblib/src/lib/lock/ob_scond.h
index b5db57c1d..61bd46797 100644
--- a/deps/oblib/src/lib/lock/ob_scond.h
+++ b/deps/oblib/src/lib/lock/ob_scond.h
@@ -138,17 +138,22 @@ struct SCondTemp
typedef SCondCounter Counter;
typedef SCondSimpleIdGen IdGen;
enum { CPU_COUNT = OB_MAX_CPU_NUM, COND_COUNT = CPU_COUNT, LOOP_LIMIT = 8 };
- void signal(uint32_t x = 1, int prio=0) {
- uint32_t icpu_id = id_gen_.get();
- for (int p = PRIO-1; p >= prio && x > 0; p--) {
- x -= conds_[icpu_id % COND_COUNT][p].signal(x);
+ void signal(uint32_t x = 1, int prio=0, bool fixed_wakeup_order = false) {
+ uint32_t icpu_id = 0;
+ if (fixed_wakeup_order == false) {
+ icpu_id = id_gen_.get();
+ for (int p = PRIO-1; p >= prio && x > 0; p--) {
+ x -= conds_[icpu_id % COND_COUNT][p].signal(x);
+ }
}
if (x > 0) {
- n2wakeup_.add(x, icpu_id);
+ if (fixed_wakeup_order == false) {
+ n2wakeup_.add(x, icpu_id);
+ }
int64_t loop_cnt = 0;
while(loop_cnt < LOOP_LIMIT) {
if (lock_.trylock()) {
- do_wakeup();
+ do_wakeup(fixed_wakeup_order);
lock_.unlock();
break;
} else {
@@ -157,13 +162,13 @@ struct SCondTemp
}
}
if (loop_cnt > LOOP_LIMIT) {
- do_wakeup();
+ do_wakeup(fixed_wakeup_order);
}
}
}
- void prepare(int prio=0) {
+ void prepare(int prio=0, int32_t index = -1) {
uint32_t id = 0;
- uint32_t key = get_key(prio, id);
+ uint32_t key = get_key(prio, id, index);
id += (prio << 16);
get_wait_key() = ((uint64_t)id<<32) + key;
}
@@ -172,7 +177,15 @@ struct SCondTemp
return wait((uint32_t)(wait_key>>32), (uint32_t)wait_key, timeout);
}
protected:
- uint32_t get_key(int prio, uint32_t& id) { return conds_[id = (id_gen_.next() % COND_COUNT)][prio].get_key(); }
+ uint32_t get_key(int prio, uint32_t& id, int32_t index)
+ {
+ if (index >= 0) {
+ id = index % COND_COUNT;
+ } else {
+ id = (id_gen_.next() % COND_COUNT);
+ }
+ return conds_[id][prio].get_key();
+ }
int wait(uint32_t id, uint32_t key, int64_t timeout) {
return conds_[((uint16_t)id) % COND_COUNT][id >> 16].wait(key, timeout);
}
@@ -181,11 +194,15 @@ struct SCondTemp
RLOCAL(uint64_t, key);
return key;
}
- void do_wakeup() {
+ void do_wakeup(bool fixed_wakeup_order = false) {
uint32_t n2wakeup = 0;
- //for (int p = PRIO - 1; p >= 0; p--) {
- n2wakeup = n2wakeup_.fetch();
- // }
+ if (fixed_wakeup_order == false) {
+ //for (int p = PRIO - 1; p >= 0; p--) {
+ n2wakeup = n2wakeup_.fetch();
+ // }
+ } else {
+ n2wakeup = 1;
+ }
for (int p = PRIO - 1; n2wakeup > 0 && p >= 0; p--) {
for(int i = 0; n2wakeup > 0 && i < COND_COUNT; i++) {
n2wakeup -= conds_[i][p].signal(n2wakeup);
diff --git a/deps/oblib/src/lib/ob_define.h b/deps/oblib/src/lib/ob_define.h
index b611be203..3b82468aa 100644
--- a/deps/oblib/src/lib/ob_define.h
+++ b/deps/oblib/src/lib/ob_define.h
@@ -500,6 +500,7 @@ const int64_t OB_MAX_READ_ONLY_STATE_LENGTH = 16;
//At present, the log module reads and writes the buffer using OB_MAX_LOG_BUFFER_SIZE, the length of the transaction submitted to the log module is required to be less than the length of the log module can read and write the log, minus the length of the log header, the BLOCK header and the EOF, here is defined a length minus 1024B
const int64_t OB_MAX_LOG_ALLOWED_SIZE = 1965056L; //OB_MAX_LOG_BUFFER_SIZE - 1024B
const int64_t OB_MAX_LOG_BUFFER_SIZE = 1966080L; // 1.875MB
+
const int64_t OB_MAX_TRIGGER_VCHAR_PARAM_LENGTH = 128;
const int64_t OB_TRIGGER_MSG_LENGTH = 3 * MAX_IP_ADDR_LENGTH
+ OB_TRIGGER_TYPE_LENGTH + 3 * OB_MAX_TRIGGER_VCHAR_PARAM_LENGTH;
diff --git a/deps/oblib/src/lib/ob_running_mode.cpp b/deps/oblib/src/lib/ob_running_mode.cpp
index d09a4320f..c57eed51e 100644
--- a/deps/oblib/src/lib/ob_running_mode.cpp
+++ b/deps/oblib/src/lib/ob_running_mode.cpp
@@ -23,7 +23,6 @@ const int64_t ObRunningModeConfig::MINI_MEM_LOWER = 1L << 30;
const int64_t ObRunningModeConfig::MINI_MEM_UPPER = 12L << 30;
const int64_t ObRunningModeConfig::MINI_CPU_UPPER = 8;
-bool OB_WEAK_SYMBOL mtl_is_mini_mode() { return false; }
} //end of namespace lib
} //end of namespace oceanbase
diff --git a/deps/oblib/src/lib/ob_running_mode.h b/deps/oblib/src/lib/ob_running_mode.h
index e13d22abe..7a2a270ba 100644
--- a/deps/oblib/src/lib/ob_running_mode.h
+++ b/deps/oblib/src/lib/ob_running_mode.h
@@ -22,7 +22,6 @@ namespace oceanbase
{
namespace lib
{
-extern bool mtl_is_mini_mode();
struct ObRunningModeConfig
{
@@ -46,7 +45,7 @@ inline ObRunningModeConfig &ObRunningModeConfig::instance()
inline bool is_mini_mode()
{
- return ObRunningModeConfig::instance().mini_mode_ || mtl_is_mini_mode();
+ return ObRunningModeConfig::instance().mini_mode_;
}
inline bool is_mini_cpu_mode()
diff --git a/deps/oblib/src/lib/objectpool/ob_server_object_pool.h b/deps/oblib/src/lib/objectpool/ob_server_object_pool.h
index b06256ab6..23c096097 100644
--- a/deps/oblib/src/lib/objectpool/ob_server_object_pool.h
+++ b/deps/oblib/src/lib/objectpool/ob_server_object_pool.h
@@ -171,7 +171,7 @@ class ObServerObjectPool
}
/**
- * Brutally allocate 16 available objects for each entry at Pool construction time
+ * Pre-allocate a fixed number of available objects for each arena at pool construction time.
* Memory is directly allocated in one go using ob_malloc based on the total size
* All objects are placed into their respective entries
* Since it is a global singleton, this work is completed at program startup
@@ -188,10 +188,11 @@ class ObServerObjectPool
{
int ret = OB_SUCCESS;
const bool is_mini = is_mini_mode_;
- arena_num_ = min(64/*upper_bound*/, max(4/*lower_bound*/, static_cast(cpu_count_) * 2));
+ const int cpu_factor = is_mini ? 1 : 2;
+ arena_num_ = min(64/*upper_bound*/, max(4/*lower_bound*/, static_cast(cpu_count_) * cpu_factor));
//If the assignment logic of buf_ below is not reached, buf_ will not be initialized
buf_ = NULL;
- cnt_per_arena_ = is_mini ? 8 : 64;
+ cnt_per_arena_ = is_mini ? 4 : 64;
int64_t s = (sizeof(T) + sizeof(Meta)); // Each cached object header has a Meta field to store necessary information and linked list pointers
item_size_ = upper_align(s, CACHE_ALIGN_SIZE); // Align according to the cache line to ensure that there will be no false sharing between objects
ObMemAttr attr(tenant_id_, LABEL);
diff --git a/deps/oblib/src/lib/oblog/ob_log.h b/deps/oblib/src/lib/oblog/ob_log.h
index b51f068e2..32d1a5abe 100644
--- a/deps/oblib/src/lib/oblog/ob_log.h
+++ b/deps/oblib/src/lib/oblog/ob_log.h
@@ -1016,6 +1016,10 @@ void ObLogger::log_it(const char *mod_name,
// (before main() enters or after main() returns). Fall back to stderr.
if (OB_UNLIKELY(!g_ob_log_main_entered)) {
if (OB_NOT_NULL(file) && OB_NOT_NULL(function)) {
+ // Static init is noisy at INFO; stderr fallback cannot honor module filters — drop chatter only.
+ if (level == OB_LOG_LEVEL_INFO || level == OB_LOG_LEVEL_TRACE || level == OB_LOG_LEVEL_DEBUG) {
+ return;
+ }
static constexpr const char *const lvlstr[] = {"ERROR", "WARN", "INFO", "EDIAG", "WDIAG", "TRACE", "DEBUG"};
const char *lvl = (level >= 0 && level < (int)(sizeof(lvlstr)/sizeof(lvlstr[0]))) ? lvlstr[level] : "?";
fprintf(stderr, "[PRE-MAIN] %-5s %s:%d %s\n", lvl, file, line, function);
diff --git a/deps/oblib/src/lib/profile/ob_trace_id.h b/deps/oblib/src/lib/profile/ob_trace_id.h
index 039f39deb..d288077f1 100644
--- a/deps/oblib/src/lib/profile/ob_trace_id.h
+++ b/deps/oblib/src/lib/profile/ob_trace_id.h
@@ -24,8 +24,11 @@ namespace oceanbase
{
namespace common
{
-#define TRACE_ID_FORMAT_V4 "Y%lX-%016lX-%lx-%lx"
-#define TRACE_ID_FORMAT_V6 "Z%X-%016lX-%lx-%lx"
+// Use %llX (%lX on Linux = 64-bit, but %lX on Windows = 32-bit which truncates
+// the port field stored in the upper 32 bits of uval_[0]). %llX is always
+// 64-bit on both platforms.
+#define TRACE_ID_FORMAT_V4 "Y%llX-%016llX-%llx-%llx"
+#define TRACE_ID_FORMAT_V6 "Z%X-%016llX-%llx-%llx"
struct ObCurTraceId
{
class SeqGenerator
@@ -121,9 +124,14 @@ struct ObCurTraceId
{
int64_t pos = 0;
if (!id_.is_ipv6_) {
- common::databuff_printf(buf, buf_len, pos, TRACE_ID_FORMAT_V4, uval_[0], uval_[1], uval_[2], uval_[3]);
+ common::databuff_printf(buf, buf_len, pos, TRACE_ID_FORMAT_V4,
+ (unsigned long long)uval_[0], (unsigned long long)uval_[1],
+ (unsigned long long)uval_[2], (unsigned long long)uval_[3]);
} else {
- common::databuff_printf(buf, buf_len, pos, TRACE_ID_FORMAT_V6, id_.bytes_no_ip_, uval_[1], uval_[2], uval_[3]);
+ common::databuff_printf(buf, buf_len, pos, TRACE_ID_FORMAT_V6,
+ id_.bytes_no_ip_,
+ (unsigned long long)uval_[1], (unsigned long long)uval_[2],
+ (unsigned long long)uval_[3]);
}
return pos;
@@ -152,9 +160,14 @@ struct ObCurTraceId
int ret = OB_SUCCESS;
int n_match = 0;
if (TRACE_ID_FORMAT_V4[0] == buf[0]) {
- n_match = sscanf(buf, TRACE_ID_FORMAT_V4, &uval_[0], &uval_[1], &uval_[2], &uval_[3]);
+ n_match = sscanf(buf, TRACE_ID_FORMAT_V4,
+ (unsigned long long*)&uval_[0], (unsigned long long*)&uval_[1],
+ (unsigned long long*)&uval_[2], (unsigned long long*)&uval_[3]);
} else if (TRACE_ID_FORMAT_V6[0] == buf[0]) {
- n_match = sscanf(buf, TRACE_ID_FORMAT_V6, &id_.bytes_no_ip_, &uval_[1], &uval_[2], &uval_[3]);
+ n_match = sscanf(buf, TRACE_ID_FORMAT_V6,
+ &id_.bytes_no_ip_,
+ (unsigned long long*)&uval_[1], (unsigned long long*)&uval_[2],
+ (unsigned long long*)&uval_[3]);
}
if (n_match != 0 && n_match != 4) {
ret = OB_INVALID_ARGUMENT;
diff --git a/deps/oblib/src/lib/queue/ob_priority_queue.h b/deps/oblib/src/lib/queue/ob_priority_queue.h
index 92d912fbc..6358cd79c 100644
--- a/deps/oblib/src/lib/queue/ob_priority_queue.h
+++ b/deps/oblib/src/lib/queue/ob_priority_queue.h
@@ -263,7 +263,7 @@ class ObPriorityQueue2
return pos;
}
- int push(ObLink* data, int priority)
+ int push(ObLink* data, int priority, bool fixed_wakeup_order = false)
{
int ret = OB_SUCCESS;
int to_push_idx = queue_num_ <= 1 ? 0 : AFFINITY_CTRL.get_tls_node() % queue_num_;
@@ -286,11 +286,11 @@ class ObPriorityQueue2
// do nothing
} else {
if (priority < HIGH_PRIOS) {
- mq_[to_push_idx]->cond.signal(1, 0);
+ mq_[to_push_idx]->cond.signal(1, 0, fixed_wakeup_order);
} else if (priority < NORMAL_PRIOS + HIGH_PRIOS) {
- mq_[to_push_idx]->cond.signal(1, 1);
+ mq_[to_push_idx]->cond.signal(1, 1, fixed_wakeup_order);
} else {
- mq_[to_push_idx]->cond.signal(1, 2);
+ mq_[to_push_idx]->cond.signal(1, 2, fixed_wakeup_order);
}
}
@@ -300,9 +300,14 @@ class ObPriorityQueue2
return ret;
}
- int pop(ObLink*& data, int64_t timeout_us)
+ void wakeup(int priority = PRIO_CNT - 1) {
+ int to_wake_idx = queue_num_ <= 1 ? 0 : AFFINITY_CTRL.get_tls_node() % queue_num_;
+ mq_[to_wake_idx]->cond.signal(1, priority);
+ }
+
+ int pop(ObLink*& data, int64_t timeout_us, int32_t index = -1)
{
- return do_pop(data, PRIO_CNT, timeout_us);
+ return do_pop(data, PRIO_CNT, timeout_us, index);
}
int pop_normal(ObLink*& data, int64_t timeout_us)
@@ -347,7 +352,7 @@ class ObPriorityQueue2
return ret;
}
- inline int do_pop(ObLink*& data, int64_t plimit, int64_t timeout_us)
+ inline int do_pop(ObLink*& data, int64_t plimit, int64_t timeout_us, int32_t index = -1)
{
int ret = OB_ENTRY_NOT_EXIST;
int to_pop_idx = queue_num_ <= 1 ? 0 : AFFINITY_CTRL.get_tls_node() % queue_num_;
@@ -358,11 +363,11 @@ class ObPriorityQueue2
COMMON_LOG(ERROR, "timeout is invalid", K(ret), K(timeout_us));
} else {
if (plimit <= HIGH_PRIOS) {
- mq_[to_pop_idx]->cond.prepare(0);
+ mq_[to_pop_idx]->cond.prepare(0, index);
} else if (plimit <= NORMAL_PRIOS + HIGH_PRIOS) {
- mq_[to_pop_idx]->cond.prepare(1);
+ mq_[to_pop_idx]->cond.prepare(1, index);
} else {
- mq_[to_pop_idx]->cond.prepare(2);
+ mq_[to_pop_idx]->cond.prepare(2, index);
}
if (OB_SUCC(try_pop(data, plimit, to_pop_idx))) {
mq_[to_pop_idx]->is_queue_idle = false;
diff --git a/deps/oblib/src/lib/restore/CMakeLists.txt b/deps/oblib/src/lib/restore/CMakeLists.txt
index c57187242..7e67f7d77 100644
--- a/deps/oblib/src/lib/restore/CMakeLists.txt
+++ b/deps/oblib/src/lib/restore/CMakeLists.txt
@@ -15,53 +15,4 @@ oblib_add_library(restore OBJECT
ob_object_storage_base.cpp
)
-oblib_add_library(s3 ob_storage_s3_base.cpp)
-if(WIN32)
- # Use Vcpkg-installed AWS SDK: vcpkg install aws-sdk-cpp[s3]; VCPKG_LIB_DIR from deps/oblib/src/CMakeLists.txt
- if(DEFINED VCPKG_LIB_DIR AND EXISTS "${VCPKG_LIB_DIR}/aws-cpp-sdk-s3.lib")
- target_link_directories(s3 PUBLIC ${VCPKG_LIB_DIR})
- target_link_libraries(s3
- PUBLIC
- ${VCPKG_LIB_DIR}/aws-cpp-sdk-s3.lib
- ${VCPKG_LIB_DIR}/aws-cpp-sdk-core.lib
- ${VCPKG_LIB_DIR}/aws-crt-cpp.lib
- ${VCPKG_LIB_DIR}/aws-c-event-stream.lib
- ${VCPKG_LIB_DIR}/aws-c-s3.lib
- ${VCPKG_LIB_DIR}/aws-c-auth.lib
- ${VCPKG_LIB_DIR}/aws-c-http.lib
- ${VCPKG_LIB_DIR}/aws-c-io.lib
- ${VCPKG_LIB_DIR}/aws-c-compression.lib
- ${VCPKG_LIB_DIR}/aws-c-cal.lib
- ${VCPKG_LIB_DIR}/aws-c-sdkutils.lib
- ${VCPKG_LIB_DIR}/aws-checksums.lib
- ${VCPKG_LIB_DIR}/aws-c-common.lib
- )
- # aws-c-mqtt may be optional for S3-only; add if present
- if(EXISTS "${VCPKG_LIB_DIR}/aws-c-mqtt.lib")
- target_link_libraries(s3 PUBLIC ${VCPKG_LIB_DIR}/aws-c-mqtt.lib)
- endif()
- endif()
-else()
- target_link_directories(s3 PUBLIC ${DEP_3RD_DIR}/usr/local/oceanbase/deps/devel/lib64 ${DEP_3RD_DIR}/usr/local/oceanbase/deps/devel/lib)
- target_link_libraries(s3
- PUBLIC
- libaws-cpp-sdk-s3.a
- libaws-cpp-sdk-core.a
- libaws-crt-cpp.a
- libaws-c-mqtt.a
- libaws-c-event-stream.a
- libaws-c-s3.a
- libaws-c-auth.a
- libaws-c-http.a
- libaws-c-io.a
- $<$>:libs2n.a>
- libaws-c-compression.a
- libaws-c-cal.a
- libaws-c-sdkutils.a
- libaws-checksums.a
- libaws-c-common.a
- )
-endif()
-target_link_libraries(s3 PUBLIC oblib_base_without_pass)
-
-target_link_libraries(restore PUBLIC oblib_base s3)
+target_link_libraries(restore PUBLIC oblib_base)
diff --git a/deps/oblib/src/lib/restore/ob_i_storage.cpp b/deps/oblib/src/lib/restore/ob_i_storage.cpp
index 5b073e401..1a8787c85 100644
--- a/deps/oblib/src/lib/restore/ob_i_storage.cpp
+++ b/deps/oblib/src/lib/restore/ob_i_storage.cpp
@@ -149,7 +149,7 @@ int get_storage_prefix_from_path(const common::ObString &uri, const char *&prefi
{
int ret = OB_SUCCESS;
if (uri.prefix_match(OB_S3_PREFIX)) {
- prefix = OB_S3_PREFIX;
+ ret = reject_s3_storage("get storage prefix", uri);
} else if (uri.prefix_match(OB_FILE_PREFIX)) {
prefix = OB_FILE_PREFIX;
} else if (uri.prefix_match(OB_AZBLOB_PREFIX)) {
@@ -372,6 +372,21 @@ int ob_set_field(const char *value, char *field, const uint32_t field_length)
return ret;
}
+int reject_s3_storage(const char *op_name)
+{
+ int ret = OB_NOT_SUPPORTED;
+ LOG_USER_ERROR(OB_NOT_SUPPORTED, "S3 storage");
+ STORAGE_LOG(WARN, "S3 storage is not supported", KR(ret), KCSTRING(op_name));
+ return ret;
+}
+
+int reject_s3_storage(const char *op_name, const common::ObString &uri)
+{
+ int ret = OB_NOT_SUPPORTED;
+ LOG_USER_ERROR(OB_NOT_SUPPORTED, "S3 storage");
+ STORAGE_LOG(WARN, "S3 storage is not supported", KR(ret), KCSTRING(op_name), K(uri));
+ return ret;
+}
/*--------------------------------ObAppendableFragmentMeta--------------------------------*/
OB_SERIALIZE_MEMBER(ObAppendableFragmentMeta, start_, end_);
@@ -825,19 +840,15 @@ int ObStoragePartInfoHandler::add_part_info(
static lib::ObMemAttr get_mem_attr_from_storage_info(const ObObjectStorageInfo *storage_info)
{
- static lib::ObMemAttr s3_attr;
static lib::ObMemAttr nfs_attr;
static lib::ObMemAttr default_attr;
- s3_attr.label_ = "S3_SDK";
nfs_attr.label_ = "NFS_SDK";
default_attr.label_ = "OBJECT_STORAGE";
lib::ObMemAttr ret_attr = default_attr;
if (OB_NOT_NULL(storage_info) && storage_info->is_valid()) {
const ObStorageType type = storage_info->get_type();
- if (OB_STORAGE_S3 == type) {
- ret_attr = s3_attr;
- } else if (OB_STORAGE_FILE == type) {
+ if (OB_STORAGE_FILE == type) {
ret_attr = nfs_attr;
}
}
diff --git a/deps/oblib/src/lib/restore/ob_i_storage.h b/deps/oblib/src/lib/restore/ob_i_storage.h
index 80be5c76d..eac2ee708 100644
--- a/deps/oblib/src/lib/restore/ob_i_storage.h
+++ b/deps/oblib/src/lib/restore/ob_i_storage.h
@@ -77,6 +77,8 @@ int check_files_map_validity(const hash::ObHashMap &files_to_
int record_failed_files_idx(const hash::ObHashMap &files_to_delete,
ObIArray &failed_files_idx);
int ob_set_field(const char *value, char *field, const uint32_t field_length);
+int reject_s3_storage(const char *op_name);
+int reject_s3_storage(const char *op_name, const common::ObString &uri);
struct ObStorageObjectMetaBase
{
diff --git a/deps/oblib/src/lib/restore/ob_storage.cpp b/deps/oblib/src/lib/restore/ob_storage.cpp
index d4a0765f9..80b2c14e6 100644
--- a/deps/oblib/src/lib/restore/ob_storage.cpp
+++ b/deps/oblib/src/lib/restore/ob_storage.cpp
@@ -56,7 +56,9 @@ void print_access_storage_log(
int validate_uri_type(const common::ObString &uri)
{
int ret = OB_SUCCESS;
- if (uri.prefix_match(OB_OSS_PREFIX)) {
+ if (uri.prefix_match(OB_S3_PREFIX)) {
+ ret = reject_s3_storage("validate uri type", uri);
+ } else if (uri.prefix_match(OB_OSS_PREFIX)) {
ret = OB_NOT_SUPPORTED;
LOG_USER_ERROR(OB_NOT_SUPPORTED, "OSS storage");
STORAGE_LOG(WARN, "OSS storage is not supported", KR(ret), KS(uri));
@@ -83,7 +85,7 @@ int get_storage_type_from_path(const common::ObString &uri, ObStorageType &type)
type = OB_STORAGE_MAX_TYPE;
if (uri.prefix_match(OB_S3_PREFIX)) {
- type = OB_STORAGE_S3;
+ ret = reject_s3_storage("get storage type", uri);
} else if (uri.prefix_match(OB_FILE_PREFIX)) {
type = OB_STORAGE_FILE;
} else if (uri.prefix_match(OB_AZBLOB_PREFIX)) {
@@ -136,8 +138,8 @@ bool is_object_storage_type(const ObStorageType &type)
bool is_adaptive_append_mode(const ObObjectStorageInfo &storage_info)
{
- const ObStorageType type = storage_info.get_type();
- return ObStorageType::OB_STORAGE_S3 == type;
+ UNUSED(storage_info);
+ return false;
}
bool is_io_error(const int result)
@@ -518,7 +520,6 @@ ObExternalIOCounterGuard::~ObExternalIOCounterGuard()
*/
ObStorageUtil::ObStorageUtil()
: file_util_(),
- s3_util_(),
util_(NULL),
storage_info_(NULL),
init_state(false),
@@ -542,7 +543,7 @@ int ObStorageUtil::open(common::ObObjectStorageInfo *storage_info)
} else if (OB_STORAGE_FILE == device_type_) {
util_ = &file_util_;
} else if (OB_STORAGE_S3 == device_type_) {
- util_ = &s3_util_;
+ ret = reject_s3_storage("open storage util");
} else {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "Invalid device type", K(ret), K_(device_type));
@@ -1865,7 +1866,6 @@ ObStorageReader::ObStorageReader()
: file_length_(-1),
reader_(NULL),
file_reader_(),
- s3_reader_(),
start_ts_(0),
has_meta_(false),
storage_info_(nullptr)
@@ -1913,7 +1913,7 @@ int ObStorageReader::open(const common::ObString &uri,
} else if (OB_STORAGE_FILE == type) {
reader_ = &file_reader_;
} else if (OB_STORAGE_S3 == type) {
- reader_ = &s3_reader_;
+ ret = reject_s3_storage("open storage reader", uri);
} else {
ret = OB_ERR_SYS;
STORAGE_LOG(ERROR, "unkown storage type", K(ret), K(uri));
@@ -2013,7 +2013,6 @@ ObStorageAdaptiveReader::ObStorageAdaptiveReader()
object_(),
reader_(NULL),
file_reader_(),
- s3_reader_(),
start_ts_(0),
storage_info_(NULL)
{
@@ -2050,7 +2049,7 @@ static int alloc_reader(ObIAllocator &allocator, const ObStorageType &type, ObIS
if (OB_STORAGE_FILE == type) {
ret = alloc_reader_type(allocator, reader);
} else if (OB_STORAGE_S3 == type) {
- ret = alloc_reader_type(allocator, reader);
+ ret = reject_s3_storage("allocate adaptive reader");
}
if (OB_FAIL(ret)) {
@@ -2093,7 +2092,7 @@ int ObStorageAdaptiveReader::open(const common::ObString &uri,
} else if (OB_STORAGE_FILE == type) {
reader_ = &file_reader_;
} else if (OB_STORAGE_S3 == type) {
- reader_ = &s3_reader_;
+ ret = reject_s3_storage("open adaptive reader", uri);
} else {
ret = OB_ERR_SYS;
STORAGE_LOG(ERROR, "unkown storage type", K(ret), K(uri));
@@ -2264,7 +2263,6 @@ int ObStorageAdaptiveReader::close()
ObStorageWriter::ObStorageWriter()
: writer_(NULL),
file_writer_(),
- s3_writer_(),
start_ts_(0),
storage_info_(nullptr)
{
@@ -2310,7 +2308,7 @@ int ObStorageWriter::open(const common::ObString &uri, common::ObObjectStorageIn
} else if (OB_STORAGE_FILE == type) {
writer_ = &file_writer_;
} else if (OB_STORAGE_S3 == type) {
- writer_ = &s3_writer_;
+ ret = reject_s3_storage("open storage writer", uri);
} else {
ret = OB_ERR_SYS;
STORAGE_LOG(ERROR, "unkown storage type", K(ret), K(uri));
@@ -2386,7 +2384,6 @@ int ObStorageWriter::close()
ObStorageAppender::ObStorageAppender()
: appender_(NULL),
file_appender_(),
- s3_appender_(),
start_ts_(0),
is_opened_(false),
storage_info_(),
@@ -2399,7 +2396,6 @@ ObStorageAppender::ObStorageAppender()
ObStorageAppender::ObStorageAppender(StorageOpenMode mode)
: appender_(NULL),
file_appender_(mode),
- s3_appender_(),
start_ts_(0),
is_opened_(false),
storage_info_(nullptr),
@@ -2449,7 +2445,7 @@ int ObStorageAppender::open(
} else if (OB_STORAGE_FILE == type_) {
appender_ = &file_appender_;
} else if (OB_STORAGE_S3 == type_) {
- appender_ = &s3_appender_;
+ ret = reject_s3_storage("open storage appender", uri);
} else {
ret = OB_ERR_SYS;
STORAGE_LOG(ERROR, "unkown storage type", K(ret), K(uri));
@@ -2550,9 +2546,7 @@ int ObStorageAppender::pwrite(const char *buf, const int64_t size, const int64_t
STORAGE_LOG(WARN, "failed to write", K(ret));
}
- // no need to adjust the function repeatable_pwrite_
- // because S3 will not return OB_OBJECT_STORAGE_PWRITE_OFFSET_NOT_MATCH
- if (OB_OBJECT_STORAGE_PWRITE_OFFSET_NOT_MATCH == ret && appender_ != &s3_appender_) {
+ if (OB_OBJECT_STORAGE_PWRITE_OFFSET_NOT_MATCH == ret) {
if (OB_FAIL(repeatable_pwrite_(buf, size, offset))) {
STORAGE_LOG(WARN, "failed to repeatable_pwrite", K(ret));
} else {
@@ -2688,7 +2682,6 @@ int ObStorageAppender::seal_for_adaptive()
ObStorageMultiPartWriter::ObStorageMultiPartWriter()
: multipart_writer_(NULL),
file_multipart_writer_(),
- s3_multipart_writer_(),
start_ts_(0),
is_opened_(false),
storage_info_(nullptr),
@@ -2734,7 +2727,7 @@ int ObStorageMultiPartWriter::open(
} else if (OB_STORAGE_FILE == type) {
multipart_writer_ = &file_multipart_writer_;
} else if (OB_STORAGE_S3 == type) {
- multipart_writer_ = &s3_multipart_writer_;
+ ret = reject_s3_storage("open multipart writer", uri);
} else {
ret = OB_ERR_SYS;
STORAGE_LOG(ERROR, "unkown storage type", K(ret), K(uri));
@@ -2907,7 +2900,6 @@ int ObStorageMultiPartWriter::close()
ObStorageParallelMultiPartWriterBase::ObStorageParallelMultiPartWriterBase()
: multipart_writer_(nullptr),
file_multipart_writer_(),
- s3_multipart_writer_(),
start_ts_(0),
is_opened_(false),
storage_info_(nullptr)
@@ -2955,7 +2947,7 @@ int ObStorageParallelMultiPartWriterBase::open(
} else if (OB_STORAGE_FILE == type) {
multipart_writer_ = &file_multipart_writer_;
} else if (OB_STORAGE_S3 == type) {
- multipart_writer_ = &s3_multipart_writer_;
+ ret = reject_s3_storage("open parallel multipart writer", uri);
} else {
ret = OB_ERR_SYS;
STORAGE_LOG(ERROR, "unkown storage type", K(ret), K(uri), K(type));
diff --git a/deps/oblib/src/lib/restore/ob_storage.h b/deps/oblib/src/lib/restore/ob_storage.h
index 419c68ab6..64d3a9bed 100644
--- a/deps/oblib/src/lib/restore/ob_storage.h
+++ b/deps/oblib/src/lib/restore/ob_storage.h
@@ -18,7 +18,6 @@
#define SRC_LIBRARY_SRC_LIB_RESTORE_OB_STORAGE_H_
#include "ob_i_storage.h"
#include "ob_storage_file.h"
-#include "ob_storage_s3_base.h"
#include "common/storage/ob_io_device.h"
namespace oceanbase
@@ -355,7 +354,6 @@ class ObStorageUtil
int head_object_meta_(const ObString &uri, ObStorageObjectMetaBase &obj_meta);
ObStorageFileUtil file_util_;
- ObStorageS3Util s3_util_;
ObIStorageUtil* util_;
common::ObObjectStorageInfo* storage_info_;
bool init_state;
@@ -425,7 +423,6 @@ class ObStorageReader : public ObStorageAccesser
int64_t file_length_;
ObIStorageReader *reader_;
ObStorageFileReader file_reader_;
- ObStorageS3Reader s3_reader_;
int64_t start_ts_;
char uri_[OB_MAX_URI_LENGTH];
bool has_meta_;
@@ -453,7 +450,6 @@ class ObStorageAdaptiveReader : public ObStorageAccesser
ObString object_;
ObIStorageReader *reader_;
ObStorageFileReader file_reader_;
- ObStorageS3Reader s3_reader_;
int64_t start_ts_;
char uri_[OB_MAX_URI_LENGTH];
ObObjectStorageInfo *storage_info_;
@@ -471,7 +467,6 @@ class ObStorageWriter : public ObStorageAccesser
protected:
ObIStorageWriter *writer_;
ObStorageFileSingleWriter file_writer_;
- ObStorageS3Writer s3_writer_;
int64_t start_ts_;
char uri_[OB_MAX_URI_LENGTH];
ObObjectStorageInfo *storage_info_;
@@ -504,7 +499,6 @@ class ObStorageAppender : public ObStorageAccesser
private:
ObIStorageWriter *appender_;
ObStorageFileAppender file_appender_;
- ObStorageS3AppendWriter s3_appender_;
int64_t start_ts_;
bool is_opened_;
char uri_[OB_MAX_URI_LENGTH];
@@ -535,7 +529,6 @@ class ObStorageMultiPartWriter : public ObStorageAccesser
protected:
ObIStorageMultiPartWriter *multipart_writer_;
ObStorageFileMultiPartWriter file_multipart_writer_;
- ObStorageS3MultiPartWriter s3_multipart_writer_;
int64_t start_ts_;
bool is_opened_;
char uri_[OB_MAX_URI_LENGTH];
@@ -556,7 +549,6 @@ class ObStorageParallelMultiPartWriterBase
protected:
ObIStorageParallelMultipartWriter *multipart_writer_;
ObStorageParallelFileMultiPartWriter file_multipart_writer_;
- ObStorageParallelS3MultiPartWriter s3_multipart_writer_;
int64_t start_ts_;
bool is_opened_;
char uri_[OB_MAX_URI_LENGTH];
diff --git a/deps/oblib/src/lib/restore/ob_storage_s3_base.cpp b/deps/oblib/src/lib/restore/ob_storage_s3_base.cpp
deleted file mode 100644
index 0f64eccaa..000000000
--- a/deps/oblib/src/lib/restore/ob_storage_s3_base.cpp
+++ /dev/null
@@ -1,2766 +0,0 @@
-/*
- * Copyright (c) 2025 OceanBase.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "lib/restore/ob_storage.h"
-#include "ob_storage_s3_base.h"
-namespace oceanbase
-{
-namespace common
-{
-using namespace Aws::S3;
-using namespace Aws::Client;
-using namespace Aws::Utils;
-
-/*--------------------------------ObS3MemoryManager-------------------------*/
-
-int ObS3MemoryManager::init()
-{
- int ret = OB_SUCCESS;
- if (OB_FAIL(allocator_.init(DEFAULT_BLOCK_SIZE, mem_limiter_, attr_))) {
- OB_LOG(WARN, "init allocator error", K(ret));
- } else {
- allocator_.set_nway(N_WAY);
- }
- return ret;
-}
-
-void *ObS3MemoryManager::AllocateMemory(std::size_t blockSize,
- std::size_t alignment,
- const char *allocationTag)
-{
- // When memory allocation fails, S3 SDK calls abort, causing a program crash.
- // Replace memalign with ObVSliceAlloc to get lower CPU usage and faster memory allocation efficiency
- // ObVSliceAlloc mainly optimizes small block allocation
- // and directly passes large block(OB_MALLOC_BIG_BLOCK_SIZE 2MB) allocation to the lower-level allocator (ob_malloc)
- // thus when ob_malloc fails to allocate memory and the duration exceeds 1s, the program will crash
- // in addition, memory alignment is performed before allocating memory
- int ret = OB_SUCCESS;
- UNUSED(allocationTag);
- void *ptr = nullptr;
- do {
- ptr = allocator_.alloc_align(blockSize, alignment);
- if (OB_ISNULL(ptr)) {
- ::usleep(10000); // 10ms
- if (TC_REACH_TIME_INTERVAL(10 * 1000 * 1000)) {
- ret = OB_ALLOCATE_MEMORY_FAILED;
- OB_LOG(ERROR, "ObVSliceAlloc failed to allocate memory",
- K(blockSize), K(alignment), K(allocationTag));
- }
- }
- } while (OB_ISNULL(ptr));
- return ptr;
-}
-
-void ObS3MemoryManager::FreeMemory(void *memoryPtr)
-{
- allocator_.free_align(memoryPtr);
- memoryPtr = nullptr;
-}
-
-/*--------------------------------ObS3Logger--------------------------------*/
-Logging::LogLevel ObS3Logger::GetLogLevel(void) const
-{
- Logging::LogLevel log_level = Logging::LogLevel::Info;
- int32_t ob_log_level = OB_LOGGER.get_log_level();
- switch (ob_log_level) {
- case OB_LOG_LEVEL_INFO:
- break;
- case OB_LOG_LEVEL_ERROR:
- log_level = Logging::LogLevel::Error;
- break;
- case OB_LOG_LEVEL_WARN:
- log_level = Logging::LogLevel::Warn;
- break;
- case OB_LOG_LEVEL_TRACE:
- log_level = Logging::LogLevel::Debug;
- break;
- case OB_LOG_LEVEL_DEBUG:
- log_level = Logging::LogLevel::Trace;
- break;
- default:
- break;
- }
- return log_level;
-}
-
-void ObS3Logger::Log(Logging::LogLevel logLevel, const char* tag, const char* formatStr, ...)
-{
- int ret = OB_SUCCESS;
-
- const int64_t buf_len = 4096;
- char arg_buf[buf_len] = {0};
- va_list args;
- va_start(args, formatStr);
- int psize = vsnprintf(arg_buf, buf_len - 1, formatStr, args);
- va_end(args);
-
- if (psize > 0) {
- const char *new_format = "[S3] module=%s, %s";
- switch (logLevel) {
- case Logging::LogLevel::Fatal:
- case Logging::LogLevel::Error:
- case Logging::LogLevel::Warn: {
- if (OB_NOT_NULL(STRSTR(arg_buf, "HTTP response code: 404"))) {
- // skip NO_SUCH_KEY error
- } else {
- ret = OB_OBJECT_STORAGE_IO_ERROR;
- _OB_LOG(WARN, new_format, tag, arg_buf);
- }
- break;
- }
- case Logging::LogLevel::Info:
- _OB_LOG(INFO, new_format, tag, arg_buf);
- break;
- // NOTICE: the s3 Debug and Trace level keeps the reverse order with them in ob.
- case Logging::LogLevel::Trace:
- _OB_LOG(DEBUG, new_format, tag, arg_buf);
- break;
- case Logging::LogLevel::Debug:
- _OB_LOG(TRACE, new_format, tag, arg_buf);
- break;
- default:
- _OB_LOG(WARN, new_format, tag, arg_buf);
- break;
- }
- }
-}
-
-void ObS3Logger::vaLog(Logging::LogLevel logLevel, const char* tag, const char* formatStr, va_list args)
-{
- int ret = OB_SUCCESS;
-
- const int64_t buf_len = 4096;
- char arg_buf[buf_len] = {0};
- int psize = vsnprintf(arg_buf, buf_len - 1, formatStr, args);
-
- if (psize > 0) {
- const char *new_format = "[S3] module=%s, %s";
- switch (logLevel) {
- case Logging::LogLevel::Fatal:
- case Logging::LogLevel::Error:
- case Logging::LogLevel::Warn: {
- if (OB_NOT_NULL(STRSTR(arg_buf, "HTTP response code: 404"))) {
- } else {
- ret = OB_OBJECT_STORAGE_IO_ERROR;
- _OB_LOG(WARN, new_format, tag, arg_buf);
- }
- break;
- }
- case Logging::LogLevel::Info:
- _OB_LOG(INFO, new_format, tag, arg_buf);
- break;
- case Logging::LogLevel::Trace:
- _OB_LOG(DEBUG, new_format, tag, arg_buf);
- break;
- case Logging::LogLevel::Debug:
- _OB_LOG(TRACE, new_format, tag, arg_buf);
- break;
- default:
- _OB_LOG(WARN, new_format, tag, arg_buf);
- break;
- }
- }
-}
-
-void ObS3Logger::LogStream(Logging::LogLevel logLevel, const char* tag, const Aws::OStringStream &messageStream)
-{
- Log(logLevel, tag, "msg=%s", messageStream.str().c_str());
-}
-
-/*--------------------------------ObS3Client--------------------------------*/
-// max allowed idle duration for a s3 client: 20min
-static int64_t MAX_S3_CLIENT_IDLE_DURATION_US = 20LL * 600LL * 1000LL * 1000LL;
-
-int set_max_s3_client_idle_duration_us(const int64_t duration_us)
-{
- int ret = OB_SUCCESS;
- if (OB_UNLIKELY(duration_us <= 0)) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "invalid arg", K(ret), K(duration_us));
- } else {
- MAX_S3_CLIENT_IDLE_DURATION_US = duration_us;
- }
- return ret;
-}
-
-int64_t get_max_s3_client_idle_duration_us()
-{
- return MAX_S3_CLIENT_IDLE_DURATION_US;
-}
-
-ObS3Client::ObS3Client()
- : lock_(ObLatchIds::OBJECT_DEVICE_LOCK),
- is_inited_(false),
- stopped_(false),
- ref_cnt_(0),
- last_modified_ts_(0),
- client_(nullptr)
-{
-}
-
-ObS3Client::~ObS3Client()
-{
- destroy();
-}
-
-// Disable the internal retry mechanism within the S3 SDK
-class ObStorageS3DisabledRetryStrategy : public Aws::Client::DefaultRetryStrategy
-{
-public:
- ObStorageS3DisabledRetryStrategy() : DefaultRetryStrategy() {}
- virtual ~ObStorageS3DisabledRetryStrategy() {}
-
- virtual bool ShouldRetry(const AWSError& error, long attemptedRetries) const override
- {
- return false;
- }
-};
-
-int ObS3Client::init_s3_client_configuration_(const ObS3Account &account,
- S3ClientConfiguration &config)
-{
- int ret = OB_SUCCESS;
- if (OB_UNLIKELY(!account.is_valid())) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "S3 account not valid", K(ret));
- } else {
- if (OB_NOT_NULL(account.region_) && account.region_[0] != '\0') {
- config.region = account.region_;
- }
- config.scheme = Aws::Http::Scheme::HTTP; // if change to HTTPS, be careful about checksum logic.
- config.verifySSL = true;
- config.connectTimeoutMs = S3_CONNECT_TIMEOUT_MS;
- config.requestTimeoutMs = S3_REQUEST_TIMEOUT_MS;
- config.maxConnections = MAX_S3_CONNECTIONS_PER_CLIENT;
- config.payloadSigningPolicy = Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never;
- config.endpointOverride = account.endpoint_;
- config.executor = nullptr;
- if (account.addressing_model_ == ObStorageAddressingModel::OB_PATH_STYLE) {
- config.useVirtualAddressing = false;
- }
-
- // Default maxRetries is 10
- std::shared_ptr retryStrategy =
- Aws::MakeShared(S3_SDK);
- config.retryStrategy = retryStrategy;
- }
- return ret;
-}
-
-int ObS3Client::init(const ObS3Account &account)
-{
- int ret = OB_SUCCESS;
- void *client_buf = nullptr;
- // Disables IMDS to prevent auto-region detection during construction.
- ClientConfigurationInitValues init_values;
- init_values.shouldDisableIMDS = true;
- S3ClientConfiguration config(init_values);
- // Re-enables IMDS access for subsequent operations if needed
- config.disableIMDS = false;
- Aws::Auth::AWSCredentials credentials;
- const char *sts_data = account.sts_token_.get_data();
- if (OB_NOT_NULL(sts_data)) {
- credentials = Aws::Auth::AWSCredentials(account.access_id_, account.secret_key_, sts_data);
- } else {
- credentials = Aws::Auth::AWSCredentials(account.access_id_, account.secret_key_);
- }
-
-
- SpinWLockGuard guard(lock_);
- if (OB_UNLIKELY(is_inited_)) {
- ret = OB_INIT_TWICE;
- OB_LOG(WARN, "ObS3Client init twice", K(ret));
- } else if (OB_UNLIKELY(!account.is_valid())) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "S3 account not valid", K(ret));
- } else if (OB_FAIL(init_s3_client_configuration_(account, config))) {
- OB_LOG(WARN, "failed to init s3 client config", K(ret), K(account));
- } else if (OB_ISNULL(client_buf = ob_malloc(sizeof(Aws::S3::S3Client), OB_STORAGE_S3_ALLOCATOR))) {
- ret = OB_ALLOCATE_MEMORY_FAILED;
- OB_LOG(WARN, "failed to alloc buf for aws s3 client", K(ret));
- } else {
- try {
- client_ = new(client_buf) Aws::S3::S3Client(credentials,
- Aws::MakeShared(S3_SDK), config);
- } catch (const std::exception &e) {
- ret = OB_OBJECT_STORAGE_IO_ERROR;
- OB_LOG(WARN, "caught exception when initing ObS3Client",
- K(ret), K(e.what()), KP(this), K(*this));
- } catch (...) {
- ret = OB_OBJECT_STORAGE_IO_ERROR;
- OB_LOG(WARN, "caught unknown exception when initing ObS3Client", K(ret), KP(this), K(*this));
- }
-
- if (OB_SUCC(ret)) {
- last_modified_ts_ = ObTimeUtility::current_time();
- is_inited_ = true;
- } else {
- ob_free(client_buf);
- }
- }
- return ret;
-}
-
-void ObS3Client::destroy()
-{
- SpinWLockGuard guard(lock_);
- is_inited_ = false;
- stopped_ = false;
- ref_cnt_ = 0;
- last_modified_ts_ = 0;
- if (OB_NOT_NULL(client_)) {
- client_->~S3Client();
- ob_free(client_);
- client_ = NULL;
- }
-}
-
-bool ObS3Client::is_stopped() const
-{
- SpinRLockGuard guard(lock_);
- return stopped_;
-}
-
-bool ObS3Client::try_stop(const int64_t timeout)
-{
- bool is_stopped = true;
- const int64_t abs_timeout_us = ObTimeUtility::current_time() + timeout;
- if (OB_SUCCESS == lock_.wrlock(abs_timeout_us)) {
- if (is_inited_) {
- const int64_t cur_time_us = ObTimeUtility::current_time();
- if (ref_cnt_ <= 0
- && cur_time_us - last_modified_ts_ >= get_max_s3_client_idle_duration_us()) {
- stopped_ = true;
- } else {
- is_stopped = false;
- }
- }
- lock_.unlock();
- } else {
- is_stopped = false;
- }
- return is_stopped;
-}
-
-void ObS3Client::stop()
-{
- SpinWLockGuard guard(lock_);
- stopped_ = true;
-}
-
-void ObS3Client::increase()
-{
- SpinWLockGuard guard(lock_);
- ref_cnt_++;
- last_modified_ts_ = ObTimeUtility::current_time();
-}
-
-void ObS3Client::release()
-{
- SpinWLockGuard guard(lock_);
- ref_cnt_--;
- last_modified_ts_ = ObTimeUtility::current_time();
-}
-
-int ObS3Client::check_status()
-{
- int ret = OB_SUCCESS;
- SpinRLockGuard guard(lock_);
- if (!is_inited_) {
- ret = OB_NOT_INIT;
- OB_LOG(WARN, "ObS3Client not init", K(ret));
- } else if (stopped_) {
- ret = OB_IN_STOP_STATE;
- OB_LOG(WARN, "ObS3Client has been stopped", K(ret));
- } else if (OB_ISNULL(client_)) {
- ret = OB_OBJECT_STORAGE_IO_ERROR;
- OB_LOG(WARN, "client is NULL in ObS3Client", K(ret), KP(client_));
- }
- return ret;
-}
-
-template
-int ObS3Client::do_s3_operation_(S3OperationFunc s3_op_func,
- const RequestType &request, OutcomeType &outcome,
- const int64_t retry_timeout_us)
-{
- int ret = OB_SUCCESS;
- if (OB_FAIL(check_status())) {
- OB_LOG(WARN, "ObS3Client is not in running state", K(ret));
- } else {
- ObStorageS3RetryStrategy strategy(retry_timeout_us);
- outcome = execute_until_timeout(strategy, std::mem_fn(s3_op_func), client_, request);
- last_modified_ts_ = ObTimeUtility::current_time();
- }
- return ret;
-}
-
-int ObS3Client::head_object(const Aws::S3::Model::HeadObjectRequest &request,
- Aws::S3::Model::HeadObjectOutcome &outcome)
-{
- S3OperationFunc
- s3_op_func = &S3Client::HeadObject;
- return do_s3_operation_(s3_op_func, request, outcome);
-}
-
-int ObS3Client::put_object(const Model::PutObjectRequest &request,
- Model::PutObjectOutcome &outcome)
-{
- S3OperationFunc
- s3_op_func = &S3Client::PutObject;
- return do_s3_operation_(s3_op_func, request, outcome);
-}
-
-int ObS3Client::get_object(const Model::GetObjectRequest &request,
- Model::GetObjectOutcome &outcome)
-{
- S3OperationFunc
- s3_op_func = &S3Client::GetObject;
- return do_s3_operation_(s3_op_func, request, outcome);
-}
-
-int ObS3Client::delete_object(const Model::DeleteObjectRequest &request,
- Model::DeleteObjectOutcome &outcome)
-{
- S3OperationFunc
- s3_op_func = &S3Client::DeleteObject;
- return do_s3_operation_(s3_op_func, request, outcome);
-}
-
-int ObS3Client::delete_objects(const Model::DeleteObjectsRequest &request,
- Model::DeleteObjectsOutcome &outcome)
-{
- S3OperationFunc
- s3_op_func = &S3Client::DeleteObjects;
- return do_s3_operation_(s3_op_func, request, outcome);
-}
-
-int ObS3Client::put_object_tagging(const Model::PutObjectTaggingRequest &request,
- Model::PutObjectTaggingOutcome &outcome)
-{
- S3OperationFunc
- s3_op_func = &S3Client::PutObjectTagging;
- return do_s3_operation_(s3_op_func, request, outcome);
-}
-
-
-int ObS3Client::list_objects(const Aws::S3::Model::ListObjectsRequest &request,
- Model::ListObjectsOutcome &outcome)
-{
- S3OperationFunc
- s3_op_func = &S3Client::ListObjects;
- return do_s3_operation_(s3_op_func, request, outcome);
-}
-
-int ObS3Client::get_object_tagging(const Model::GetObjectTaggingRequest &request,
- Model::GetObjectTaggingOutcome &outcome)
-{
- S3OperationFunc
- s3_op_func = &S3Client::GetObjectTagging;
- return do_s3_operation_(s3_op_func, request, outcome);
-}
-
-int ObS3Client::create_multipart_upload(const Model::CreateMultipartUploadRequest &request,
- Model::CreateMultipartUploadOutcome &outcome)
-{
- S3OperationFunc
- s3_op_func = &S3Client::CreateMultipartUpload;
- return do_s3_operation_(s3_op_func, request, outcome);
-}
-
-
-int ObS3Client::complete_multipart_upload(const Model::CompleteMultipartUploadRequest &request,
- Model::CompleteMultipartUploadOutcome &outcome)
-{
- S3OperationFunc
- s3_op_func = &S3Client::CompleteMultipartUpload;
- return do_s3_operation_(s3_op_func, request, outcome, DO_NOT_RETRY/*retry_timeout_us*/);
-}
-
-int ObS3Client::abort_multipart_upload(const Model::AbortMultipartUploadRequest &request,
- Model::AbortMultipartUploadOutcome &outcome)
-{
- S3OperationFunc
- s3_op_func = &S3Client::AbortMultipartUpload;
- return do_s3_operation_(s3_op_func, request, outcome);
-}
-
-int ObS3Client::upload_part(const Model::UploadPartRequest &request,
- Model::UploadPartOutcome &outcome)
-{
- S3OperationFunc
- s3_op_func = &S3Client::UploadPart;
- return do_s3_operation_(s3_op_func, request, outcome);
-}
-
-int ObS3Client::list_multipart_uploads(const Model::ListMultipartUploadsRequest &request,
- Model::ListMultipartUploadsOutcome &outcome)
-{
- S3OperationFunc
- s3_op_func = &S3Client::ListMultipartUploads;
- return do_s3_operation_(s3_op_func, request, outcome);
-}
-
-/*--------------------------------GLOBAL--------------------------------*/
-int init_s3_env()
-{
- return ObS3Env::get_instance().init();
-}
-
-void fin_s3_env()
-{
- ObS3Env::get_instance().stop();
-
- // wait doing io finish before destroy s3 env.
- const int64_t start_time = ObTimeUtility::current_time();
- const int64_t timeout = ObExternalIOCounter::FLYING_IO_WAIT_TIMEOUT;
- int64_t flying_io_cnt = ObExternalIOCounter::get_flying_io_cnt();
- while(0 < flying_io_cnt) {
- const int64_t end_time = ObTimeUtility::current_time();
- if (end_time - start_time > timeout) {
- int ret = OB_TIMEOUT;
- OB_LOG(WARN, "force fin_s3_env", K(ret), K(flying_io_cnt));
- break;
- }
- ob_usleep(100 * 1000L); // 100ms
- flying_io_cnt = ObExternalIOCounter::get_flying_io_cnt();
- }
-
- ObS3Env::get_instance().destroy();
-}
-
-std::shared_ptr s3_clientBootstrap_create_fn()
-{
- return nullptr;
-};
-
-std::shared_ptr s3_logger_create_fn()
-{
- return std::make_shared();
-}
-
-ObS3Env::ObS3Env()
- : lock_(ObLatchIds::OBJECT_DEVICE_LOCK),
- is_inited_(false), s3_mem_manger_(),
- aws_options_(), s3_client_map_()
-{
- aws_options_.ioOptions.clientBootstrap_create_fn = s3_clientBootstrap_create_fn;
- ObS3Logger s3_logger;
- aws_options_.loggingOptions.logLevel = s3_logger.GetLogLevel();
- aws_options_.loggingOptions.logger_create_fn = s3_logger_create_fn;
- // Adhere to RFC 3986, supporting encoding of reserved characters
- // such as '-', '_', '.', '$', '@', etc.
- // Thus, it alleviates the issue of inconsistent server behavior when accessing
- // COS using S3 SDK.
- aws_options_.httpOptions.compliantRfc3986Encoding = true;
-}
-
-ObS3Env &ObS3Env::get_instance()
-{
- static ObS3Env s3_env_instance;
- return s3_env_instance;
-}
-
-int ObS3Env::init()
-{
- int ret = OB_SUCCESS;
- SpinWLockGuard guard(lock_);
- if (OB_UNLIKELY(is_inited_)) {
- ret = OB_INIT_TWICE;
- OB_LOG(WARN, "S3 env init twice", K(ret));
- } else if (OB_FAIL(s3_client_map_.create(MAX_S3_CLIENT_NUM, OB_STORAGE_S3_ALLOCATOR))) {
- OB_LOG(WARN, "failed to create s3 client map", K(ret));
- } else if (OB_FAIL(s3_mem_manger_.init())) {
- OB_LOG(WARN, "failed to init S3 Memory Manger", K(ret));
- } else {
- aws_options_.memoryManagementOptions.memoryManager = &s3_mem_manger_;
- Aws::InitAPI(aws_options_);
- is_inited_ = true;
- // TO make sure Aws::ShutdownAPI is called before OPENSSL_cleanup
- // fin_s3_env is called when the singleton ObDeviceManager is destructed.
- // Its destructor is called after the cleanup of the openssl environment.
- // So, when executing fin_s3_env, openssl has already been cleaned up.
- // However, fin_s3_env invokes Aws::ShutDownAPI, which requires calling functions from openssl.
- // This can lead to deadlocks or segmentation faults.
- // To avoid this, we register fin_s3_env in advance,
- // ensuring that its invocation occurs before the cleanup of openssl
- atexit(fin_s3_env);
- }
- return ret;
-}
-
-void ObS3Env::destroy()
-{
- SpinWLockGuard guard(lock_);
- if (OB_LIKELY(is_inited_)) {
- hash::ObHashMap::iterator iter = s3_client_map_.begin();
- while (iter != s3_client_map_.end()) {
- if (OB_NOT_NULL(iter->second)) {
- // force destroy s3 client
- ObS3Client *s3_client = iter->second;
- s3_client->~ObS3Client();
- ob_free(iter->second);
- }
- iter++;
- }
- s3_client_map_.destroy();
- Aws::ShutdownAPI(aws_options_);
- is_inited_ = false;
- }
-}
-
-int ObS3Env::get_or_create_s3_client(const ObS3Account &account, ObS3Client *&client)
-{
- int ret = OB_SUCCESS;
- const int64_t key = account.hash();
- client = nullptr;
- SpinWLockGuard guard(lock_);
- if (OB_UNLIKELY(!is_inited_)) {
- ret = OB_NOT_INIT;
- OB_LOG(WARN, "ob s3 env not init", K(ret));
- } else if (OB_UNLIKELY(!account.is_valid())) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "S3 account not valid", K(ret));
- } else if (REACH_TIME_INTERVAL(get_max_s3_client_idle_duration_us())) {
- int tmp_ret = OB_SUCCESS;
- if (OB_TMP_FAIL(clean_s3_client_map_())) {
- OB_LOG(WARN, "failed to clean s3 client map", K(tmp_ret), K(s3_client_map_.size()));
- }
- }
-
- if (FAILEDx(s3_client_map_.get_refactored(key, client))) {
- if (ret == OB_HASH_NOT_EXIST) {
- ret = OB_SUCCESS;
- void *client_buf = nullptr;
- if (OB_ISNULL(client_buf = ob_malloc(sizeof(ObS3Client), OB_STORAGE_S3_ALLOCATOR))) {
- ret = OB_ALLOCATE_MEMORY_FAILED;
- OB_LOG(WARN, "failed to alloc buf for ob s3 client", K(ret));
- } else {
- client = new(client_buf) ObS3Client();
- if (OB_FAIL(client->init(account))) {
- OB_LOG(WARN, "failed to init ObS3Client", K(ret), K(account));
- } else if (OB_FAIL(s3_client_map_.set_refactored(key, client))) {
- OB_LOG(WARN, "failed to insert into s3 client map", K(ret), K(account));
- } else {
- OB_LOG(DEBUG, "succeed create new s3 client", K(account), K(s3_client_map_.size()));
- }
-
- if (OB_FAIL(ret)) {
- if (OB_NOT_NULL(client)) {
- client->~ObS3Client();
- client = nullptr;
- }
- if (OB_NOT_NULL(client_buf)) {
- ob_free(client_buf);
- client_buf = nullptr;
- }
- }
- }
- } else {
- OB_LOG(WARN, "failed to get s3 client from map", K(ret), K(account));
- }
- } else if (OB_UNLIKELY(client->is_stopped())) {
- ret = OB_ERR_UNEXPECTED;
- OB_LOG(WARN, "an stopped client remained in s3 client map", K(ret), KP(client));
- }
-
- if (OB_SUCC(ret)) {
- client->increase();
- }
- return ret;
-}
-
-int ObS3Env::clean_s3_client_map_()
-{
- int ret = OB_SUCCESS;
- hash::ObHashMap::iterator iter = s3_client_map_.begin();
- ObArray s3_clients_to_clean;
- while (OB_SUCC(ret) && iter != s3_client_map_.end()) {
- if (OB_NOT_NULL(iter->second)) {
- ObS3Client *s3_client = iter->second;
- if (s3_client->try_stop()) {
- if (OB_FAIL(s3_clients_to_clean.push_back(iter->first))) {
- OB_LOG(WARN, "failed to push back into s3_clients_to_clean",
- K(ret), K(iter->first), KP(s3_client));
- } else {
- s3_client->~ObS3Client();
- ob_free(iter->second);
- iter->second = NULL;
- }
- }
- } else if (OB_FAIL(s3_clients_to_clean.push_back(iter->first))) {
- OB_LOG(WARN, "failed to push back into s3_clients_to_clean",
- K(ret), K(iter->first), KP(iter->second));
- }
- iter++;
- }
-
- for (int64_t i = 0; OB_SUCC(ret) && i < s3_clients_to_clean.count(); i++) {
- if (OB_FAIL(s3_client_map_.erase_refactored(s3_clients_to_clean[i]))) {
- OB_LOG(WARN, "failed to clean s3 client map", K(ret));
- }
- }
-
- return ret;
-}
-
-void ObS3Env::stop()
-{
- SpinRLockGuard guard(lock_);
- hash::ObHashMap::iterator iter = s3_client_map_.begin();
- while (iter != s3_client_map_.end()) {
- if (OB_NOT_NULL(iter->second)) {
- ObS3Client *s3_client = iter->second;
- s3_client->stop();
- }
- iter++;
- }
-}
-
-/*--------------------------------ERROR HANDLE--------------------------------*/
-
-template
-static bool is_gcs_destination(const OutcomeType &outcome)
-{
- // The x-guploader-uploadid header is a unique identifier provided in Cloud Storage responses.
- return outcome.GetError().ResponseHeaderExists("x-guploader-uploadid");
-}
-
-const int S3_BAD_REQUEST = 400;
-const int S3_PERMISSION_DENIED = 403;
-const int S3_ITEM_NOT_EXIST = 404;
-const int S3_SLOW_DOWN = 503;
-
-static void convert_http_error(const Aws::S3::S3Error &s3_err, int &ob_errcode)
-{
- const int http_code = static_cast(s3_err.GetResponseCode());
- const Aws::String &exception = s3_err.GetExceptionName();
- const Aws::String &err_msg = s3_err.GetMessage();
-
- switch (http_code) {
- case S3_BAD_REQUEST: {
- if (exception == "InvalidRequest" && err_msg.find("x-amz-checksum") != std::string::npos) {
- ob_errcode = OB_OBJECT_STORAGE_CHECKSUM_ERROR;
- } else if (exception == "InvalidRequest" && err_msg.find("Appid, Bucket") != std::string::npos) {
- ob_errcode = OB_INVALID_OBJECT_STORAGE_ENDPOINT;
- } else if (err_msg.find("region") != std::string::npos
- && err_msg.find("is wrong; expecting") != std::string::npos) {
- ob_errcode = OB_S3_REGION_MISMATCH;
- } else if (exception == "InvalidRegionName" || exception == "InvalidBucketName") {
- // When accessing COS and OSS using the S3 SDK, if the endpoint parameter is incorrect,
- // S3 does not capture the exception. Instead, we need to set ob_errorcode individually
- // based on the HTTP response code and the exception name.
- ob_errcode = OB_INVALID_OBJECT_STORAGE_ENDPOINT;
- }
- // S3 reports different errors for object names that exceed the limited length
- // put: KeyTooLongError, delete: InvalidObjectName, list: InvalidArgument(OBS)
- else if (err_msg.find("KeyTooLongError") != std::string::npos
- || err_msg.find("InvalidObjectName") != std::string::npos
- || err_msg.find("InvalidArgument") != std::string::npos) {
- ob_errcode = OB_INVALID_ARGUMENT;
- } else if (exception == "InvalidURI") {
- ob_errcode = OB_INVALID_ARGUMENT;
- } else {
- ob_errcode = OB_OBJECT_STORAGE_IO_ERROR;
- }
- break;
- }
- case S3_ITEM_NOT_EXIST: {
- if (exception == "NoSuchTagSet") {
- // When using the getObjectTagging function to access an OBS object that does not have tags,
- // a NoSuchTagSet error will be returned.
- ob_errcode = OB_ITEM_NOT_SETTED;
- } else {
- ob_errcode = OB_OBJECT_NOT_EXIST;
- }
- break;
- }
- case S3_PERMISSION_DENIED: {
- if (exception == "InvalidAccessKeyId") {
- ob_errcode = OB_OBJECT_STORAGE_PERMISSION_DENIED;
- } else {
- ob_errcode = OB_OBJECT_STORAGE_IO_ERROR;
- }
- break;
- }
- case S3_SLOW_DOWN: {
- ob_errcode = OB_IO_LIMIT;
- break;
- }
- default: {
- if (err_msg.find("curlCode: 28") != std::string::npos) {
- ob_errcode = OB_TIMEOUT;
- } else {
- ob_errcode = OB_OBJECT_STORAGE_IO_ERROR;
- }
- break;
- }
- }
-}
-
-static void convert_io_error(const Aws::S3::S3Error &s3_err, int &ob_errcode)
-{
- switch (s3_err.GetErrorType()) {
- case Aws::S3::S3Errors::NO_SUCH_KEY: {
- ob_errcode = OB_OBJECT_NOT_EXIST;
- break;
- }
- case Aws::S3::S3Errors::RESOURCE_NOT_FOUND: {
- ob_errcode = OB_OBJECT_NOT_EXIST;
- break;
- }
- case Aws::S3::S3Errors::SLOW_DOWN: {
- ob_errcode = OB_IO_LIMIT;
- break;
- }
- case Aws::S3::S3Errors::ACCESS_DENIED: {
- ob_errcode = OB_OBJECT_STORAGE_PERMISSION_DENIED;
- break;
- }
- case Aws::S3::S3Errors::NO_SUCH_BUCKET: {
- ob_errcode = OB_INVALID_OBJECT_STORAGE_ENDPOINT;
- break;
- }
- default: {
- convert_http_error(s3_err, ob_errcode);
- break;
- }
- }
-}
-
-template
-static void log_s3_status(OutcomeType &outcome, int &ob_errcode)
-{
- const char *request_id = outcome.GetResult().GetRequestId().c_str();
- if (outcome.GetResult().GetRequestId().empty()) {
- const Aws::Http::HeaderValueCollection &headers = outcome.GetError().GetResponseHeaders();
- Aws::Http::HeaderValueCollection::const_iterator it = headers.find("x-amz-request-id");
- if (it != headers.end()) {
- request_id = it->second.c_str();
- }
- }
- const int code = static_cast(outcome.GetError().GetResponseCode());
- const char *exception = outcome.GetError().GetExceptionName().c_str();
- const char *err_msg = outcome.GetError().GetMessage().c_str();
- if (OB_OBJECT_NOT_EXIST != ob_errcode || code / 100 != 2) {
- // force printing log
- allow_next_syslog();
- }
- if (OB_OBJECT_STORAGE_CHECKSUM_ERROR == ob_errcode) {
- OB_LOG_RET(ERROR, ob_errcode, "S3 info", K(request_id), K(code), K(exception), K(err_msg));
- // checksum error are offten caused by network issues, so we convert it to
- // io error to make it easier for user to retry.
- ob_errcode = OB_OBJECT_STORAGE_IO_ERROR;
- } else {
- OB_LOG_RET(WARN, ob_errcode, "S3 info", K(request_id), K(code), K(exception), K(err_msg));
- }
-}
-
-template
-static void handle_s3_outcome(OutcomeType &outcome, int &ob_errcode)
-{
- const Aws::S3::S3Error &s3_err = outcome.GetError();
- convert_io_error(s3_err, ob_errcode);
- log_s3_status(outcome, ob_errcode);
-}
-
-/*--------------------------------Checksum Util--------------------------------*/
-template
-static int set_request_checkusum_algorithm(RequestType &request,
- const ObStorageChecksumType checksum_type)
-{
- int ret = OB_SUCCESS;
- if (OB_UNLIKELY(!is_s3_supported_checksum(checksum_type))) {
- ret = OB_CHECKSUM_TYPE_NOT_SUPPORTED;
- OB_LOG(WARN, "that checksum algorithm is not supported for s3", K(ret), K(checksum_type));
- } else {
- if (checksum_type == ObStorageChecksumType::OB_CRC32_ALGO) {
- request.SetChecksumAlgorithm(Aws::S3::Model::ChecksumAlgorithm::CRC32);
- } else {
- // default md5
- }
- }
- return ret;
-}
-
-// 1. OBS/GCS is currently accessed through the S3 SDK without any dedicated OBS/GCS prefix or type,
-// and using the host to determine if the endpoint is OBS/GCS is not secure.
-// 2. The S3 SDK can only acquire the response header to distinguish the server type
-// when a request fails.
-// 3. Furthermore, when the S3 SDK is used to access OBS/GCS with the checksum type set to crc32,
-// the OBS/GCS server will only return an error in the complete_multipart_upload interface.
-static int set_completed_part_checksum(Aws::S3::Model::CompletedPart &completed_part,
- const ObStorageChecksumType checksum_type,
- const char *checksum)
-{
- int ret = OB_SUCCESS;
- if (OB_UNLIKELY(!is_s3_supported_checksum(checksum_type))) {
- ret = OB_CHECKSUM_TYPE_NOT_SUPPORTED;
- OB_LOG(WARN, "that checksum algorithm is not supported for s3", K(ret), K(checksum_type));
- } else {
- if (checksum_type == ObStorageChecksumType::OB_CRC32_ALGO) {
- if (OB_ISNULL(checksum)) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "checksum is null", K(ret), K(checksum_type), KP(checksum));
- } else {
- completed_part.SetChecksumCRC32(checksum);
- }
- } else {
- // default md5
- }
- }
- return ret;
-}
-
-static int get_completed_part_checksum(const Aws::S3::Model::UploadPartResult &result,
- const ObStorageChecksumType checksum_type,
- const char *&checksum_str)
-{
- int ret = OB_SUCCESS;
- checksum_str = nullptr;
- if (OB_UNLIKELY(!is_s3_supported_checksum(checksum_type))) {
- ret = OB_CHECKSUM_TYPE_NOT_SUPPORTED;
- OB_LOG(WARN, "that checksum algorithm is not supported for s3", K(ret), K(checksum_type));
- } else {
- if (checksum_type == ObStorageChecksumType::OB_CRC32_ALGO) {
- if (OB_UNLIKELY(result.GetChecksumCRC32().empty())) {
- ret = OB_CHECKSUM_TYPE_NOT_SUPPORTED;
- OB_LOG(WARN, "returned checksum is null", K(ret), K(checksum_type));
- } else {
- checksum_str = result.GetChecksumCRC32().c_str();
- }
- } else {
- // default md5
- }
- }
- return ret;
-}
-
-// The check_xxx functions are used to verify that the checksum of the returned data
-// when reading a complete object matches the checksum value carried in the response header.
-// As it's unclear what checksum algorithm was specifically used during data upload,
-// the check_xxx functions should not be used in isolation. Instead, use validate_response_checksum.
-// Therefore, the check_xxx functions do not validate the input arguments for their effectiveness;
-// validate_response_checksum is responsible for performing a unified validation.
-static int check_crc32(const char *buf, const int64_t size, Aws::S3::Model::GetObjectResult &result)
-{
- int ret = OB_SUCCESS;
- const Aws::String &response_checksum = result.GetChecksumCRC32();
- if (!response_checksum.empty()) {
- Aws::String buf_str(buf, size);
- Aws::Utils::ByteBuffer checksum_buf = Aws::Utils::HashingUtils::CalculateCRC32(buf_str);
- Aws::String returned_data_checksum = Aws::Utils::HashingUtils::Base64Encode(checksum_buf);
-
- if (OB_UNLIKELY(returned_data_checksum != response_checksum)) {
- ret = OB_OBJECT_STORAGE_CHECKSUM_ERROR;
- OB_LOG(ERROR, "crc32 mismatch",
- K(ret), K(size), K(returned_data_checksum.c_str()), K(response_checksum.c_str()));
- }
- }
- return ret;
-}
-
-static int check_crc32c(const char *buf, const int64_t size, Aws::S3::Model::GetObjectResult &result)
-{
- int ret = OB_SUCCESS;
- const Aws::String &response_checksum = result.GetChecksumCRC32C();
- if (!response_checksum.empty()) {
- Aws::String buf_str(buf, size);
- Aws::Utils::ByteBuffer checksum_buf = Aws::Utils::HashingUtils::CalculateCRC32C(buf_str);
- Aws::String returned_data_checksum = Aws::Utils::HashingUtils::Base64Encode(checksum_buf);
-
- if (OB_UNLIKELY(returned_data_checksum != response_checksum)) {
- ret = OB_OBJECT_STORAGE_CHECKSUM_ERROR;
- OB_LOG(ERROR, "crc32c mismatch",
- K(ret), K(size), K(returned_data_checksum.c_str()), K(response_checksum.c_str()));
- }
- }
- return ret;
-}
-
-static int check_sha1(const char *buf, const int64_t size, Aws::S3::Model::GetObjectResult &result)
-{
- int ret = OB_SUCCESS;
- const Aws::String &response_checksum = result.GetChecksumSHA1();
- if (!response_checksum.empty()) {
- Aws::String buf_str(buf, size);
- Aws::Utils::ByteBuffer checksum_buf = Aws::Utils::HashingUtils::CalculateSHA1(buf_str);
- Aws::String returned_data_checksum = Aws::Utils::HashingUtils::Base64Encode(checksum_buf);
-
- if (OB_UNLIKELY(returned_data_checksum != response_checksum)) {
- ret = OB_OBJECT_STORAGE_CHECKSUM_ERROR;
- OB_LOG(ERROR, "sha1 mismatch",
- K(ret), K(size), K(returned_data_checksum.c_str()), K(response_checksum.c_str()));
- }
- }
- return ret;
-}
-
-
-static int validate_response_checksum(
- const char *buf, const int64_t size, Aws::S3::Model::GetObjectResult &result)
-{
- int ret = OB_SUCCESS;
- if (OB_ISNULL(buf) || OB_UNLIKELY(size <= 0)) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "invalid argument", K(ret), KP(buf), K(size));
- } else if (OB_FAIL(check_crc32(buf, size, result))) {
- OB_LOG(WARN, "failed to check crc32", K(ret));
- } else if (OB_FAIL(check_crc32c(buf, size, result))) {
- OB_LOG(WARN, "failed to check crc32c", K(ret));
- } else if (OB_FAIL(check_sha1(buf, size, result))) {
- OB_LOG(WARN, "failed to check sha1", K(ret));
- } else if (OB_FAIL(check_sha1(buf, size, result))) {
- OB_LOG(WARN, "failed to check sha256", K(ret));
- }
- return ret;
-}
-
-/*--------------------------------ObS3Account--------------------------------*/
-ObS3Account::ObS3Account()
-{
- reset();
-}
-
-ObS3Account::~ObS3Account()
-{
- if (is_valid_) {
- reset();
- }
-}
-
-void ObS3Account::reset()
-{
- is_valid_ = false;
- delete_mode_ = ObStorageDeleteMode::STORAGE_DELETE_MODE;
- MEMSET(region_, 0, sizeof(region_));
- MEMSET(endpoint_, 0, sizeof(endpoint_));
- MEMSET(access_id_, 0, sizeof(access_id_));
- MEMSET(secret_key_, 0, sizeof(secret_key_));
- sts_token_.reset();
- addressing_model_ = ObStorageAddressingModel::OB_VIRTUAL_HOSTED_STYLE;
-}
-
-int64_t ObS3Account::hash() const
-{
- int64_t hash_value = 0;
- hash_value = murmurhash(region_, static_cast(strlen(region_)), hash_value);
- hash_value = murmurhash(endpoint_, static_cast(strlen(endpoint_)), hash_value);
- hash_value = murmurhash(access_id_, static_cast(strlen(access_id_)), hash_value);
- hash_value = murmurhash(secret_key_, static_cast(strlen(secret_key_)), hash_value);
- hash_value = murmurhash(&addressing_model_, sizeof(addressing_model_), hash_value);
- return hash_value;
-}
-
-int ObS3Account::parse_from(const char *storage_info_str, const int64_t size)
-{
- int ret = OB_SUCCESS;
- if (OB_ISNULL(storage_info_str) || size >= OB_MAX_BACKUP_STORAGE_INFO_LENGTH) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "failed to init s3 account, invalid argument", K(ret), KP(storage_info_str), K(size));
- } else {
- char tmp[OB_MAX_BACKUP_STORAGE_INFO_LENGTH];
- char *token = NULL;
- char *saved_ptr = NULL;
-
- uint8_t bitmap = 0;
- MEMCPY(tmp, storage_info_str, size);
- tmp[size] = '\0';
- token = tmp;
- for (char *str = token; OB_SUCC(ret); str = NULL) {
-#ifdef _WIN32
- token = ::strtok_s(str, "&", &saved_ptr);
-#else
- token = ::strtok_r(str, "&", &saved_ptr);
-#endif
- if (OB_ISNULL(token)) {
- break;
- } else if (0 == strncmp(REGION, token, strlen(REGION))) {
- if (OB_FAIL(ob_set_field(token + strlen(REGION), region_, sizeof(region_)))) {
- OB_LOG(WARN, "failed to set s3 region", K(ret), KCSTRING(token));
- }
- } else if (0 == strncmp(HOST, token, strlen(HOST))) {
- if (OB_FAIL(ob_set_field(token + strlen(HOST), endpoint_, sizeof(endpoint_)))) {
- OB_LOG(WARN, "failed to set s3 endpoint", K(ret), KCSTRING(token));
- } else {
- bitmap |= 1;
- }
- } else if (0 == strncmp(ACCESS_ID, token, strlen(ACCESS_ID))) {
- if (OB_FAIL(ob_set_field(token + strlen(ACCESS_ID), access_id_, sizeof(access_id_)))) {
- OB_LOG(WARN, "failed to set s3 access id", K(ret), KCSTRING(token));
- } else {
- bitmap |= (1 << 1);
- }
- } else if (0 == strncmp(ACCESS_KEY, token, strlen(ACCESS_KEY))) {
- if (OB_FAIL(ob_set_field(token + strlen(ACCESS_KEY), secret_key_, sizeof(secret_key_)))) {
- OB_LOG(WARN, "failed to set s3 secret key", K(ret), KP(token));
- } else {
- bitmap |= (1 << 2);
- }
- } else if (0 == strncmp(DELETE_MODE, token, strlen(DELETE_MODE))) {
- if (0 == strcmp(token + strlen(DELETE_MODE), "delete")) {
- delete_mode_ = ObStorageDeleteMode::STORAGE_DELETE_MODE;
- } else if (0 == strcmp(token + strlen(DELETE_MODE), "tagging")) {
- delete_mode_ = ObStorageDeleteMode::STORAGE_TAGGING_MODE;
- } else {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "delete mode is invalid", K(ret), KCSTRING(token));
- }
- } else if (0 == strncmp(ADDRESSING_MODEL, token, strlen(ADDRESSING_MODEL))) {
- if (0 == strcmp(token + strlen(ADDRESSING_MODEL), ADDRESSING_MODEL_VIRTUAL_HOSTED_STYLE)) {
- addressing_model_ = ObStorageAddressingModel::OB_VIRTUAL_HOSTED_STYLE;
- } else if (0 == strcmp(token + strlen(ADDRESSING_MODEL), ADDRESSING_MODEL_PATH_STYLE)) {
- addressing_model_ = ObStorageAddressingModel::OB_PATH_STYLE;
- } else {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "addressing model is invalid", K(ret), KCSTRING(token));
- }
- } else {
- OB_LOG(DEBUG, "unknown s3 info", K(*token));
- }
- }
-
- if (OB_SUCC(ret) && bitmap != 0x07) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "failed to init s3 account", K(ret), KCSTRING(region_),
- KCSTRING(endpoint_), KCSTRING(access_id_), K(bitmap));
- }
- }
-
- if (OB_SUCC(ret)) {
- is_valid_ = true;
- OB_LOG(DEBUG, "succeed to init s3 account",
- KCSTRING(region_), KCSTRING(endpoint_), KCSTRING(access_id_));
- } else {
- reset();
- }
- return ret;
-}
-
-/*--------------------------------ObStorageS3Base--------------------------------*/
-ObStorageS3Base::ObStorageS3Base()
- : allocator_(OB_STORAGE_S3_ALLOCATOR, OB_MALLOC_NORMAL_BLOCK_SIZE, ObObjectStorageTenantGuard::get_tenant_id()),
- s3_client_(NULL),
- bucket_(),
- object_(),
- checksum_type_(ObStorageChecksumType::OB_MD5_ALGO),
- is_inited_(false),
- s3_account_()
-{
-}
-
-ObStorageS3Base::~ObStorageS3Base()
-{
- ObStorageS3Base::reset();
-}
-
-void ObStorageS3Base::reset()
-{
- is_inited_ = false;
- s3_account_.reset();
- allocator_.clear();
- checksum_type_ = ObStorageChecksumType::OB_MD5_ALGO;
- if (OB_NOT_NULL(s3_client_)) {
- s3_client_->release();
- s3_client_ = NULL;
- }
-}
-
-int ObStorageS3Base::open(const ObString &uri, ObObjectStorageInfo *storage_info)
-{
- int ret = OB_SUCCESS;
- if (OB_UNLIKELY(is_inited_)) {
- ret = OB_INIT_TWICE;
- OB_LOG(WARN, "s3 base alreagy inited", K(ret));
- } else if (OB_FAIL(inner_open(uri, storage_info))) {
- OB_LOG(WARN, "failed to inner open", K(ret), K(uri), KPC(storage_info));
- }
- // object name should not be empty
- if (OB_SUCC(ret) && OB_UNLIKELY(object_.empty())) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "object name is empty", K(uri), K(ret), K(uri));
- reset();
- }
- return ret;
-}
-
-
-int ObStorageS3Base::inner_open(const ObString &uri, ObObjectStorageInfo *storage_info)
-{
- int ret = OB_SUCCESS;
- char info_str[common::OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 };
- if (OB_UNLIKELY(is_inited_)) {
- ret = OB_INIT_TWICE;
- OB_LOG(WARN, "s3 base alreagy inited", K(ret));
- } else if (OB_ISNULL(storage_info) || OB_UNLIKELY(uri.empty() || !storage_info->is_valid())) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "failed to init s3 base, invalid arguments", K(ret), K(uri), KPC(storage_info));
- } else if (OB_FAIL(build_bucket_and_object_name(allocator_, uri, bucket_, object_))) {
- OB_LOG(WARN, "failed to parse uri", K(ret), K(uri));
- } else if (OB_FAIL(storage_info->get_authorization_str(info_str, sizeof(info_str), s3_account_.sts_token_))) {
- OB_LOG(WARN, "failed to get authorization str", K(ret), KPC(storage_info));
- } else if (OB_FAIL(s3_account_.parse_from(info_str, strlen(info_str)))) {
- OB_LOG(WARN, "failed to build s3 account", K(ret));
- } else if (OB_FAIL(ObS3Env::get_instance().get_or_create_s3_client(s3_account_, s3_client_))) {
- OB_LOG(WARN, "faied to get s3 client", K(ret));
- } else {
- checksum_type_ = storage_info->get_checksum_type();
-#ifdef ERRSIM
- if (OB_NOT_NULL(storage_info) && (OB_SUCCESS != EventTable::EN_ENABLE_LOG_OBJECT_STORAGE_CHECKSUM_TYPE)) {
- OB_LOG(ERROR, "errsim backup io with checksum type", "checksum_type", storage_info->get_checksum_type_str());
- }
-#endif
- if (OB_UNLIKELY(!is_s3_supported_checksum(checksum_type_))) {
- ret = OB_CHECKSUM_TYPE_NOT_SUPPORTED;
- OB_LOG(WARN, "that checksum algorithm is not supported for s3", K(ret), K_(checksum_type));
- } else {
- is_inited_ = true;
- }
- }
- if (OB_FAIL(ret)) {
- reset();
- }
- return ret;
-}
-
-
-// can only be used to get the metadata of normal objects
-int ObStorageS3Base::get_s3_file_meta_(S3ObjectMeta &meta)
-{
- int ret = OB_SUCCESS;
- meta.reset();
- if (OB_UNLIKELY(!is_inited_)) {
- ret = OB_NOT_INIT;
- OB_LOG(WARN, "s3 base not inited", K(ret));
- } else {
- Aws::S3::Model::HeadObjectRequest request;
- if (OB_UNLIKELY(object_.empty())) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "object name is empty", K(ret), K_(bucket), K_(object));
- } else {
- request.WithBucket(bucket_.ptr()).WithKey(object_.ptr());
- }
-
- Aws::S3::Model::HeadObjectOutcome outcome;
- if (FAILEDx(s3_client_->head_object(request, outcome))) {
- OB_LOG(WARN, "failed to head s3 object", K(ret));
- } else if (!outcome.IsSuccess()) {
- handle_s3_outcome(outcome, ret);
- if (OB_OBJECT_NOT_EXIST == ret) {
- ret = OB_SUCCESS;
- } else {
- OB_LOG(WARN, "failed to head s3 object", K(ret), K_(bucket), K_(object));
- }
- } else {
- meta.is_exist_ = true;
- meta.length_ = outcome.GetResult().GetContentLength();
- }
- }
- return ret;
-}
-
-int ObStorageS3Base::do_list_(const int64_t max_list_num, const char *delimiter,
- const Aws::String &next_marker, Aws::S3::Model::ListObjectsOutcome &outcome)
-{
- int ret = OB_SUCCESS;
- char dir_path_buf[OB_MAX_URI_LENGTH] = {0};
-
- if (OB_UNLIKELY(!is_inited_)) {
- ret = OB_NOT_INIT;
- OB_LOG(WARN, "s3 base not inited", K(ret));
- } else if (OB_UNLIKELY(max_list_num <= 0 || max_list_num > OB_STORAGE_LIST_MAX_NUM)) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "invalid arguments", K(ret), K_(bucket), K_(object), K(max_list_num));
- } else if (OB_UNLIKELY(!is_null_or_end_with_slash(object_.ptr()))) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "uri is not a valid dir path", K(ret), K_(object), K_(bucket));
- } else {
- Aws::S3::Model::ListObjectsRequest request;
- // if object_ is empty, list all objects in the bucket
- if (!object_.empty()) {
- request.WithBucket(bucket_.ptr()).WithPrefix(object_.ptr()).WithMaxKeys(max_list_num);
- } else {
- request.WithBucket(bucket_.ptr()).WithMaxKeys(max_list_num);
- }
- if (NULL != delimiter && strlen(delimiter) > 0) {
- request.SetDelimiter(delimiter);
- }
- if (!next_marker.empty()) {
- request.SetMarker(next_marker);
- }
-
- if (OB_FAIL(s3_client_->list_objects(request, outcome))) {
- OB_LOG(WARN, "failed to list s3 objects", K(ret));
- } else if (!outcome.IsSuccess()) {
- handle_s3_outcome(outcome, ret);
- OB_LOG(WARN, "failed to list s3 objects", K(ret),
- K_(bucket), K_(object), K(max_list_num), K(delimiter));
- }
- }
- return ret;
-}
-
-/*--------------------------------ObStorageS3Writer--------------------------------*/
-ObStorageS3Writer::ObStorageS3Writer()
- : ObStorageS3Base(),
- is_opened_(false),
- file_length_(-1)
-{
-}
-
-ObStorageS3Writer::~ObStorageS3Writer()
-{
- if (is_opened_) {
- close();
- }
-}
-
-int ObStorageS3Writer::open_(const ObString &uri, ObObjectStorageInfo *storage_info)
-{
- int ret = OB_SUCCESS;
- if (OB_UNLIKELY(is_opened_)) {
- ret = OB_OBJECT_STORAGE_IO_ERROR;
- OB_LOG(WARN, "s3 writer already open, cannot open again", K(ret), K(uri));
- } else if (OB_FAIL(ObStorageS3Base::open(uri, storage_info))) {
- OB_LOG(WARN, "failed to open in s3 base", K(ret), K(uri));
- } else {
- is_opened_ = true;
- file_length_ = 0;
- }
- return ret;
-}
-
-static int init_put_object_request(const char *bucket, const char *object,
- const char *buf, const int64_t size, const ObStorageChecksumType checksum_type,
- Aws::S3::Model::PutObjectRequest &request)
-{
- int ret = OB_SUCCESS;
- if (OB_ISNULL(bucket) || OB_ISNULL(object) || OB_ISNULL(buf) || OB_UNLIKELY(size < 0)) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "invalid arguments", K(ret), KP(bucket), KP(object), KP(buf), K(size));
- } else if (OB_FAIL(set_request_checkusum_algorithm(request, checksum_type))) {
- OB_LOG(WARN, "fail to set checksum algorithm", K(ret), K(checksum_type));
- } else {
- request.WithBucket(bucket).WithKey(object);
- std::shared_ptr data_stream =
- Aws::MakeShared(S3_SDK);
- data_stream->write(buf, size);
- data_stream->flush();
- request.SetBody(data_stream);
- request.SetContentLength(static_cast(request.GetBody()->tellp()));
- }
- return ret;
-}
-
-int ObStorageS3Writer::write_(const char *buf, const int64_t size)
-{
- int ret = OB_SUCCESS;
- Aws::S3::Model::PutObjectRequest request;
- ObExternalIOCounterGuard io_guard;
- if (OB_UNLIKELY(!is_opened_)) {
- ret = OB_NOT_INIT;
- OB_LOG(WARN, "s3 writer not opened", K(ret));
- } else if (OB_ISNULL(buf) || OB_UNLIKELY(size < 0)) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "buf is NULL or size is invalid", K(ret), KP(buf), K(size));
- } else if (OB_FAIL(write_obj_(object_.ptr(), buf, size))) {
- OB_LOG(WARN, "fail to put s3 object", K(ret), K_(bucket), K_(object));
- } else {
- file_length_ = size;
- }
- return ret;
-}
-
-int ObStorageS3Writer::write_obj_(const char *obj_name, const char *buf, const int64_t size)
-{
- int ret = OB_SUCCESS;
- Aws::S3::Model::PutObjectRequest request;
- if (OB_UNLIKELY(!is_opened_)) {
- ret = OB_NOT_INIT;
- OB_LOG(WARN, "s3 writer not opened", K(ret));
- } else if (OB_ISNULL(obj_name) || OB_ISNULL(buf) || OB_UNLIKELY(size < 0)) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "invalid arguments", K(ret), KP(obj_name), KP(buf), K(size));
- } else if (OB_FAIL(init_put_object_request(bucket_.ptr(), obj_name, buf,
- size, checksum_type_, request))) {
- OB_LOG(WARN, "failed to init put obejct request", K(ret),
- K_(bucket), K(obj_name), KP(buf), K(size), K_(checksum_type));
- } else {
- Aws::S3::Model::PutObjectOutcome outcome;
- if (OB_FAIL(s3_client_->put_object(request, outcome))) {
- OB_LOG(WARN, "failed to put s3 object", K(ret));
-#ifdef ERRSIM
- } else if (OB_FAIL(EventTable::EN_OBJECT_STORAGE_CHECKSUM_ERROR)) {
- ret = OB_OBJECT_STORAGE_CHECKSUM_ERROR;
-#endif
- } else if (!outcome.IsSuccess()) {
- handle_s3_outcome(outcome, ret);
- OB_LOG(WARN, "failed to write object into s3",
- K(ret), K_(bucket), K(obj_name), KP(buf), K(size));
- }
- }
- return ret;
-}
-
-int ObStorageS3Writer::pwrite_(const char *buf, const int64_t size, const int64_t offset)
-{
- int ret = OB_NOT_SUPPORTED;
- UNUSED(buf);
- UNUSED(size);
- UNUSED(offset);
- return ret;
-}
-
-int ObStorageS3Writer::close_()
-{
- int ret = OB_SUCCESS;
- is_opened_ = false;
- file_length_ = -1;
- ObStorageS3Base::reset();
- return ret;
-}
-
-/*--------------------------------ObStorageS3Reader--------------------------------*/
-ObStorageS3Reader::ObStorageS3Reader()
- : ObStorageS3Base(),
- is_opened_(false),
- has_meta_(false),
- file_length_(-1)
-{
-}
-
-ObStorageS3Reader::~ObStorageS3Reader()
-{
- close();
-}
-
-void ObStorageS3Reader::reset()
-{
- ObStorageS3Base::reset();
- is_opened_ = false;
- has_meta_ = false;
- file_length_ = -1;
-}
-
-int ObStorageS3Reader::open_(const ObString &uri,
- ObObjectStorageInfo *storage_info, const bool head_meta)
-{
- int ret = OB_SUCCESS;
- S3ObjectMeta meta;
- ObExternalIOCounterGuard io_guard;
- if (OB_UNLIKELY(is_opened_)) {
- ret = OB_OBJECT_STORAGE_IO_ERROR;
- OB_LOG(WARN, "s3 reader already open, cannot open again", K(ret), K(uri));
- } else if (OB_FAIL(ObStorageS3Base::open(uri, storage_info))) {
- OB_LOG(WARN, "failed to open in s3 base", K(ret), K(uri));
- } else {
- if (head_meta) {
- if (OB_FAIL(get_s3_file_meta(meta))) {
- OB_LOG(WARN, "failed to get s3 object meta", K(ret), K(uri));
- } else if (!meta.is_exist_) {
- ret = OB_OBJECT_NOT_EXIST;
- OB_LOG(WARN, "object is not exist", K(ret), K(uri), K_(bucket), K_(object));
- } else {
- file_length_ = meta.length_;
- has_meta_ = true;
- }
- }
- }
-
- if (OB_FAIL(ret)) {
- reset();
- } else {
- is_opened_ = true;
- }
- return ret;
-}
-
-int ObStorageS3Reader::pread_(char *buf,
- const int64_t buf_size, const int64_t offset, int64_t &read_size)
-{
- int ret = OB_SUCCESS;
- ObExternalIOCounterGuard io_guard;
- if (OB_UNLIKELY(!is_opened_)) {
- ret = OB_OBJECT_STORAGE_IO_ERROR;
- OB_LOG(WARN, "s3 reader not opened", K(ret));
- } else if (OB_ISNULL(buf) || OB_UNLIKELY(buf_size <= 0 || offset < 0)) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "invalid arguments", K(ret), KP(buf), K(buf_size), K(offset));
- } else {
- int64_t get_data_size = buf_size;
- if (has_meta_) {
- if (file_length_ < offset) {
- ret = OB_FILE_LENGTH_INVALID;
- OB_LOG(WARN, "offset is larger than file length",
- K(ret), K(offset), K_(file_length), K_(bucket), K_(object));
- } else {
- get_data_size = MIN(buf_size, file_length_ - offset);
- }
- }
-
- if (OB_FAIL(ret)) {
- } else if (get_data_size == 0) {
- read_size = 0;
- } else {
- Aws::S3::Model::GetObjectRequest request;
- Aws::S3::Model::GetObjectOutcome outcome;
- char range_read[64] = { 0 };
- request.WithBucket(bucket_.ptr()).WithKey(object_.ptr());
-
- if (OB_FAIL(databuff_printf(range_read, sizeof(range_read),
- "bytes=%ld-%ld", offset, offset + get_data_size - 1))) {
- OB_LOG(WARN, "fail to set range to read", K(ret),
- K_(bucket), K_(object), K(offset), K(buf_size), K_(has_meta), K_(file_length));
- } else if (FALSE_IT(request.SetRange(range_read))) {
- } else if (OB_FAIL(s3_client_->get_object(request, outcome))) {
- OB_LOG(WARN, "failed to get s3 object", K(ret), K(range_read));
- } else if (!outcome.IsSuccess()) {
- handle_s3_outcome(outcome, ret);
- OB_LOG(WARN, "failed to read object from s3",
- K(ret), K_(bucket), K_(object), K(range_read));
- } else if (FALSE_IT(read_size = outcome.GetResult().GetContentLength())) {
- } else if (OB_UNLIKELY(has_meta_ && read_size != get_data_size)) {
- ret = OB_OBJECT_STORAGE_IO_ERROR;
- OB_LOG(WARN, "returned data size is not equal to expected", K(ret),
- K_(bucket), K_(object), K(offset), K(buf_size), K_(has_meta), K_(file_length));
- log_s3_status(outcome, ret);
- } else {
- outcome.GetResult().GetBody().read(buf, read_size);
-
- // read size <= get_data_size <= buf_size
- // For nohead read, file_length_ is always -1, so the logic below will not be executed
- if (read_size == file_length_) {
- if (OB_FAIL(validate_response_checksum(buf, read_size, outcome.GetResult()))) {
- OB_LOG(WARN, "fail to validate_response_checksum",
- K(ret), K(read_size), K(get_data_size), K(buf_size));
- }
- }
-
- }
- }
- }
- return ret;
-}
-
-int ObStorageS3Reader::close_()
-{
- int ret = OB_SUCCESS;
- ObExternalIOCounterGuard io_guard;
- reset();
- return ret;
-}
-
-/*--------------------------------ObStorageS3Util--------------------------------*/
-ObStorageS3Util::ObStorageS3Util() : is_opened_(false), storage_info_(NULL)
-{
-}
-
-ObStorageS3Util::~ObStorageS3Util()
-{
- close();
-}
-
-int ObStorageS3Util::open(ObObjectStorageInfo *storage_info)
-{
- int ret = OB_SUCCESS;
- if (OB_UNLIKELY(is_opened_)) {
- ret = OB_OBJECT_STORAGE_IO_ERROR;
- OB_LOG(WARN, "s3 util already open, cannot open again", K(ret));
- } else if (OB_ISNULL(storage_info)) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "storage info is NULL", K(ret), KP(storage_info));
- } else {
- is_opened_ = true;
- storage_info_ = storage_info;
- }
- return ret;
-}
-
-void ObStorageS3Util::close()
-{
- is_opened_ = false;
- storage_info_ = NULL;
-}
-
-int ObStorageS3Util::is_exist_(const ObString &uri, bool &exist)
-{
- int ret = OB_SUCCESS;
- exist = false;
- ObStorageObjectMetaBase obj_meta;
- if (OB_FAIL(head_object_meta(uri, obj_meta))) {
- OB_LOG(WARN, "fail to head object meta", K(ret), K(uri));
- } else {
- exist = obj_meta.is_exist_;
- }
- return ret;
-}
-
-int ObStorageS3Util::get_file_length_(const ObString &uri, int64_t &file_length)
-{
- int ret = OB_SUCCESS;
- file_length = 0;
- ObStorageObjectMetaBase obj_meta;
- if (OB_FAIL(head_object_meta(uri, obj_meta))) {
- OB_LOG(WARN, "fail to head object meta", K(ret), K(uri));
- } else if (!obj_meta.is_exist_) {
- ret = OB_OBJECT_NOT_EXIST;
- OB_LOG(WARN, "object is not exist", K(ret), K(uri));
- } else {
- file_length = obj_meta.length_;
- }
- return ret;
-}
-
-int ObStorageS3Util::head_object_meta(const ObString &uri, ObStorageObjectMetaBase &obj_meta)
-{
- int ret = OB_SUCCESS;
- S3ObjectMeta meta;
- ObStorageS3Base s3_base;
- obj_meta.reset();
- ObExternalIOCounterGuard io_guard;
- if (OB_UNLIKELY(!is_opened_)) {
- ret = OB_OBJECT_STORAGE_IO_ERROR;
- OB_LOG(WARN, "s3 util not opened", K(ret));
- } else if (OB_UNLIKELY(uri.empty())) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "uri is empty", K(ret), K(uri));
- } else if (OB_FAIL(s3_base.open(uri, storage_info_))) {
- OB_LOG(WARN, "failed to open s3 base", K(ret), K(uri));
- } else if (OB_FAIL(s3_base.get_s3_file_meta(meta))) {
- OB_LOG(WARN, "failed to get s3 file meta", K(ret), K(uri));
- } else {
- obj_meta.is_exist_ = meta.is_exist_;
- if (obj_meta.is_exist_) {
- obj_meta.length_ = meta.length_;
- }
- }
- return ret;
-}
-
-int ObStorageS3Util::delete_object_(ObStorageS3Base &s3_base)
-{
- int ret = OB_SUCCESS;
- Aws::S3::Model::DeleteObjectRequest request;
- request.WithBucket(s3_base.bucket_.ptr()).WithKey(s3_base.object_.ptr());
- Aws::S3::Model::DeleteObjectOutcome outcome;
- if (OB_FAIL(s3_base.s3_client_->delete_object(request, outcome))) {
- OB_LOG(WARN, "failed to delete s3 object", K(ret));
- } else if (!outcome.IsSuccess()) {
- handle_s3_outcome(outcome, ret);
- OB_LOG(WARN, "failed to delete s3 object",
- K(ret), K(s3_base.bucket_), K(s3_base.object_));
- } else {
- OB_LOG(DEBUG, "delete s3 object succeed", K(s3_base.bucket_), K(s3_base.object_));
- }
- return ret;
-}
-
-int ObStorageS3Util::tagging_object_(ObStorageS3Base &s3_base)
-{
- int ret = OB_SUCCESS;
- Aws::S3::Model::PutObjectTaggingRequest request;
- request.WithBucket(s3_base.bucket_.ptr()).WithKey(s3_base.object_.ptr());
- Aws::S3::Model::Tag tag;
- tag.WithKey("delete_mode").WithValue("tagging");
- Aws::S3::Model::Tagging tagging;
- tagging.AddTagSet(tag);
- request.SetTagging(tagging);
- Aws::S3::Model::PutObjectTaggingOutcome outcome;
- if (OB_FAIL(s3_base.s3_client_->put_object_tagging(request, outcome))) {
- OB_LOG(WARN, "failed to put s3 object tagging", K(ret));
- } else if (!outcome.IsSuccess()) {
- handle_s3_outcome(outcome, ret);
- OB_LOG(WARN, "failed to tagging s3 object",
- K(ret), K(s3_base.bucket_), K(s3_base.object_));
- } else {
- OB_LOG(DEBUG, "tagging s3 object succeed", K(s3_base.bucket_), K(s3_base.object_));
- }
- return ret;
-}
-
-int ObStorageS3Util::del_file_(const ObString &uri)
-{
- int ret = OB_SUCCESS;
- ObStorageS3Base s3_base;
- ObExternalIOCounterGuard io_guard;
- if (OB_UNLIKELY(!is_opened_)) {
- ret = OB_OBJECT_STORAGE_IO_ERROR;
- OB_LOG(WARN, "s3 util not opened", K(ret));
- } else if (OB_UNLIKELY(uri.empty())) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "invalid arguments", K(ret), K(uri));
- } else if (OB_FAIL(s3_base.open(uri, storage_info_))) {
- OB_LOG(WARN, "failed to open s3 base", K(ret), K(uri));
- } else {
- const int64_t delete_mode = s3_base.s3_account_.delete_mode_;
- if (ObStorageDeleteMode::STORAGE_DELETE_MODE == delete_mode) {
- if (OB_FAIL(delete_object_(s3_base))) {
- if (OB_OBJECT_NOT_EXIST == ret) {
- // Uniform handling of 'object not found' scenarios across different object storage services:
- // GCS returns the OB_OBJECT_NOT_EXIST error when an object does not exist,
- // whereas other object storage services may not report an error.
- // Therefore, to maintain consistency,
- // no error code is returned when attempting to delete a non-existent object
- ret = OB_SUCCESS;
- } else {
- OB_LOG(WARN, "failed to delete s3 object", K(ret), K(uri));
- }
- }
- } else if (ObStorageDeleteMode::STORAGE_TAGGING_MODE == delete_mode) {
- if (OB_FAIL(tagging_object_(s3_base))) {
- OB_LOG(WARN, "failed to tag s3 object", K(ret), K(uri));
- }
- } else {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "s3 delete mode invalid", K(ret), K(uri), K(delete_mode));
- }
- }
- return ret;
-}
-
-int ObStorageS3Util::batch_del_files_(
- const ObString &uri,
- hash::ObHashMap &files_to_delete,
- ObIArray &failed_files_idx)
-{
- int ret = OB_SUCCESS;
- ObStorageS3Base s3_base;
- const int64_t n_files_to_delete = files_to_delete.size();
- ObArenaAllocator allocator(OB_STORAGE_S3_ALLOCATOR, OB_MALLOC_NORMAL_BLOCK_SIZE, ObObjectStorageTenantGuard::get_tenant_id());
- ObExternalIOCounterGuard io_guard;
-
- if (OB_UNLIKELY(!is_opened_)) {
- ret = OB_NOT_INIT;
- OB_LOG(WARN, "s3 util not opened", K(ret), K(uri));
- } else if (OB_FAIL(check_files_map_validity(files_to_delete))) {
- OB_LOG(WARN, "files_to_delete is invalid", K(ret), K(uri));
- } else if (OB_FAIL(s3_base.open(uri, storage_info_))) {
- OB_LOG(WARN, "failed to open s3 base", K(ret), K(uri), KPC_(storage_info));
- } else if (OB_UNLIKELY(ObStorageDeleteMode::STORAGE_TAGGING_MODE == s3_base.s3_account_.delete_mode_)) {
- ret = OB_NOT_SUPPORTED;
- OB_LOG(WARN, "batch tagging is not supported", K(ret), K(uri), K(s3_base.s3_account_));
- } else {
- char *tmp_key = nullptr;
- Aws::S3::Model::Delete delete_info;
- delete_info.SetQuiet(false); // return all object's status
- hash::ObHashMap::const_iterator iter = files_to_delete.begin();
- while (OB_SUCC(ret) && iter != files_to_delete.end()) {
- if (OB_FAIL(ob_dup_cstring(allocator, iter->first, tmp_key))) {
- OB_LOG(WARN, "fail to copy c string", K(ret), K(uri),
- K(iter->first), KPC_(storage_info), K(s3_base.bucket_), K(s3_base.object_));
- } else {
- delete_info.AddObjects(Aws::S3::Model::ObjectIdentifier().WithKey(tmp_key));
- iter++;
- }
- }
-
- Aws::S3::Model::DeleteObjectsRequest request;
- Aws::S3::Model::DeleteObjectsOutcome outcome;
- request.WithBucket(s3_base.bucket_.ptr()).WithDelete(delete_info);
- if (FAILEDx(s3_base.s3_client_->delete_objects(request, outcome))) {
- OB_LOG(WARN, "failed to delete multiple objects", K(ret), K(s3_base.s3_account_));
- } else if (!outcome.IsSuccess()) {
- handle_s3_outcome(outcome, ret);
- if (is_gcs_destination(outcome)) {
- // For GCS, ignore current error
- ret = OB_NOT_SUPPORTED;
- OB_LOG(WARN, "delete objects interface is not supported for GCS",
- K(ret), K(uri), K(s3_base.bucket_), K(s3_base.object_), KPC_(storage_info));
- } else {
- OB_LOG(WARN, "failed to delete objects",
- K(ret), K(uri), K(s3_base.bucket_), K(s3_base.object_), KPC_(storage_info));
- }
- } else {
- // The deleted_object_list contains all the objects that were successfully deleted.
- // By comparing it to files_to_delete, we can identify the objects that failed to be deleted.
- const Aws::Vector &deleted_object_list = outcome.GetResult().GetDeleted();
- const char *object_name = nullptr;
- int64_t object_name_len = 0;
- for (int64_t i = 0; OB_SUCC(ret) && i < deleted_object_list.size(); i++) {
- object_name = deleted_object_list[i].GetKey().c_str();
- object_name_len = deleted_object_list[i].GetKey().size();
- if (OB_ISNULL(object_name) || OB_UNLIKELY(object_name_len <= 0)) {
- ret = OB_ERR_UNEXPECTED;
- OB_LOG(WARN, "returned object name is null",
- K(ret), K(s3_base.s3_account_), K(i), K(object_name), K(object_name_len));
- }
- // S3 returns the successfully deleted object in the structure of the basic_string.
- // We use the size of string to construct ObString.
- else if (OB_FAIL(files_to_delete.erase_refactored(ObString(object_name_len, object_name)))) {
- OB_LOG(WARN, "fail to erase succeed deleted object", K(ret),
- K(s3_base.s3_account_), K(i), K(object_name), K(object_name_len));
- } else {
- OB_LOG(DEBUG, "succeed deleting object", K(object_name));
- }
- }
-
- if (FAILEDx(record_failed_files_idx(files_to_delete, failed_files_idx))) {
- OB_LOG(WARN, "fail to record failed del",
- K(ret), K(s3_base.s3_account_), K(n_files_to_delete), K(files_to_delete.size()));
- }
- }
- }
- return ret;
-}
-
-int ObStorageS3Util::write_single_file_(const ObString &uri, const char *buf, const int64_t size)
-{
- int ret = OB_SUCCESS;
- ObStorageS3Writer s3_writer;
- ObExternalIOCounterGuard io_guard;
- if (OB_UNLIKELY(!is_opened_)) {
- ret = OB_OBJECT_STORAGE_IO_ERROR;
- OB_LOG(WARN, "s3 util not opened", K(ret));
- } else if (OB_UNLIKELY(uri.empty() || size < 0) || OB_ISNULL(buf)) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "invalid arguments", K(ret), K(uri), KP(buf), K(size));
- } else if (OB_FAIL(s3_writer.open(uri, storage_info_))) {
- OB_LOG(WARN, "failed to open s3 writer", K(ret), K(uri));
- } else if (OB_FAIL(s3_writer.write(buf, size))) {
- OB_LOG(WARN, "failed to write into s3", K(ret), K(uri), KP(buf), K(size));
- } else if (OB_FAIL(s3_writer.close())) {
- OB_LOG(WARN, "failed to close s3 writer", K(ret), K(uri), KP(buf), K(size));
- }
- return ret;
-}
-
-int ObStorageS3Util::mkdir_(const ObString &uri)
-{
- int ret = OB_SUCCESS;
- OB_LOG(DEBUG, "no need to create dir in s3", K(uri));
- UNUSED(uri);
- return ret;
-}
-
-
-int ObStorageS3Util::list_files_(const ObString &uri, ObBaseDirEntryOperator &op)
-{
- int ret = OB_SUCCESS;
- ObStorageS3Base s3_base;
- ObExternalIOCounterGuard io_guard;
- const char *full_dir_path = NULL;
-
- if (OB_UNLIKELY(!is_opened_)) {
- ret = OB_OBJECT_STORAGE_IO_ERROR;
- OB_LOG(WARN, "s3 util not opened", K(ret));
- } else if (OB_UNLIKELY(uri.empty())) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "invalid arguments", K(ret), K(uri));
- } else if (OB_FAIL(s3_base.inner_open(uri, storage_info_))) {
- OB_LOG(WARN, "fail to open s3 base", K(ret), K(uri));
- } else if (FALSE_IT(full_dir_path = s3_base.object_.ptr())) {
- } else if (OB_UNLIKELY(!is_null_or_end_with_slash(full_dir_path))) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "uri is not terminated with '/'", K(ret), K(uri), K(full_dir_path));
- } else {
- const int64_t full_dir_path_len = get_safe_str_len(full_dir_path);
- Aws::S3::Model::ListObjectsOutcome outcome;
- Aws::String next_marker;
- do {
- if (OB_FAIL(s3_base.do_list_(OB_STORAGE_LIST_MAX_NUM, NULL/*delimiter*/,
- next_marker, outcome))) {
- OB_LOG(WARN, "fail to list s3 objects", K(ret), K(uri));
- } else {
- const char *request_id = outcome.GetResult().GetRequestId().c_str();
- const Aws::Vector &contents = outcome.GetResult().GetContents();
- for (int64_t i = 0; OB_SUCC(ret) && i < contents.size(); i++) {
- const Aws::S3::Model::Object &obj = contents[i];
- const char *obj_path = obj.GetKey().c_str();
- const int64_t obj_path_len = obj.GetKey().size();
-
- // For example, we can use oss console to create a 'read dir', like aaa/bbb/ccc/.
- // When list 'aaa/bbb/ccc/' this dir, we will get it self, that means we will get
- // a object whose name length is same with its parent dir length.
- if (OB_ISNULL(obj_path) || OB_UNLIKELY(false == ObString(obj_path).prefix_match(full_dir_path))) {
- ret = OB_OBJECT_STORAGE_IO_ERROR;
- OB_LOG(WARN, "returned object prefix not match",
- K(ret), K(request_id), K(obj_path), K(full_dir_path), K(uri));
- } else if (OB_UNLIKELY(obj_path_len == full_dir_path_len)) {
- // skip
- OB_LOG(INFO, "exist object path length is same with dir path length",
- K(request_id), K(obj_path), K(full_dir_path), K(full_dir_path_len));
- } else if (OB_FAIL(handle_listed_object(op, obj_path + full_dir_path_len,
- obj_path_len - full_dir_path_len,
- obj.GetSize()))) {
- OB_LOG(WARN, "fail to handle listed s3 object", K(ret), K(request_id),
- K(obj_path), K(obj_path_len), K(full_dir_path), K(full_dir_path_len), K(uri));
- }
- } // end for
- if (OB_SUCC(ret) && outcome.GetResult().GetIsTruncated()) {
- // We cannot set next_marker to outcome.GetResult().GetNextMarker() directly
- // because GetNextMarker() might return empty data.
- // Below is the description of next marker from the S3 official documentation:
- // This element is returned only if you have the delimiter request parameter specified.
- // If the response does not include the NextMarker element and it is truncated,
- // you can use the value of the last Key element in the response
- // as the marker parameter in the subsequent request to get the next set of object keys.
- if (contents.size() > 0) {
- next_marker = contents[contents.size() - 1].GetKey();
- } else {
- // if result is truncated, contents should not be empty
- ret = OB_ERR_UNEXPECTED;
- OB_LOG(WARN, "listed s3 objects are empty", K(ret), K(request_id), K(contents.size()));
- }
- }
- }
- } while (OB_SUCC(ret) && outcome.GetResult().GetIsTruncated());
- }
- return ret;
-}
-
-int ObStorageS3Util::list_files2_(
- const ObString &uri,
- ObStorageListCtxBase &ctx_base)
-{
- int ret = OB_SUCCESS;
- ObStorageS3Base s3_base;
- ObExternalIOCounterGuard io_guard;
- const char *full_dir_path = NULL;
- ObStorageListObjectsCtx &list_ctx = static_cast(ctx_base);
-
- if (OB_UNLIKELY(!is_opened_)) {
- ret = OB_OBJECT_STORAGE_IO_ERROR;
- OB_LOG(WARN, "s3 util not opened", K(ret));
- } else if (OB_UNLIKELY(uri.empty() || !list_ctx.is_valid())) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "invalid arguments", K(ret), K(uri), K(list_ctx));
- } else if (OB_FAIL(s3_base.inner_open(uri, storage_info_))) {
- OB_LOG(WARN, "fail to open s3 base", K(ret), K(uri));
- } else if (FALSE_IT(full_dir_path = s3_base.object_.ptr())) {
- } else if (OB_UNLIKELY(!is_null_or_end_with_slash(full_dir_path))) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "uri is not terminated with '/'", K(ret), K(uri), K(full_dir_path));
- } else {
- const int64_t full_dir_path_len = get_safe_str_len(full_dir_path);
- Aws::S3::Model::ListObjectsOutcome outcome;
- Aws::String next_marker;
- if (list_ctx.next_token_ != NULL && list_ctx.next_token_[0] != '\0') {
- next_marker.assign(list_ctx.next_token_);
- }
- const int64_t max_list_num = MIN(OB_STORAGE_LIST_MAX_NUM, list_ctx.max_list_num_);
-
- if (OB_FAIL(s3_base.do_list_(max_list_num, NULL/*delimiter*/,
- next_marker, outcome))) {
- OB_LOG(WARN, "fail to list s3 objects", K(ret), K(uri), K(max_list_num));
- } else {
- const char *request_id = outcome.GetResult().GetRequestId().c_str();
- const Aws::Vector &contents = outcome.GetResult().GetContents();
- if (outcome.GetResult().GetIsTruncated()) {
- if (contents.size() > 0) {
- next_marker = contents[contents.size() - 1].GetKey();
- } else {
- ret = OB_ERR_UNEXPECTED;
- OB_LOG(WARN, "listed s3 objects are empty", K(ret), K(request_id), K(contents.size()));
- }
- } else {
- next_marker = "";
- }
-
- if (OB_FAIL(ret)) {
- } else if (contents.size() > list_ctx.max_list_num_) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "can't hold all contents", K(ret), K(request_id), K(list_ctx), K(contents.size()));
- } else if (OB_FAIL(list_ctx.set_next_token(outcome.GetResult().GetIsTruncated(),
- next_marker.c_str(),
- next_marker.length()))) {
- OB_LOG(WARN, "fail to set next token when listing s3 objects", K(ret), K(request_id));
- }
-
- for (int64_t i = 0; OB_SUCC(ret) && (i < contents.size()); i++) {
- const Aws::S3::Model::Object &obj = contents[i];
- const int64_t obj_size = obj.GetSize();
- const char *cur_obj_path = obj.GetKey().c_str(); // object full path
- const int64_t cur_obj_path_len = obj.GetKey().size();
- OB_LOG(DEBUG, "s3 list files content", K(cur_obj_path), K(cur_obj_path_len));
-
- if (OB_ISNULL(cur_obj_path) || OB_UNLIKELY(false == ObString(cur_obj_path).prefix_match(full_dir_path))) {
- ret = OB_OBJECT_STORAGE_IO_ERROR;
- OB_LOG(WARN, "returned object prefix not match",
- K(ret), K(request_id), K(cur_obj_path), K(full_dir_path), K(uri));
- } else if (OB_UNLIKELY(cur_obj_path_len == full_dir_path_len)) {
- // skip
- OB_LOG(INFO, "exist object path length is same with dir path length",
- K(request_id), K(cur_obj_path), K(full_dir_path), K(full_dir_path_len));
- } else if (OB_FAIL(list_ctx.handle_object(cur_obj_path, cur_obj_path_len, obj_size))) {
- OB_LOG(WARN, "fail to add listed s3 obejct meta into ctx",
- K(ret), K(request_id), K(cur_obj_path), K(cur_obj_path_len), K(obj_size));
- }
- } // end for
- }
- }
- return ret;
-}
-
-int ObStorageS3Util::del_dir_(const ObString &uri)
-{
- int ret = OB_SUCCESS;
- OB_LOG(DEBUG, "no need to del dir in s3", K(uri));
- UNUSED(uri);
- return ret;
-}
-
-int ObStorageS3Util::list_directories_(const ObString &uri, ObBaseDirEntryOperator &op)
-{
- int ret = OB_SUCCESS;
- ObStorageS3Base s3_base;
- const char *delimiter = "/";
- const char *full_dir_path = NULL;
- ObExternalIOCounterGuard io_guard;
-
- if (OB_UNLIKELY(!is_opened_)) {
- ret = OB_OBJECT_STORAGE_IO_ERROR;
- OB_LOG(WARN, "s3 util not opened", K(ret));
- } else if (OB_UNLIKELY(uri.empty())) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "invalid arguments", K(ret), K(uri));
- } else if (OB_FAIL(s3_base.inner_open(uri, storage_info_))) {
- OB_LOG(WARN, "fail to open s3 base", K(ret), K(uri));
- } else if (FALSE_IT(full_dir_path = s3_base.object_.ptr())) {
- } else if (OB_UNLIKELY(!is_null_or_end_with_slash(full_dir_path))) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "uri is not terminated with '/'", K(ret), K(uri), K(full_dir_path));
- } else {
- const int64_t full_dir_path_len = get_safe_str_len(full_dir_path);
- Aws::S3::Model::ListObjectsOutcome outcome;
- Aws::String next_marker;
- do {
- if (OB_FAIL(s3_base.do_list_(OB_STORAGE_LIST_MAX_NUM, delimiter,
- next_marker, outcome))) {
- OB_LOG(WARN, "fail to list s3 objects", K(ret), K(uri), K(delimiter));
- } else {
- const char *request_id = outcome.GetResult().GetRequestId().c_str();
- const Aws::Vector &common_prefixes = outcome.GetResult().GetCommonPrefixes();
- for (int64_t i = 0; OB_SUCC(ret) && i < common_prefixes.size(); i++) {
- const Aws::S3::Model::CommonPrefix &tmp_common_prefix = common_prefixes[i];
- // For example,
- // dir1
- // --file1
- // --dir11
- // --file11
- // if we list directories in 'dir1', then full_dir_path == 'dir1/'
- // and listed_dir_full_path == 'dir1/dir11/', which represents the full directory path of 'dir11'
- const char *listed_dir_full_path = tmp_common_prefix.GetPrefix().c_str();
- const int64_t listed_dir_full_path_len = tmp_common_prefix.GetPrefix().size();
- OB_LOG(DEBUG, "s3 list directories", K(i), K(listed_dir_full_path));
-
- if (OB_ISNULL(listed_dir_full_path) || OB_UNLIKELY(false == ObString(listed_dir_full_path).prefix_match(full_dir_path))) {
- ret = OB_OBJECT_STORAGE_IO_ERROR;
- OB_LOG(WARN, "returned object prefix not match",
- K(ret), K(request_id), K(listed_dir_full_path), K(full_dir_path), K(uri));
- } else if (OB_UNLIKELY(!is_end_with_slash(listed_dir_full_path))) {
- ret = OB_OBJECT_STORAGE_IO_ERROR;
- OB_LOG(WARN, "the data has no directory",
- K(ret), K(request_id), K(full_dir_path), K(listed_dir_full_path), K(uri));
- } else if (OB_FAIL(handle_listed_directory(op,
- listed_dir_full_path + full_dir_path_len,
- listed_dir_full_path_len - 1 - full_dir_path_len))) {
- OB_LOG(WARN, "fail to handle s3 directory name", K(ret),
- K(request_id), K(listed_dir_full_path), K(full_dir_path), K(full_dir_path_len));
- }
- } // end for
- if (OB_SUCC(ret) && outcome.GetResult().GetIsTruncated()) {
- next_marker = outcome.GetResult().GetNextMarker();
- if (next_marker.empty()) {
- ret = OB_OBJECT_STORAGE_IO_ERROR;
- OB_LOG(WARN, "when listing s3 directories, next marker should not be empty if result is truncated",
- K(ret), K(request_id), K(outcome.GetResult().GetIsTruncated()), K(next_marker.c_str()));
- }
- }
- }
- } while (OB_SUCC(ret) && outcome.GetResult().GetIsTruncated());
- }
- return ret;
-}
-
-int ObStorageS3Util::is_tagging_(const ObString &uri, bool &is_tagging)
-{
- int ret = OB_SUCCESS;
- ObStorageS3Base s3_base;
- ObExternalIOCounterGuard io_guard;
- is_tagging = false;
- if (OB_UNLIKELY(!is_opened_)) {
- ret = OB_OBJECT_STORAGE_IO_ERROR;
- OB_LOG(WARN, "s3 util not opened", K(ret));
- } else if (OB_UNLIKELY(uri.empty())) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "invalid arguments", K(ret), K(uri));
- } else if (OB_FAIL(s3_base.open(uri, storage_info_))) {
- OB_LOG(WARN, "failed to open s3 base", K(ret), K(uri));
- } else {
- Aws::S3::Model::GetObjectTaggingRequest request;
- request.WithBucket(s3_base.bucket_.ptr()).WithKey(s3_base.object_.ptr());
- Aws::S3::Model::GetObjectTaggingOutcome outcome;
- if (OB_FAIL(s3_base.s3_client_->get_object_tagging(request, outcome))) {
- OB_LOG(WARN, "failed to get s3 object tagging", K(ret));
- } else if (!outcome.IsSuccess()) {
- handle_s3_outcome(outcome, ret);
- // When using the getObjectTagging function to access an OBS object that does not have tags,
- // a NoSuchTagSet error will be returned.
- // The handle_s3_outcome function will translate the NoSuchTagSet error returned by S3
- // into the OB_ITEM_NOT_SETTED internal error
- if (OB_ITEM_NOT_SETTED == ret) {
- ret = OB_SUCCESS;
- is_tagging = false;
- } else {
- OB_LOG(WARN, "failed to get s3 object tagging",
- K(ret), K(uri), K(s3_base.bucket_), K(s3_base.object_));
- }
- } else {
- for (const Aws::S3::Model::Tag &tag : outcome.GetResult().GetTagSet()) {
- if (tag.GetKey() == "delete_mode" && tag.GetValue() == "tagging") {
- is_tagging = true;
- break;
- }
- }
- }
- }
- return ret;
-}
-
-int ObStorageS3Util::del_unmerged_parts_(const ObString &uri)
-{
- int ret = OB_SUCCESS;
- ObStorageS3Base s3_base;
- ObExternalIOCounterGuard io_guard;
- if (OB_UNLIKELY(!is_opened_)) {
- ret = OB_OBJECT_STORAGE_IO_ERROR;
- OB_LOG(WARN, "s3 util not opened", K(ret));
- } else if (OB_UNLIKELY(uri.empty())) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "invalid arguments", K(ret), K(uri));
- } else if (OB_FAIL(s3_base.open(uri, storage_info_))) {
- OB_LOG(WARN, "failed to open s3 base", K(ret), K(uri), KPC_(storage_info));
- } else {
- Aws::S3::Model::ListMultipartUploadsRequest request;
- Aws::S3::Model::ListMultipartUploadsOutcome outcome;
- request.WithBucket(s3_base.bucket_.ptr()).WithPrefix(s3_base.object_.ptr());
- request.SetMaxUploads(OB_STORAGE_LIST_MAX_NUM);
- Aws::S3::Model::AbortMultipartUploadRequest abort_request;
- Aws::S3::Model::AbortMultipartUploadOutcome abort_outcome;
- abort_request.WithBucket(s3_base.bucket_.ptr());
- do {
- if (OB_FAIL(s3_base.s3_client_->list_multipart_uploads(request, outcome))) {
- OB_LOG(WARN, "failed to list s3 multipart uploads", K(ret));
- } else if (!outcome.IsSuccess()) {
- handle_s3_outcome(outcome, ret);
- OB_LOG(WARN, "failed to list s3 multipart uploads",
- K(ret), K(uri), K(s3_base.bucket_), K(s3_base.object_));
- } else {
- const char *list_request_id = outcome.GetResult().GetRequestId().c_str();
- const Aws::Vector &uploads = outcome.GetResult().GetUploads();
- for (int64_t i = 0; OB_SUCC(ret) && i < uploads.size(); i++) {
- const Aws::String &obj = uploads[i].GetKey();
- const Aws::String &upload_id = uploads[i].GetUploadId();
- abort_request.WithKey(obj).WithUploadId(upload_id);
- OB_LOG(DEBUG, "list s3 multipat upload",
- K(ret), K(s3_base.bucket_), K(obj.c_str()), K(upload_id.c_str()));
-
- if (OB_FAIL(s3_base.s3_client_->abort_multipart_upload(abort_request, abort_outcome))) {
- OB_LOG(WARN, "failed to abort s3 multipart upload", K(ret),
- K(list_request_id), K(s3_base.bucket_), K(obj.c_str()), K(upload_id.c_str()));
- } else if (!abort_outcome.IsSuccess()) {
- handle_s3_outcome(abort_outcome, ret);
- OB_LOG(WARN, "failed to abort s3 multipart upload", K(ret),
- K(list_request_id), K(s3_base.bucket_), K(obj.c_str()), K(upload_id.c_str()));
- } else {
- OB_LOG(INFO, "succeed to abort s3 multipart upload",
- K(s3_base.bucket_), K(obj.c_str()), K(upload_id.c_str()));
- }
- }
- request.SetKeyMarker(outcome.GetResult().GetNextKeyMarker());
- request.SetUploadIdMarker(outcome.GetResult().GetNextUploadIdMarker());
- }
- } while (OB_SUCC(ret) && outcome.GetResult().GetIsTruncated());
- }
- return ret;
-}
-
-/*--------------------------------ObStorageS3AppendWriter--------------------------------*/
-ObStorageS3AppendWriter::ObStorageS3AppendWriter()
- : ObStorageS3Writer(),
- storage_info_(NULL)
-{
-}
-
-ObStorageS3AppendWriter::~ObStorageS3AppendWriter()
-{
- close();
-}
-
-int ObStorageS3AppendWriter::open_(const ObString &uri, ObObjectStorageInfo *storage_info)
-{
- int ret = OB_SUCCESS;
- ObExternalIOCounterGuard io_guard;
- if (OB_UNLIKELY(is_opened_)) {
- ret = OB_OPEN_TWICE;
- OB_LOG(WARN, "s3 append writer already open, cannot open again", K(ret), K(uri));
- } else if (OB_FAIL(ObStorageS3Writer::open(uri, storage_info))) {
- OB_LOG(WARN, "failed to open in s3 base", K(ret), K(uri), KPC(storage_info));
- } else {
- is_opened_ = true;
- storage_info_ = storage_info;
- }
- return ret;
-}
-
-int ObStorageS3AppendWriter::write_(const char *buf, const int64_t size)
-{
- int ret = OB_NOT_SUPPORTED;
- UNUSED(buf);
- UNUSED(size);
- return ret;
-}
-
-int ObStorageS3AppendWriter::pwrite_(const char *buf, const int64_t size, const int64_t offset)
-{
- int ret = OB_SUCCESS;
- char fragment_name[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 };
- Aws::S3::Model::PutObjectRequest request;
- ObExternalIOCounterGuard io_guard;
- if(OB_UNLIKELY(!is_opened_)) {
- ret = OB_NOT_INIT;
- OB_LOG(WARN, "s3 append writer cannot write before it is not opened", K(ret));
- } else if(OB_ISNULL(buf) || OB_UNLIKELY(size <= 0 || offset < 0)) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "invalid arguments", K(ret), KP(buf), K(size), K(offset));
- } else {
- // write the format file when writing the first fragment because the appender may open multiple times
- if (offset == 0) {
- if (OB_FAIL(construct_fragment_full_name(object_, OB_ADAPTIVELY_APPENDABLE_FORMAT_META,
- fragment_name, sizeof(fragment_name)))) {
- OB_LOG(WARN, "failed to construct s3 mock append object foramt name",
- K(ret), K_(bucket), K_(object));
- } else if (OB_FAIL(write_obj_(fragment_name, OB_ADAPTIVELY_APPENDABLE_FORMAT_CONTENT_V1,
- strlen(OB_ADAPTIVELY_APPENDABLE_FORMAT_CONTENT_V1)))) {
- OB_LOG(WARN, "fail to write s3 mock append object format file", K(ret), K(fragment_name));
- }
- }
-
- if (OB_FAIL(ret)) {
- } else if (OB_FAIL(construct_fragment_full_name(object_, offset, offset + size,
- fragment_name, sizeof(fragment_name)))) {
- // fragment_name: /xxx/xxx/appendable_obj_name/start-end,
- // the data range covered by this file is from start to end == [start, end), include start not include end
- // start == offset, end == offset + size
- // fragment length == size
- OB_LOG(WARN, "failed to set fragment name for s3 append writer",
- K(ret), K_(bucket), K_(object), K(buf), K(size), K(offset));
- } else if (OB_FAIL(write_obj_(fragment_name, buf, size))) {
- OB_LOG(WARN, "fail to append a fragment into s3",
- K(ret), K_(bucket), K_(object), K(fragment_name), K(size));
- }
- }
-
- if (OB_SUCC(ret)) {
- OB_LOG(DEBUG, "succeed to append a fragment into s3",
- K_(bucket), K_(object), K(fragment_name), K(size), K(offset));
- }
- return ret;
-}
-
-int ObStorageS3AppendWriter::close_()
-{
- int ret = OB_SUCCESS;
-
- is_opened_ = false;
- storage_info_ = NULL;
- reset();
- return ret;
-}
-
-int64_t ObStorageS3AppendWriter::get_length() const
-{
- int ret = OB_NOT_SUPPORTED;
- OB_LOG(WARN, "s3 appender do not support get length now", K(ret), K_(bucket), K_(object));
- return -1;
-}
-
-/*--------------------------------ObStorageS3MultiPartWriter--------------------------------*/
-ObStorageS3MultiPartWriter::ObStorageS3MultiPartWriter()
- : ObStorageS3Writer(),
- base_buf_(NULL), base_buf_pos_(-1),
- upload_id_(NULL),
- partnum_(0)
-{}
-
-ObStorageS3MultiPartWriter::~ObStorageS3MultiPartWriter()
-{
- close();
-}
-
-void ObStorageS3MultiPartWriter::reset()
-{
- is_opened_ = false;
- base_buf_ = NULL;
- base_buf_pos_ = -1;
- upload_id_ = NULL;
- partnum_ = 0;
- file_length_ = -1;
- reset_part_info();
- ObStorageS3Base::reset();
-}
-
-int ObStorageS3MultiPartWriter::open_(const ObString &uri, ObObjectStorageInfo *storage_info)
-{
- int ret = OB_SUCCESS;
- ObExternalIOCounterGuard io_guard;
- if (OB_UNLIKELY(is_opened_)) {
- ret = OB_OBJECT_STORAGE_IO_ERROR;
- OB_LOG(WARN, "s3 multipart writer already opened, cannot open again", K(ret), K(uri));
- } else if (OB_FAIL(ObStorageS3Writer::open(uri, storage_info))) {
- OB_LOG(WARN, "failed to open in s3 base", K(ret), K(uri));
- } else if (OB_FAIL(ObStoragePartInfoHandler::init())) {
- OB_LOG(WARN, "fail to init part info handler", K(ret), K(uri));
- } else {
- Aws::S3::Model::CreateMultipartUploadRequest request;
- request.WithBucket(bucket_.ptr()).WithKey(object_.ptr());
- Aws::S3::Model::CreateMultipartUploadOutcome outcome;
-
- if (OB_FAIL(set_request_checkusum_algorithm(request, checksum_type_))) {
- OB_LOG(WARN, "fail to set checksum algorithm", K(ret), K_(checksum_type));
- } else if (OB_FAIL(s3_client_->create_multipart_upload(request, outcome))) {
- OB_LOG(WARN, "failed to create s3 multipart upload", K(ret));
- } else if (!outcome.IsSuccess()) {
- handle_s3_outcome(outcome, ret);
- OB_LOG(WARN, "failed to create multipart upload for s3",
- K(ret), K_(bucket), K_(object));
- } else {
- const Aws::String &upload_id = outcome.GetResult().GetUploadId();
- if (OB_ISNULL(upload_id_ = static_cast(allocator_.alloc(upload_id.size() + 1)))) {
- ret = OB_ALLOCATE_MEMORY_FAILED;
- OB_LOG(WARN, "failed to alloc buf for s3 multipartupload upload id",
- K(ret), K(upload_id.size()));
- } else if (OB_ISNULL(base_buf_ =
- static_cast(allocator_.alloc(S3_MULTIPART_UPLOAD_BUFFER_SIZE)))) {
- ret = OB_ALLOCATE_MEMORY_FAILED;
- OB_LOG(WARN, "failed to alloc buf for s3 multipartupload buffer",
- K(ret), K(S3_MULTIPART_UPLOAD_BUFFER_SIZE));
- } else {
- STRNCPY(upload_id_, upload_id.c_str(), upload_id.size());
- upload_id_[upload_id.size()] = '\0';
- base_buf_pos_ = 0;
- file_length_ = 0;
- is_opened_ = true;
- }
- }
- }
-
- if (OB_FAIL(ret)) {
- reset();
- }
- return ret;
-}
-
-int ObStorageS3MultiPartWriter::write_(const char *buf, const int64_t size)
-{
- int ret = OB_SUCCESS;
- int64_t fill_size = 0;
- int64_t buf_pos = 0;
- ObExternalIOCounterGuard io_guard;
- if (OB_UNLIKELY(!is_opened_)) {
- ret = OB_NOT_INIT;
- OB_LOG(WARN, "s3 multipart writer not opened", K(ret));
- } else if (OB_ISNULL(buf) || OB_UNLIKELY(size < 0)) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "buf is NULL or size is invalid", K(ret), KP(buf), K(size));
- }
-
- while (OB_SUCC(ret) && buf_pos != size) {
- fill_size = MIN(S3_MULTIPART_UPLOAD_BUFFER_SIZE - base_buf_pos_, size - buf_pos);
- memcpy(base_buf_ + base_buf_pos_, buf + buf_pos, fill_size);
- base_buf_pos_ += fill_size;
- buf_pos += fill_size;
- if (base_buf_pos_ == S3_MULTIPART_UPLOAD_BUFFER_SIZE) {
- if (OB_FAIL(write_single_part_())) {
- OB_LOG(WARN, "failed to write single s3 part", K(ret), K_(bucket), K_(object));
- } else {
- base_buf_pos_ = 0;
- }
- }
- }
-
- if (OB_SUCCESS == ret) {
- file_length_ += size;
- }
- return ret;
-}
-
-int ObStorageS3MultiPartWriter::pwrite_(const char *buf, const int64_t size, const int64_t offset)
-{
- UNUSED(offset);
- return write(buf, size);
-}
-
-static int construct_completed_multipart_upload(
- const ObStoragePartInfoHandler::PartInfoMap &part_info_map,
- const ObStorageChecksumType checksum_type,
- Aws::S3::Model::CompletedMultipartUpload &completed_multipart_upload)
-{
- int ret = OB_SUCCESS;
- const int64_t max_part_id = part_info_map.size();
- ObStoragePartInfoHandler::PartInfo tmp_part_info;
-
- // allow to be empty parts
- if (OB_UNLIKELY(max_part_id < 0)) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "invalid args", K(ret), K(max_part_id));
- }
- for (int64_t i = 1; OB_SUCC(ret) && i <= max_part_id; i++) {
- if (OB_FAIL(part_info_map.get_refactored(i, tmp_part_info))) {
- if (OB_HASH_NOT_EXIST == ret) {
- OB_LOG(WARN, "part ids should be 1 ~ max_part_id", K(ret), K(i), K(max_part_id));
- } else {
- OB_LOG(WARN, "fail to get part info", K(ret), K(i), K(max_part_id));
- }
- } else if (OB_ISNULL(tmp_part_info.first)) { // etag
- ret = OB_ERR_UNEXPECTED;
- OB_LOG(WARN, "etag is null", K(ret), K(i), K(max_part_id));
- } else {
- Aws::S3::Model::CompletedPart tmp_part;
- if (OB_FAIL(set_completed_part_checksum(tmp_part, checksum_type, tmp_part_info.second))) {
- OB_LOG(WARN, "fail to set checksum", K(ret), K(i), K(checksum_type), K(max_part_id));
- } else {
- tmp_part.WithPartNumber(i).WithETag(tmp_part_info.first);
- completed_multipart_upload.AddParts(std::move(tmp_part));
- }
- }
- }
- return ret;
-}
-
-int ObStorageS3MultiPartWriter::complete_()
-{
- int ret = OB_SUCCESS;
- ObExternalIOCounterGuard io_guard;
- if (OB_UNLIKELY(!is_opened_)) {
- ret = OB_NOT_INIT;
- OB_LOG(WARN, "s3 multipart writer cannot compelete before it is opened", K(ret));
- } else if (base_buf_pos_ > 0) {
- if (OB_FAIL(write_single_part_())) {
- OB_LOG(WARN, "failed to upload last part into s3", K(ret), K_(base_buf_pos));
- } else {
- base_buf_pos_ = 0;
- }
- }
-
- Aws::S3::Model::CompletedMultipartUpload completed_multipart_upload;
- if (FAILEDx(construct_completed_multipart_upload(part_info_map_,
- checksum_type_,
- completed_multipart_upload))) {
- OB_LOG(WARN, "fail to construct completed multipart upload", K(ret), K_(checksum_type));
- } else {
- // complete upload
- Aws::S3::Model::CompleteMultipartUploadRequest complete_multipart_upload_request;
- complete_multipart_upload_request.WithBucket(bucket_.ptr()).WithKey(object_.ptr());
- complete_multipart_upload_request.WithUploadId(upload_id_);
- complete_multipart_upload_request.WithMultipartUpload(completed_multipart_upload);
-
- Aws::S3::Model::CompleteMultipartUploadOutcome complete_multipart_upload_outcome;
- if (OB_FAIL(ret)) {
- } else if (OB_UNLIKELY(size() == 0)) {
- // If 'complete' without uploading any data, S3 will return the error
- // 'InvalidRequest，You must specify at least one part'
- // write an empty object instead
- if (OB_FAIL(write_obj_(object_.ptr(), "", 0))) {
- OB_LOG(WARN, "complete an empty multipart upload, but fail to write an empty object",
- K(ret), K_(partnum), K_(upload_id), K_(bucket), K_(object));
- }
- } else if (OB_FAIL(s3_client_->complete_multipart_upload(complete_multipart_upload_request,
- complete_multipart_upload_outcome))) {
- OB_LOG(WARN, "failed to complete s3 multipart upload",
- K(ret), K_(partnum), K(size()), K_(bucket), K_(object), K_(upload_id));
- } else if (!complete_multipart_upload_outcome.IsSuccess()) {
- handle_s3_outcome(complete_multipart_upload_outcome, ret);
- OB_LOG(WARN, "failed to complete multipart upload for s3",
- K(ret), K_(bucket), K_(object), K_(upload_id), K_(partnum), K(size()));
- }
-
- }
- return ret;
-}
-
-int ObStorageS3MultiPartWriter::close_()
-{
- int ret = OB_SUCCESS;
- ObExternalIOCounterGuard io_guard;
- reset();
- return ret;
-}
-
-int ObStorageS3MultiPartWriter::abort_()
-{
- int ret = OB_SUCCESS;
- ObExternalIOCounterGuard io_guard;
- if (OB_UNLIKELY(!is_opened_)) {
- ret = OB_NOT_INIT;
- OB_LOG(WARN, "s3 multipart writer not opened", K(ret));
- } else {
- Aws::S3::Model::AbortMultipartUploadRequest request;
- request.WithBucket(bucket_.ptr()).WithKey(object_.ptr());
- request.WithUploadId(upload_id_);
- Aws::S3::Model::AbortMultipartUploadOutcome outcome;
- if (OB_FAIL(s3_client_->abort_multipart_upload(request, outcome))) {
- OB_LOG(WARN, "failed to abort s3 multipart upload", K(ret));
- } else if (!outcome.IsSuccess()) {
- handle_s3_outcome(outcome, ret);
- OB_LOG(WARN, "failed to abort s3 multipart upload",
- K(ret), K_(bucket), K_(object), K_(partnum), K_(upload_id));
- }
- }
- return ret;
-}
-
-int ObStorageS3MultiPartWriter::write_single_part_()
-{
- // TODO @fangdan: compress data
- int ret = OB_SUCCESS;
- ObExternalIOCounterGuard io_guard;
- ++partnum_; // partnum is between 1 and 10000
- if (OB_UNLIKELY(!is_opened_)) {
- ret = OB_NOT_INIT;
- OB_LOG(WARN, "s3 multipart writer not opened", K(ret));
- } else if (partnum_ > MAX_S3_PART_NUM) {
- ret = OB_OUT_OF_ELEMENT;
- OB_LOG(WARN, "out of s3 part num effective range", K(ret), K_(partnum), K(MAX_S3_PART_NUM));
- } else {
- Aws::S3::Model::UploadPartRequest request;
- request.WithBucket(bucket_.ptr()).WithKey(object_.ptr());
- request.WithPartNumber(partnum_).WithUploadId(upload_id_);
- std::shared_ptr data_stream =
- Aws::MakeShared(S3_SDK);
- data_stream->write(base_buf_, base_buf_pos_);
- data_stream->flush();
- request.SetBody(data_stream);
- request.SetContentLength(static_cast(request.GetBody()->tellp()));
-
- Aws::S3::Model::UploadPartOutcome outcome;
- if (OB_FAIL(set_request_checkusum_algorithm(request, checksum_type_))) {
- OB_LOG(WARN, "fail to set checksum algorithm", K(ret),
- K_(checksum_type), K_(bucket), K_(object), K_(upload_id));
- } else if (OB_FAIL(s3_client_->upload_part(request, outcome))) {
- OB_LOG(WARN, "failed to upload s3 multipart", K(ret), K_(bucket), K_(object), K_(upload_id));
- } else if (!outcome.IsSuccess()) {
- handle_s3_outcome(outcome, ret);
- OB_LOG(WARN, "failed to upload part into s3", K(ret), K_(bucket), K_(object), K_(partnum));
- } else {
- OB_LOG(DEBUG, "succed upload a part into s3", K_(partnum), K_(bucket), K_(object));
-
- const char *etag = outcome.GetResult().GetETag().c_str();
- const char *checksum = nullptr;
- if (OB_FAIL(get_completed_part_checksum(outcome.GetResult(), checksum_type_, checksum))) {
- OB_LOG(WARN, "fail to get completed part checksum", K(ret),
- K_(checksum_type), K_(bucket), K_(object), K_(upload_id));
- } else if (OB_FAIL(add_part_info(partnum_, etag, checksum))) {
- OB_LOG(WARN, "fail to add part info", K(ret),
- K_(bucket), K_(object), K_(partnum), K(etag));
- }
- }
- }
- return ret;
-}
-
-ObStorageParallelS3MultiPartWriter::ObStorageParallelS3MultiPartWriter()
- : ObStorageS3Writer(),
- upload_id_(nullptr)
-{}
-
-ObStorageParallelS3MultiPartWriter::~ObStorageParallelS3MultiPartWriter()
-{
- close();
-}
-
-void ObStorageParallelS3MultiPartWriter::reset()
-{
- is_opened_ = false;
- upload_id_ = nullptr;
- reset_part_info();
- ObStorageS3Base::reset();
-}
-
-int ObStorageParallelS3MultiPartWriter::open_(const ObString &uri, ObObjectStorageInfo *storage_info)
-{
- int ret = OB_SUCCESS;
- ObExternalIOCounterGuard io_guard;
- if (OB_UNLIKELY(is_opened_)) {
- ret = OB_OBJECT_STORAGE_IO_ERROR;
- OB_LOG(WARN, "s3 multipart writer already opened, cannot open again", K(ret), K(uri));
- } else if (OB_FAIL(ObStorageS3Writer::open(uri, storage_info))) {
- OB_LOG(WARN, "failed to open in s3 base writer", K(ret), K(uri), KPC(storage_info));
- } else if (OB_FAIL(ObStoragePartInfoHandler::init())) {
- OB_LOG(WARN, "fail to init part info handler", K(ret), K(uri));
- } else {
- Aws::S3::Model::CreateMultipartUploadRequest request;
- request.WithBucket(bucket_.ptr()).WithKey(object_.ptr());
- Aws::S3::Model::CreateMultipartUploadOutcome outcome;
-
- if (OB_FAIL(set_request_checkusum_algorithm(request, checksum_type_))) {
- OB_LOG(WARN, "fail to set checksum algorithm", K(ret), K_(checksum_type));
- } else if (OB_FAIL(s3_client_->create_multipart_upload(request, outcome))) {
- OB_LOG(WARN, "failed to create s3 multipart upload", K(ret));
- } else if (!outcome.IsSuccess()) {
- handle_s3_outcome(outcome, ret);
- OB_LOG(WARN, "failed to create multipart upload for s3",
- K(ret), K_(bucket), K_(object));
- } else {
- const Aws::String &upload_id = outcome.GetResult().GetUploadId();
- if (OB_UNLIKELY(upload_id.empty())) {
- ret = OB_OBJECT_STORAGE_IO_ERROR;
- OB_LOG(WARN, "returned upload_id is empty", K(ret), K(upload_id.size()));
- } else if (OB_ISNULL(upload_id_ = static_cast(allocator_.alloc(upload_id.size() + 1)))) {
- ret = OB_ALLOCATE_MEMORY_FAILED;
- OB_LOG(WARN, "failed to alloc buf for s3 multipartupload upload id",
- K(ret), K(upload_id.size()));
- } else {
- STRNCPY(upload_id_, upload_id.c_str(), upload_id.size());
- upload_id_[upload_id.size()] = '\0';
- }
- }
- }
-
- if (OB_FAIL(ret)) {
- reset();
- }
- return ret;
-}
-
-int ObStorageParallelS3MultiPartWriter::upload_part_(
- const char *buf, const int64_t size, const int64_t part_id)
-{
- int ret = OB_SUCCESS;
- ObExternalIOCounterGuard io_guard;
- if (OB_UNLIKELY(!is_opened_)) {
- ret = OB_NOT_INIT;
- OB_LOG(WARN, "s3 multipart writer not opened", K(ret));
- } else if (OB_UNLIKELY(part_id < 1 || part_id > MAX_S3_PART_NUM)) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "out of s3 part num effective range", K(ret), K(part_id), K(MAX_S3_PART_NUM));
- } else if (OB_ISNULL(buf) || OB_UNLIKELY(size <= 0)) {
- ret = OB_INVALID_ARGUMENT;
- OB_LOG(WARN, "invalid argument", K(ret), KP(buf), K(size));
- } else {
- Aws::S3::Model::UploadPartRequest request;
- request.WithBucket(bucket_.ptr()).WithKey(object_.ptr());
- request.WithPartNumber(part_id).WithUploadId(upload_id_);
- std::shared_ptr data_stream =
- Aws::MakeShared(S3_SDK);
- data_stream->write(buf, size);
- data_stream->flush();
- request.SetBody(data_stream);
- request.SetContentLength(static_cast(size));
-
- Aws::S3::Model::UploadPartOutcome outcome;
- if (OB_FAIL(set_request_checkusum_algorithm(request, checksum_type_))) {
- OB_LOG(WARN, "fail to set checksum algorithm", K(ret),
- K_(checksum_type), K_(bucket), K_(object), K(part_id), K_(upload_id));
- } else if (OB_FAIL(s3_client_->upload_part(request, outcome))) {
- OB_LOG(WARN, "failed to upload s3 multipart", K(ret),
- K_(checksum_type), K_(bucket), K_(object), K(part_id), K_(upload_id));
- } else if (!outcome.IsSuccess()) {
- handle_s3_outcome(outcome, ret);
- OB_LOG(WARN, "failed to upload part into s3",
- K(ret), K_(bucket), K_(object), K(part_id), K_(upload_id));
- } else {
- OB_LOG(DEBUG, "succed upload a part into s3", K(part_id), K_(bucket), K_(object));
-
- const char *etag = outcome.GetResult().GetETag().c_str();
- const char *checksum = nullptr;
- if (OB_FAIL(get_completed_part_checksum(outcome.GetResult(), checksum_type_, checksum))) {
- OB_LOG(WARN, "fail to get completed part checksum", K(ret),
- K_(checksum_type), K_(bucket), K_(object), K_(upload_id));
- } else if (OB_FAIL(add_part_info(part_id, etag, checksum))) {
- OB_LOG(WARN, "fail to add part info", K(ret),
- K_(bucket), K_(object), K(part_id), K(etag));
- }
- }
- }
- return ret;
-}
-
-int ObStorageParallelS3MultiPartWriter::complete_()
-{
- int ret = OB_SUCCESS;
- Aws::S3::Model::CompletedMultipartUpload completed_multipart_upload;
- ObExternalIOCounterGuard io_guard;
- if (OB_UNLIKELY(!is_opened_)) {
- ret = OB_NOT_INIT;
- OB_LOG(WARN, "s3 multipart writer cannot compelete before it is opened", K(ret));
- } else if (FAILEDx(construct_completed_multipart_upload(part_info_map_,
- checksum_type_,
- completed_multipart_upload))) {
- OB_LOG(WARN, "fail to construct completed multipart upload", K(ret), K_(checksum_type));
- } else {
- // complete upload
- Aws::S3::Model::CompleteMultipartUploadRequest complete_multipart_upload_request;
- complete_multipart_upload_request.WithBucket(bucket_.ptr()).WithKey(object_.ptr());
- complete_multipart_upload_request.WithUploadId(upload_id_);
- complete_multipart_upload_request.WithMultipartUpload(completed_multipart_upload);
-
- Aws::S3::Model::CompleteMultipartUploadOutcome complete_multipart_upload_outcome;
- if (OB_FAIL(ret)) {
- } else if (OB_UNLIKELY(size() == 0)) {
- // If 'complete' without uploading any data, S3 will return the error
- // 'InvalidRequest，You must specify at least one part'
- // write an empty object instead
- if (OB_FAIL(write_obj_(object_.ptr(), "", 0))) {
- OB_LOG(WARN, "complete an empty multipart upload, but fail to write an empty object",
- K(ret), K(size()), K_(upload_id), K_(bucket), K_(object));
- }
- } else if (OB_FAIL(s3_client_->complete_multipart_upload(complete_multipart_upload_request,
- complete_multipart_upload_outcome))) {
- OB_LOG(WARN, "failed to complete s3 multipart upload",
- K(ret), K_(bucket), K_(object), K_(upload_id), K(size()));
- } else if (!complete_multipart_upload_outcome.IsSuccess()) {
- handle_s3_outcome(complete_multipart_upload_outcome, ret);
- OB_LOG(WARN, "failed to complete multipart upload for s3",
- K(ret), K_(bucket), K_(object), K_(upload_id), K(size()));
- }
- }
- return ret;
-}
-
-int ObStorageParallelS3MultiPartWriter::abort_()
-{
- int ret = OB_SUCCESS;
- ObExternalIOCounterGuard io_guard;
- if (OB_UNLIKELY(!is_opened_)) {
- ret = OB_NOT_INIT;
- OB_LOG(WARN, "s3 multipart writer not opened", K(ret));
- } else {
- Aws::S3::Model::AbortMultipartUploadRequest request;
- request.WithBucket(bucket_.ptr()).WithKey(object_.ptr());
- request.WithUploadId(upload_id_);
- Aws::S3::Model::AbortMultipartUploadOutcome outcome;
- if (OB_FAIL(s3_client_->abort_multipart_upload(request, outcome))) {
- OB_LOG(WARN, "failed to abort s3 multipart upload", K(ret));
- } else if (!outcome.IsSuccess()) {
- handle_s3_outcome(outcome, ret);
- OB_LOG(WARN, "failed to abort s3 multipart upload",
- K(ret), K_(bucket), K_(object), K_(upload_id));
- }
- }
- return ret;
-}
-
-int ObStorageParallelS3MultiPartWriter::close_()
-{
- int ret = OB_SUCCESS;
- ObExternalIOCounterGuard io_guard;
- reset();
- return ret;
-}
-
-} // common
-} // oceanbas
diff --git a/deps/oblib/src/lib/restore/ob_storage_s3_base.h b/deps/oblib/src/lib/restore/ob_storage_s3_base.h
deleted file mode 100644
index 627c387a2..000000000
--- a/deps/oblib/src/lib/restore/ob_storage_s3_base.h
+++ /dev/null
@@ -1,715 +0,0 @@
-/*
- * Copyright (c) 2025 OceanBase.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef SRC_LIBRARY_SRC_LIB_RESTORE_OB_STORAGE_S3_BASE_H_
-#define SRC_LIBRARY_SRC_LIB_RESTORE_OB_STORAGE_S3_BASE_H_
-
-#include
-#ifdef __linux__
-#include
-#elif defined(__APPLE__)
-#include // malloc is in stdlib.h on macOS
-#endif
-#include "lib/restore/ob_i_storage.h"
-#include "lib/container/ob_array.h"
-#include "lib/container/ob_se_array.h"
-#include "lib/container/ob_array_iterator.h"
-#include "lib/container/ob_se_array_iterator.h"
-#include "lib/allocator/ob_vslice_alloc.h"
-#include
-#include
-#include "lib/utility/ob_tracepoint.h"
-
-#pragma push_macro("private")
-#undef private
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#pragma pop_macro("private")
-
-namespace oceanbase
-{
-namespace common
-{
-
-// Before using s3, you need to initialize s3 enviroment.
-// Thread safe guaranteed by user.
-int init_s3_env();
-
-// You need to clean s3 resource when not use cos any more.
-// Thread safe guaranteed by user.
-void fin_s3_env();
-
-static constexpr int64_t S3_CONNECT_TIMEOUT_MS = 10 * 1000;
-static constexpr int64_t S3_REQUEST_TIMEOUT_MS = 10 * 1000;
-static constexpr int64_t MAX_S3_CONNECTIONS_PER_CLIENT = 128;
-static constexpr int64_t STOP_S3_TIMEOUT_US = 10 * 1000L; // 10ms
-
-static constexpr int MAX_S3_REGION_LENGTH = 128;
-static constexpr int MAX_S3_ENDPOINT_LENGTH = 256;
-static constexpr int MAX_S3_ACCESS_ID_LENGTH = 256; // ak, access key id
-static constexpr int MAX_S3_SECRET_KEY_LENGTH = 256; // sk, secret key
-static constexpr int MAX_S3_CLIENT_NUM = 97;
-static constexpr int MAX_S3_PART_NUM = 10000;
-static constexpr int64_t S3_MULTIPART_UPLOAD_BUFFER_SIZE = 8 * 1024 * 1024L;
-
-static constexpr char OB_STORAGE_S3_ALLOCATOR[] = "StorageS3";
-static constexpr char S3_SDK[] = "S3SDK";
-
-// TODO @fangdan: Validate the effectiveness of the ZeroCopyStreambuf
-class ZeroCopyStreambuf : public std::streambuf
-{
-public:
- ZeroCopyStreambuf(const char *base, std::size_t size)
- {
- setg(const_cast(base), const_cast(base), const_cast(base) + size);
- }
-
-protected:
- virtual std::streampos seekoff(
- std::streamoff off,
- std::ios_base::seekdir dir,
- std::ios_base::openmode which = std::ios_base::in | std::ios_base::out) override
- {
- std::streampos result = std::streampos(std::streamoff(-1));
- if (which & std::ios_base::in) {
- char *new_pos = gptr();
- if (dir == std::ios_base::beg) {
- new_pos = eback() + off;
- } else if (dir == std::ios_base::end) {
- new_pos = egptr() + off;
- } else if (dir == std::ios_base::cur) {
- new_pos = gptr() + off;
- }
-
- if (new_pos >= eback() && new_pos <= egptr()) {
- setg(eback(), new_pos, egptr());
- result = gptr() - eback();
- }
- }
- return result;
- }
-
- virtual std::streampos seekpos(
- std::streampos pos,
- std::ios_base::openmode which = std::ios_base::in | std::ios_base::out) override
- {
- return seekoff(std::streamoff(pos), std::ios_base::beg, which);
- }
-};
-
-int set_max_s3_client_idle_duration_us(const int64_t duration_us);
-int64_t get_max_s3_client_idle_duration_us();
-
-struct ObS3Account
-{
- ObS3Account();
- ~ObS3Account();
- void reset();
- bool is_valid() const { return is_valid_; }
- int64_t hash() const;
- TO_STRING_KV(K_(is_valid), K_(delete_mode), K_(region), K_(endpoint), K_(access_id), KP_(secret_key), K_(sts_token), K_(addressing_model));
-
- int parse_from(const char *storage_info_str, const int64_t size);
-
- bool is_valid_;
- int64_t delete_mode_;
- char region_[MAX_S3_REGION_LENGTH]; // region of endpoint
- char endpoint_[MAX_S3_ENDPOINT_LENGTH];
- char access_id_[MAX_S3_ACCESS_ID_LENGTH]; // ak
- char secret_key_[MAX_S3_SECRET_KEY_LENGTH]; // sk
- ObSTSToken sts_token_;
- ObStorageAddressingModel addressing_model_;
-};
-
-class ObS3MemoryManager : public Aws::Utils::Memory::MemorySystemInterface
-{
- enum
- {
- N_WAY = 32,
- DEFAULT_BLOCK_SIZE = 128 * 1024, // 128KB
- };
-public:
- ObS3MemoryManager() : attr_(), mem_limiter_(), allocator_()
- {
- attr_.label_ = S3_SDK;
- }
- virtual ~ObS3MemoryManager() {}
- // when aws init/shutdown, it will execute like this: init/shutdown_memory_system->Begin()/End()
- virtual void Begin() override {}
- virtual void End() override {}
- virtual void *AllocateMemory(std::size_t blockSize,
- std::size_t alignment, const char *allocationTag = nullptr) override;
- virtual void FreeMemory(void *memoryPtr) override;
- int init();
-private:
- ObMemAttr attr_;
- // mem_limiter_ is the allocator that actually allocates memory internally for allocator_
- // which can set the limit of memory that can be requestd, default to infinity
- // mem_limiter_ is passed in during allocator_ init
- ObBlockAllocMgr mem_limiter_;
- ObVSliceAlloc allocator_;
-};
-
-class ObS3Logger : public Aws::Utils::Logging::LogSystemInterface
-{
-public:
- ObS3Logger() {}
- virtual ~ObS3Logger() {}
- // Gets the currently configured log level for this logger.
- virtual Aws::Utils::Logging::LogLevel GetLogLevel(void) const override;
- // Does a printf style output to the output stream. Don't use this, it's unsafe. See LogStream
- virtual void Log(Aws::Utils::Logging::LogLevel logLevel, const char* tag, const char* formatStr, ...) override;
- virtual void vaLog(Aws::Utils::Logging::LogLevel logLevel, const char* tag, const char* formatStr, va_list args);
- // Writes the stream to the output stream.
- virtual void LogStream(Aws::Utils::Logging::LogLevel logLevel, const char* tag, const Aws::OStringStream &messageStream) override;
- // Writes any buffered messages to the underlying device if the logger supports buffering.
- virtual void Flush() override {}
-};
-
-class ObS3Client
-{
-public:
- ObS3Client();
- virtual ~ObS3Client();
- int init(const ObS3Account &account);
- int check_status();
- void destroy();
- bool is_stopped() const;
- bool try_stop(const int64_t timeout = STOP_S3_TIMEOUT_US);
- void stop();
- void increase();
- void release();
- TO_STRING_KV(KP(&lock_), K_(is_inited), K_(ref_cnt), K_(last_modified_ts), KP(client_));
-
- int head_object(const Aws::S3::Model::HeadObjectRequest &request,
- Aws::S3::Model::HeadObjectOutcome &outcome);
- int put_object(const Aws::S3::Model::PutObjectRequest &request,
- Aws::S3::Model::PutObjectOutcome &outcome);
- int get_object(const Aws::S3::Model::GetObjectRequest &request,
- Aws::S3::Model::GetObjectOutcome &outcome);
- int delete_object(const Aws::S3::Model::DeleteObjectRequest &request,
- Aws::S3::Model::DeleteObjectOutcome &outcome);
- int delete_objects(const Aws::S3::Model::DeleteObjectsRequest &request,
- Aws::S3::Model::DeleteObjectsOutcome &outcome);
- int put_object_tagging(const Aws::S3::Model::PutObjectTaggingRequest &request,
- Aws::S3::Model::PutObjectTaggingOutcome &outcome);
- int list_objects(const Aws::S3::Model::ListObjectsRequest &request,
- Aws::S3::Model::ListObjectsOutcome &outcome);
- int get_object_tagging(const Aws::S3::Model::GetObjectTaggingRequest &request,
- Aws::S3::Model::GetObjectTaggingOutcome &outcome);
- int create_multipart_upload(const Aws::S3::Model::CreateMultipartUploadRequest &request,
- Aws::S3::Model::CreateMultipartUploadOutcome &outcome);
- int complete_multipart_upload(const Aws::S3::Model::CompleteMultipartUploadRequest &request,
- Aws::S3::Model::CompleteMultipartUploadOutcome &outcome);
- int abort_multipart_upload(const Aws::S3::Model::AbortMultipartUploadRequest &request,
- Aws::S3::Model::AbortMultipartUploadOutcome &outcome);
- int upload_part(const Aws::S3::Model::UploadPartRequest &request,
- Aws::S3::Model::UploadPartOutcome &outcome);
- int list_multipart_uploads(const Aws::S3::Model::ListMultipartUploadsRequest &request,
- Aws::S3::Model::ListMultipartUploadsOutcome &outcome);
-
-private:
- int init_s3_client_configuration_(const ObS3Account &account,
- Aws::S3::S3ClientConfiguration &config);
-
- template
- using S3OperationFunc = OutcomeType (Aws::S3::S3Client::*)(const RequestType &) const;
-
- template
- int do_s3_operation_(S3OperationFunc s3_op_func,
- const RequestType &request, OutcomeType &outcome,
- const int64_t retry_timeout_us = ObObjectStorageTenantGuard::get_timeout_us());
-
-private:
- SpinRWLock lock_;
- bool is_inited_;
- bool stopped_;
- int64_t ref_cnt_;
- int64_t last_modified_ts_;
- Aws::S3::S3Client *client_;
-};
-
-class ObS3Env
-{
-public:
- static ObS3Env &get_instance();
-
- // global init s3 env resource, must and only can be called once
- int init();
- // global clean s3 resource when don't use s3 any more
- void destroy();
-
- int get_or_create_s3_client(const ObS3Account &account, ObS3Client *&client);
- void stop();
-
-private:
- ObS3Env();
- int clean_s3_client_map_();
-
-private:
- SpinRWLock lock_;
- bool is_inited_;
- ObS3MemoryManager s3_mem_manger_;
- Aws::SDKOptions aws_options_;
- hash::ObHashMap s3_client_map_;
-};
-
-template
-class ObStorageS3RetryStrategy : public ObStorageIORetryStrategy
-{
-public:
- using typename ObStorageIORetryStrategy::RetType;
- using ObStorageIORetryStrategy::start_time_us_;
- using ObStorageIORetryStrategy::timeout_us_;
-
- ObStorageS3RetryStrategy(const int64_t timeout_us = ObObjectStorageTenantGuard::get_timeout_us())
- : ObStorageIORetryStrategy(timeout_us)
- {}
- virtual ~ObStorageS3RetryStrategy() {}
-
- virtual void log_error(
- const RetType &outcome, const int64_t attempted_retries) const override
- {
- const char *request_id = outcome.GetResult().GetRequestId().c_str();
- if (outcome.GetResult().GetRequestId().empty()) {
- const Aws::Http::HeaderValueCollection &headers = outcome.GetError().GetResponseHeaders();
- Aws::Http::HeaderValueCollection::const_iterator it = headers.find("x-amz-request-id");
- if (it != headers.end()) {
- request_id = it->second.c_str();
- }
- }
- const int code = static_cast(outcome.GetError().GetResponseCode());
- const char *exception = outcome.GetError().GetExceptionName().c_str();
- const char *err_msg = outcome.GetError().GetMessage().c_str();
- OB_LOG_RET(WARN, OB_SUCCESS, "S3 log error",
- K(start_time_us_), K(timeout_us_), K(attempted_retries),
- K(request_id), K(code), K(exception), K(err_msg));
- }
-
-protected:
- virtual bool should_retry_impl_(
- const RetType &outcome, const int64_t attempted_retries) const override
- {
- bool bret = false;
- if (OB_SUCCESS != EventTable::EN_OBJECT_STORAGE_IO_RETRY) {
- bret = true;
- OB_LOG(INFO, "errsim object storage IO retry", K(outcome.IsSuccess()));
- } else if (outcome.IsSuccess()) {
- bret = false;
- } else if (outcome.GetError().ShouldRetry()) {
- bret = true;
- }
- return bret;
- }
-};
-
-struct S3ObjectMeta : public ObStorageObjectMetaBase
-{
-};
-
-class SafeExecutor
-{
-public:
- template
- int do_safely(Function f, Obj obj, Args && ... args)
- {
- int ret = OB_SUCCESS;
-#ifndef _WIN32
- try {
- ret = std::mem_fn(f)(obj, std::forward(args)...);
- } catch (const std::exception &e) {
- ret = OB_OBJECT_STORAGE_IO_ERROR;
- OB_LOG(WARN, "caught exception when doing s3 operation", K(ret), K(e.what()), KP(this));
- } catch (...) {
- ret = OB_OBJECT_STORAGE_IO_ERROR;
- OB_LOG(WARN, "caught unknown exception when doing s3 operation", K(ret), KP(this));
- }
-#else
- ret = std::mem_fn(f)(obj, std::forward(args)...);
-#endif
- return ret;
- }
-};
-
-class ObStorageS3Util;
-
-class ObStorageS3Base : public SafeExecutor
-{
-public:
- ObStorageS3Base();
- virtual ~ObStorageS3Base();
- virtual void reset();
-
- virtual int open(const ObString &uri, ObObjectStorageInfo *storage_info);
- virtual int inner_open(const ObString &uri, ObObjectStorageInfo *storage_info);
- virtual bool is_inited() const { return is_inited_; }
-
- int get_s3_file_meta(S3ObjectMeta &meta)
- {
- return do_safely(&ObStorageS3Base::get_s3_file_meta_, this, meta);
- }
-
-protected:
- int get_s3_file_meta_(S3ObjectMeta &meta);
- int do_list_(const int64_t max_list_num, const char *delimiter,
- const Aws::String &next_marker, Aws::S3::Model::ListObjectsOutcome &outcome);
-
-protected:
- ObArenaAllocator allocator_;
- ObS3Client *s3_client_;
- ObString bucket_;
- ObString object_;
- // The default is ObStorageChecksumType::OB_MD5_ALGO
- // The S3 SDK cannot disable checksum,
- // therefore ObStorageChecksumType::OB_NO_CHECKSUM_ALGO is not supported
- ObStorageChecksumType checksum_type_;
-
-private:
- bool is_inited_;
- ObS3Account s3_account_;
- friend class ObStorageS3Util;
- DISALLOW_COPY_AND_ASSIGN(ObStorageS3Base);
-};
-
-class ObStorageS3Writer : public ObStorageS3Base, public ObIStorageWriter
-{
-public:
- ObStorageS3Writer();
- virtual ~ObStorageS3Writer();
-
- virtual int open(const ObString &uri, ObObjectStorageInfo *storage_info) override
- {
- return do_safely(&ObStorageS3Writer::open_, this, uri, storage_info);
- }
- virtual int write(const char *buf, const int64_t size) override
- {
- return do_safely(&ObStorageS3Writer::write_, this, buf, size);
- }
- virtual int pwrite(const char *buf, const int64_t size, const int64_t offset) override
- {
- return do_safely(&ObStorageS3Writer::pwrite_, this, buf, size, offset);
- }
- virtual int close() override
- {
- return do_safely(&ObStorageS3Writer::close_, this);
- }
- virtual int64_t get_length() const override { return file_length_; }
- virtual bool is_opened() const override { return is_opened_; }
-
-protected:
- int open_(const ObString &uri, ObObjectStorageInfo *storage_info);
- int write_(const char *buf, const int64_t size);
- int write_obj_(const char *obj_name, const char *buf, const int64_t size);
- int pwrite_(const char *buf, const int64_t size, const int64_t offset);
- int close_();
-
-protected:
- bool is_opened_;
- int64_t file_length_;
-
-private:
- DISALLOW_COPY_AND_ASSIGN(ObStorageS3Writer);
-};
-
-class ObStorageS3Reader : public ObStorageS3Base, public ObIStorageReader
-{
-public:
- ObStorageS3Reader();
- virtual ~ObStorageS3Reader();
- virtual void reset() override;
-
- virtual int open(const ObString &uri,
- ObObjectStorageInfo *storage_info, const bool head_meta = true) override
- {
- return do_safely(&ObStorageS3Reader::open_, this, uri, storage_info, head_meta);
- }
- virtual int pread(char *buf, const int64_t buf_size, const int64_t offset, int64_t &read_size) override
- {
- return do_safely(&ObStorageS3Reader::pread_, this, buf, buf_size, offset, read_size);
- }
- virtual int close() override
- {
- return do_safely(&ObStorageS3Reader::close_, this);
- }
- virtual int64_t get_length() const override { return file_length_; }
- virtual bool is_opened() const override { return is_opened_; }
-
-protected:
- int open_(const ObString &uri, ObObjectStorageInfo *storage_info, const bool head_meta = true);
- int pread_(char *buf, const int64_t buf_size, const int64_t offset, int64_t &read_size);
- int close_();
-
-protected:
- bool is_opened_;
- bool has_meta_;
- int64_t file_length_;
-
-private:
- DISALLOW_COPY_AND_ASSIGN(ObStorageS3Reader);
-};
-
-class ObStorageS3Util : public SafeExecutor, public ObIStorageUtil
-{
-public:
- ObStorageS3Util();
- virtual ~ObStorageS3Util();
-
- virtual int open(ObObjectStorageInfo *storage_info) override;
- virtual void close() override;
- virtual int head_object_meta(const ObString &uri, ObStorageObjectMetaBase &obj_meta) override;
-
- virtual int is_exist(const ObString &uri, bool &exist) override
- {
- return do_safely(&ObStorageS3Util::is_exist_, this, uri, exist);
- }
- virtual int get_file_length(const ObString &uri, int64_t &file_length) override
- {
- return do_safely(&ObStorageS3Util::get_file_length_, this, uri, file_length);
- }
- virtual int del_file(const ObString &uri) override
- {
- return do_safely(&ObStorageS3Util::del_file_, this, uri);
- }
- virtual int batch_del_files(
- const ObString &uri,
- hash::ObHashMap &files_to_delete,
- ObIArray &failed_files_idx) override
- {
- return do_safely(&ObStorageS3Util::batch_del_files_, this,
- uri, files_to_delete, failed_files_idx);
- }
- virtual int write_single_file(const ObString &uri, const char *buf, const int64_t size) override
- {
- return do_safely(&ObStorageS3Util::write_single_file_, this, uri, buf, size);
- }
- virtual int mkdir(const ObString &uri) override
- {
- return do_safely(&ObStorageS3Util::mkdir_, this, uri);
- }
- virtual int list_files(const ObString &uri, ObBaseDirEntryOperator &op) override
- {
- return do_safely(&ObStorageS3Util::list_files_, this, uri, op);
- }
- virtual int list_files(const ObString &uri, ObStorageListCtxBase &list_ctx) override
- {
- return do_safely(&ObStorageS3Util::list_files2_, this, uri, list_ctx);
- }
- virtual int del_dir(const ObString &uri) override
- {
- return do_safely(&ObStorageS3Util::del_dir_, this, uri);
- }
- virtual int list_directories(const ObString &uri, ObBaseDirEntryOperator &op) override
- {
- return do_safely(&ObStorageS3Util::list_directories_, this, uri, op);
- }
- virtual int is_tagging(const ObString &uri, bool &is_tagging) override
- {
- return do_safely(&ObStorageS3Util::is_tagging_, this, uri, is_tagging);
- }
- virtual int del_unmerged_parts(const ObString &uri) override
- {
- return do_safely(&ObStorageS3Util::del_unmerged_parts_, this, uri);
- }
-
-private:
- int is_exist_(const ObString &uri, bool &exist);
- int get_file_length_(const ObString &uri, int64_t &file_length);
- int del_file_(const ObString &uri);
- int batch_del_files_(
- const ObString &uri,
- hash::ObHashMap &files_to_delete,
- ObIArray &failed_files_idx);
- int write_single_file_(const ObString &uri, const char *buf, const int64_t size);
- int mkdir_(const ObString &uri);
- int list_files_(const ObString &uri, ObBaseDirEntryOperator &op);
- int list_files2_(const ObString &uri, ObStorageListCtxBase &list_ctx);
- int del_dir_(const ObString &uri);
- int list_directories_(const ObString &uri, ObBaseDirEntryOperator &op);
- int is_tagging_(const ObString &uri, bool &is_tagging);
- int del_unmerged_parts_(const ObString &uri);
-
- int delete_object_(ObStorageS3Base &s3_base);
- int tagging_object_(ObStorageS3Base &s3_base);
-
-private:
- bool is_opened_;
- ObObjectStorageInfo *storage_info_;
-};
-
-class ObStorageS3AppendWriter : public ObStorageS3Writer
-{
-public:
- ObStorageS3AppendWriter();
- virtual ~ObStorageS3AppendWriter();
-
- virtual int open(const ObString &uri, ObObjectStorageInfo *storage_info) override
- {
- return do_safely(&ObStorageS3AppendWriter::open_, this, uri, storage_info);
- }
- virtual int write(const char *buf, const int64_t size) override
- {
- return do_safely(&ObStorageS3AppendWriter::write_, this, buf, size);
- }
- virtual int pwrite(const char *buf, const int64_t size, const int64_t offset) override
- {
- return do_safely(&ObStorageS3AppendWriter::pwrite_, this, buf, size, offset);
- }
- virtual int close() override
- {
- return do_safely(&ObStorageS3AppendWriter::close_, this);
- }
- virtual int64_t get_length() const override;
- virtual bool is_opened() const override { return is_opened_; }
-
-protected:
- int open_(const ObString &uri, ObObjectStorageInfo *storage_info);
- int write_(const char *buf, const int64_t size);
- int pwrite_(const char *buf, const int64_t size, const int64_t offset);
- int close_();
-
-private:
- ObObjectStorageInfo *storage_info_;
-
- DISALLOW_COPY_AND_ASSIGN(ObStorageS3AppendWriter);
-};
-
-class ObStorageS3MultiPartWriter : public ObStorageS3Writer,
- public ObIStorageMultiPartWriter,
- public ObStoragePartInfoHandler
-{
-public:
- ObStorageS3MultiPartWriter();
- virtual ~ObStorageS3MultiPartWriter();
- virtual void reset() override;
-
- virtual int open(const ObString &uri, ObObjectStorageInfo *storage_info) override
- {
- return do_safely(&ObStorageS3MultiPartWriter::open_, this, uri, storage_info);
- }
- virtual int write(const char *buf, const int64_t size) override
- {
- return do_safely(&ObStorageS3MultiPartWriter::write_, this, buf, size);
- }
- virtual int pwrite(const char *buf, const int64_t size, const int64_t offset) override
- {
- return do_safely(&ObStorageS3MultiPartWriter::pwrite_, this, buf, size, offset);
- }
- virtual int complete() override
- {
- return do_safely(&ObStorageS3MultiPartWriter::complete_, this);
- }
- virtual int abort() override
- {
- return do_safely(&ObStorageS3MultiPartWriter::abort_, this);
- }
- virtual int close() override
- {
- return do_safely(&ObStorageS3MultiPartWriter::close_, this);
- }
- virtual int64_t get_length() const override { return file_length_; }
- virtual bool is_opened() const override { return is_opened_; }
-
-private:
- int open_(const ObString &uri, ObObjectStorageInfo *storage_info);
- int write_(const char *buf, const int64_t size);
- int pwrite_(const char *buf, const int64_t size, const int64_t offset);
- int complete_();
- int abort_();
- int close_();
- int write_single_part_();
-
-protected:
- char *base_buf_;
- int64_t base_buf_pos_;
- char *upload_id_;
- int partnum_;
-
-private:
- DISALLOW_COPY_AND_ASSIGN(ObStorageS3MultiPartWriter);
-};
-
-class ObStorageParallelS3MultiPartWriter : public ObStorageS3Writer,
- public ObIStorageParallelMultipartWriter,
- public ObStoragePartInfoHandler
-{
-public:
- ObStorageParallelS3MultiPartWriter();
- virtual ~ObStorageParallelS3MultiPartWriter();
- virtual void reset() override;
-
- virtual int open(const ObString &uri, ObObjectStorageInfo *storage_info) override
- {
- return do_safely(&ObStorageParallelS3MultiPartWriter::open_, this, uri, storage_info);
- }
- virtual int upload_part(const char *buf, const int64_t size, const int64_t part_id) override
- {
- return do_safely(&ObStorageParallelS3MultiPartWriter::upload_part_, this, buf, size, part_id);
- }
- virtual int complete() override
- {
- return do_safely(&ObStorageParallelS3MultiPartWriter::complete_, this);
- }
- virtual int abort() override
- {
- return do_safely(&ObStorageParallelS3MultiPartWriter::abort_, this);
- }
- virtual int close() override
- {
- return do_safely(&ObStorageParallelS3MultiPartWriter::close_, this);
- }
- virtual bool is_opened() const override { return is_opened_; }
-
-private:
- int open_(const ObString &uri, ObObjectStorageInfo *storage_info);
- int upload_part_(const char *buf, const int64_t size, const int64_t part_id);
- int complete_();
- int abort_();
- int close_();
-
-protected:
- char *upload_id_;
-
-private:
- DISALLOW_COPY_AND_ASSIGN(ObStorageParallelS3MultiPartWriter);
-};
-
-} // common
-} // oceanbase
-
-#endif
diff --git a/deps/oblib/src/lib/signal/ob_signal_handlers.cpp b/deps/oblib/src/lib/signal/ob_signal_handlers.cpp
index 4ececf343..c40ab4f99 100644
--- a/deps/oblib/src/lib/signal/ob_signal_handlers.cpp
+++ b/deps/oblib/src/lib/signal/ob_signal_handlers.cpp
@@ -99,6 +99,13 @@ static inline void handler(int sig, siginfo_t *s, void *p)
int install_ob_signal_handler()
{
int ret = OB_SUCCESS;
+#ifdef __ANDROID__
+ // ART/libsigchain owns fatal signal dispatch for app processes. Installing
+ // OceanBase's process-wide crash handlers here interferes with JNI/runtime
+ // fault handling and can turn startup failures into opaque instrumentation
+ // crashes.
+ return ret;
+#else
struct sigaction sa;
sa.sa_flags = SA_SIGINFO | SA_RESTART | SA_NODEFER | SA_ONSTACK;
sa.sa_sigaction = handler;
@@ -114,6 +121,7 @@ int install_ob_signal_handler()
}
}
return ret;
+#endif
}
bool g_redirect_handler = false;
diff --git a/deps/oblib/src/lib/task/ob_timer_service.cpp b/deps/oblib/src/lib/task/ob_timer_service.cpp
index d8694c093..b15079d90 100644
--- a/deps/oblib/src/lib/task/ob_timer_service.cpp
+++ b/deps/oblib/src/lib/task/ob_timer_service.cpp
@@ -219,8 +219,7 @@ int ObTimerService::start()
} else if (OB_FAIL(worker_thread_pool_.init(
MIN_WORKER_THREAD_NUM, TASK_NUM_LIMIT, "TimerWK", tenant_id_))) {
OB_LOG(WARN, "init ObTimerTaskThreadPool failed", K_(tenant_id), K(ret));
- } else if (OB_FAIL(worker_thread_pool_.set_adaptive_thread(
- MIN_WORKER_THREAD_NUM, MAX_WORKER_THREAD_NUM))) {
+ } else if (OB_FAIL(worker_thread_pool_.set_thread_count(MAX_WORKER_THREAD_NUM))) {
OB_LOG(WARN, "set adaptive thread failed", K_(tenant_id), K(ret));
} else if (OB_FAIL(ThreadPool::start())) {
OB_LOG(WARN, "failed to start ObTimerService thread", K(ret));
diff --git a/deps/oblib/src/lib/thread/ob_adaptive_worker_pool.h b/deps/oblib/src/lib/thread/ob_adaptive_worker_pool.h
new file mode 100644
index 000000000..ad76d6e53
--- /dev/null
+++ b/deps/oblib/src/lib/thread/ob_adaptive_worker_pool.h
@@ -0,0 +1,128 @@
+/**
+ * Copyright (c) 2025 OceanBase.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _OCEABASE_LIB_THREAD_OB_ADAPTIVE_WORKER_POOL_H_
+#define _OCEABASE_LIB_THREAD_OB_ADAPTIVE_WORKER_POOL_H_
+
+#include
+#include "lib/ob_errno.h"
+#include "lib/utility/ob_macro_utils.h"
+
+namespace oceanbase
+{
+namespace lib
+{
+
+/**
+ * ObAdaptiveWorkerPool — a CRTP template providing CAS-based dynamic worker
+ * scaling. It contains no expansion/contraction *policy*: callers decide the
+ * limit and floor for each operation by passing them to try_expand_one() and
+ * try_shrink_one().
+ *
+ * ## Derived contract (CRTP)
+ * bool do_add_worker(); // create + start one worker, return success
+ * int64_t queue_size() const; // for shrink-to-0 safety guard
+ *
+ * ## Reaping stopped workers (required)
+ * Workers that exit via try_shrink_one() call Worker::stop() and then break
+ * out of the worker loop, leaving a "zombie" entry in the workers_ list.
+ * The derived class MUST run a periodic background task (e.g. timeup())
+ * to reap these stopped workers — remove their nodes from the list and
+ * destroy them. See ObTenant::check_worker_count().
+ *
+ * ## Achieving th_worker-style min/max limits
+ * Two independent limits, enforced at different call sites:
+ *
+ * min / normal ceiling — a CPU-derived count the pool should operate
+ * at under normal load.
+ * recv_request: try_expand_one(min_limit) on cold start
+ * worker loop: try_expand_one(min_limit) when expand signal fires
+ *
+ * max / rescue ceiling — a memory-derived hard cap never exceeded,
+ * only used when workers appear deadlocked.
+ * timeup: try_expand_one(max_limit) when completion stalls for
+ * N seconds with non-empty queue
+ *
+ * Callers check worker_count() against the min limit to decide *whether* to
+ * call try_expand_one(); the CAS loop inside try_expand_one() enforces the
+ * hard max cap.
+ */
+template
+class ObAdaptiveWorkerPool
+{
+public:
+ ObAdaptiveWorkerPool() : idle_cnt_(0), total_cnt_(0) {}
+
+ template
+ int pop_with_idle(PopFn &&pop_fn, bool &expand) {
+ idle_enter();
+ int ret = pop_fn();
+ expand = (idle_exit() == 1);
+ return ret;
+ }
+
+ /// CAS-based expansion up to the given limit.
+ bool try_expand_one(int64_t limit)
+ {
+ int64_t cur = total_cnt_.load(std::memory_order_relaxed);
+ while (cur < limit) {
+ if (total_cnt_.compare_exchange_weak(cur, cur + 1,
+ std::memory_order_acq_rel, std::memory_order_relaxed)) {
+ while (!self().do_add_worker()) {
+ ob_usleep(1000);
+ }
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /// CAS-based shrink down to the given floor.
+ /// Refuses to shrink to 0 when queue is non-empty.
+ bool try_shrink_one(int64_t floor)
+ {
+ int64_t cur = total_cnt_.load(std::memory_order_relaxed);
+ while (cur > floor) {
+ if (total_cnt_.compare_exchange_weak(cur, cur - 1,
+ std::memory_order_acq_rel, std::memory_order_relaxed)) {
+ if (cur - 1 == 0 && self().queue_size() > 0) {
+ total_cnt_.fetch_add(1, std::memory_order_relaxed);
+ return false;
+ }
+ return true;
+ }
+ }
+ return false;
+ }
+
+protected:
+ int64_t idle_count() const { return idle_cnt_.load(std::memory_order_relaxed); }
+ int64_t worker_count() const { return total_cnt_.load(std::memory_order_relaxed); }
+
+private:
+ int64_t idle_enter() { return idle_cnt_.fetch_add(1, std::memory_order_relaxed); }
+ int64_t idle_exit() { return idle_cnt_.fetch_sub(1, std::memory_order_relaxed); }
+ Derived &self() { return *static_cast(this); }
+ const Derived &self() const { return *static_cast(this); }
+
+ std::atomic idle_cnt_;
+ std::atomic total_cnt_;
+};
+
+} // end of namespace lib
+} // end of namespace oceanbase
+
+#endif /* _OCEABASE_LIB_THREAD_OB_ADAPTIVE_WORKER_POOL_H_ */
diff --git a/deps/oblib/src/lib/thread/ob_dynamic_thread_pool.cpp b/deps/oblib/src/lib/thread/ob_dynamic_thread_pool.cpp
index ff840a092..46cf4465d 100644
--- a/deps/oblib/src/lib/thread/ob_dynamic_thread_pool.cpp
+++ b/deps/oblib/src/lib/thread/ob_dynamic_thread_pool.cpp
@@ -357,189 +357,8 @@ void *ObDynamicThreadPool::task_thread_func(void *data)
return NULL;
}
-int ObSimpleDynamicThreadPool::init(const int64_t thread_num, const char* name, const int64_t tenant_id)
-{
- int ret = OB_SUCCESS;
- name_ = name;
- max_thread_cnt_ = thread_num;
- tenant_id_ = tenant_id;
- if (min_thread_cnt_ < 0) {
- // has not set adaptive thread
- min_thread_cnt_ = thread_num;
- }
- ret = set_thread_count(thread_num);
- return ret;
-}
-
-void ObSimpleDynamicThreadPool::stop()
-{
- IGNORE_RETURN ObSimpleThreadPoolDynamicMgr::get_instance().unbind(this);
- lib::ThreadPool::stop();
- // Wake all workers blocked in queue pop so they can check the stop flag immediately
- notify_stop();
-}
-
-void ObSimpleDynamicThreadPool::destroy()
-{
- if (min_thread_cnt_ < max_thread_cnt_) {
- IGNORE_RETURN ObSimpleThreadPoolDynamicMgr::get_instance().unbind(this);
- }
- int64_t ref_cnt = 0;
- while ((ref_cnt = get_ref_cnt()) > 0) {
- if (REACH_TIME_INTERVAL(1000L * 1000L)) {
- COMMON_LOG(INFO, "wait tenant io manager quit", K(*this), K(ref_cnt));
- }
- ob_usleep((useconds_t)10L * 1000L);
- }
- lib::ThreadPool::stop();
- notify_stop();
- lib::ThreadPool::wait();
-}
-
ObSimpleDynamicThreadPool::~ObSimpleDynamicThreadPool()
{
- destroy();
-}
-
-int ObSimpleDynamicThreadPool::set_adaptive_thread(int64_t min_thread_num, int64_t max_thread_num)
-{
- int ret = OB_SUCCESS;
- if (min_thread_num < 0 || max_thread_num <= 0
- || min_thread_num > MAX_THREAD_NUM || max_thread_num > MAX_THREAD_NUM
- || min_thread_num > max_thread_num) {
- ret = OB_INVALID_ARGUMENT;
- COMMON_LOG(WARN, "set_adaptive_thread failed", KP(this), K(min_thread_num), K(max_thread_num));
- } else if (OB_FAIL((ObSimpleThreadPoolDynamicMgr::get_instance().bind(this)))) {
- COMMON_LOG(WARN, "bind dynamic mgr failed");
- } else {
- min_thread_cnt_ = min_thread_num;
- max_thread_cnt_ = max_thread_num;
- COMMON_LOG(INFO, "set adaptive thread success", KP(this), K(min_thread_num), K(max_thread_num));
- }
- return ret;
-}
-
-int ObSimpleDynamicThreadPool::set_max_thread_count(int64_t max_thread_cnt)
-{
- int ret = OB_SUCCESS;
- if (max_thread_cnt > MAX_THREAD_NUM || max_thread_cnt <= 0) {
- ret = OB_INVALID_ARGUMENT;
- COMMON_LOG(WARN, "set_adaptive_thread failed", KP(this), K(max_thread_cnt));
- } else {
- update_threads_lock_.lock();
- bool has_set_adaptive_thread = (max_thread_cnt_ > min_thread_cnt_);
- bool has_set_adaptive_thread_new = (max_thread_cnt > min_thread_cnt_);
- if (!has_set_adaptive_thread_new && OB_FAIL(lib::ThreadPool::set_thread_count(max_thread_cnt))) {
- COMMON_LOG(ERROR, "set thread cnt failed", KP(this), K(max_thread_cnt));
- }
- if (OB_SUCC(ret)) {
- max_thread_cnt_ = max_thread_cnt;
- COMMON_LOG(INFO, "set max thread count", K(*this), K(max_thread_cnt));
- }
- update_threads_lock_.unlock();
- if (has_set_adaptive_thread != has_set_adaptive_thread_new) {
- if (has_set_adaptive_thread_new
- && OB_FAIL(ObSimpleThreadPoolDynamicMgr::get_instance().bind(this))) {
- COMMON_LOG(WARN, "bind thread pool to dynamic mgr failed", KP(this));
- } else if (!has_set_adaptive_thread_new
- && OB_FAIL(ObSimpleThreadPoolDynamicMgr::get_instance().unbind(this))) {
- COMMON_LOG(WARN, "unbind thread pool to dynamic mgr failed", KP(this));
- }
- }
- }
- return ret;
-}
-
-int ObSimpleDynamicThreadPool::set_thread_count_and_try_recycle(int64_t cnt)
-{
- int ret = OB_SUCCESS;
- const int64_t old_thread_count = get_thread_count();
- ret = Threads::do_set_thread_count(cnt, true/*async_recycle*/);
- if (OB_SUCC(ret)) {
- // When shrinking, wake up all workers to check stop flags
- if (cnt < old_thread_count) {
- notify_stop();
- }
- ret = Threads::try_thread_recycle();
- }
- return ret;
-}
-
-void ObSimpleDynamicThreadPool::try_expand_thread_count()
-{
- int ret = OB_SUCCESS;
- common::ObTimeGuard time_guard("expand_thread", 20000);
- int64_t queue_size = 0;
- if (min_thread_cnt_ >= max_thread_cnt_) {
- // not set adaptive thread, do nothing
- } else if ((queue_size = get_queue_num()) <= 0) {
- // unneed to expand thread count
- } else if (OB_SUCC(update_threads_lock_.trylock())) {
- IGNORE_RETURN Threads::try_thread_recycle();
- int64_t cur_thread_count = get_thread_count();
- int inc_cnt = 0;
- if (cur_thread_count > 0) {
- inc_cnt = queue_size / cur_thread_count;
- } else {
- inc_cnt = queue_size;
- }
- inc_cnt = min(inc_cnt, max_thread_cnt_ - cur_thread_count);
- if (inc_cnt > 0) {
- COMMON_LOG(INFO, "expand thread count", KP(this), K_(max_thread_cnt), K(cur_thread_count), K(inc_cnt), K(queue_size));
- int64_t old_thread_count = cur_thread_count;
- for (int i = 1; OB_SUCC(ret) && i <= inc_cnt; ++i) {
- if (is_server_tenant(tenant_id_)) {
- // temporarily reset ob_get_tenant_id() and run_wrapper
- // avoid the newly created thread to use the memory or run_wrapper of user tenant
- lib::IRunWrapper *run_wrapper = lib::Threads::get_expect_run_wrapper();
- lib::Threads::get_expect_run_wrapper() = NULL;
- DEFER(lib::Threads::get_expect_run_wrapper() = run_wrapper);
- ObResetThreadTenantIdGuard guard;
- ret = set_thread_count_and_try_recycle(old_thread_count + i);
- } else {
- ret = set_thread_count_and_try_recycle(old_thread_count + i);
- }
- }
- if (OB_FAIL(ret)) {
- COMMON_LOG(ERROR, "set thread count failed", KP(this), K(old_thread_count), "cur_thread_count", get_thread_count(), K(inc_cnt));
- }
- }
- update_threads_lock_.unlock();
- }
-}
-
-void ObSimpleDynamicThreadPool::try_inc_thread_count(int64_t cnt)
-{
- int ret = OB_SUCCESS;
- common::ObTimeGuard time_guard("inc_thread", 20000);
- if (min_thread_cnt_ >= max_thread_cnt_) {
- // not set adaptive thread, do nothing
- } else if (OB_SUCC(update_threads_lock_.trylock())) {
- int64_t cur_thread_count = get_thread_count();
- int64_t new_thread_count = cur_thread_count + cnt;
- new_thread_count = max(new_thread_count, min_thread_cnt_);
- new_thread_count = min(new_thread_count, max_thread_cnt_);
- COMMON_LOG(INFO, "try inc thread count", K(*this), K(cur_thread_count), K(cnt), K(new_thread_count));
- if (new_thread_count != cur_thread_count) {
- if (OB_FAIL(set_thread_count_and_try_recycle(new_thread_count))) {
- COMMON_LOG(ERROR, "set thread count failed", K(*this), K(cur_thread_count), K(cnt), K(new_thread_count));
- } else {
- COMMON_LOG(INFO, "inc thread count", K(*this), K(cur_thread_count), K(cnt), K(new_thread_count));
- }
- }
- update_threads_lock_.unlock();
- }
-}
-
-int ObSimpleDynamicThreadPool::set_thread_count(int64_t n_threads)
-{
- int ret = OB_SUCCESS;
- if (min_thread_cnt_ < max_thread_cnt_) {
- ret = lib::ThreadPool::set_thread_count(min_thread_cnt_);
- } else {
- ret = lib::ThreadPool::set_thread_count(n_threads);
- }
- return ret;
}
ObSimpleThreadPoolDynamicMgr &ObSimpleThreadPoolDynamicMgr::get_instance()
@@ -547,6 +366,7 @@ ObSimpleThreadPoolDynamicMgr &ObSimpleThreadPoolDynamicMgr::get_instance()
static ObSimpleThreadPoolDynamicMgr instance;
return instance;
}
+
int ObSimpleThreadPoolDynamicMgr::init()
{
int ret = OB_SUCCESS;
@@ -589,53 +409,14 @@ void ObSimpleThreadPoolDynamicMgr::run1()
{
ObDIActionGuard ag("DynamicThreadPool", "DynamicMgrCheck", "detect task");
lib::set_thread_name("qth_mgr");
- int64_t last_access_ts = 0;
- int64_t last_idle_ts = 0;
- uint64_t loop_cnt = 0;
while (!has_set_stop()) {
- // check global simple thread pool
- ObArray dec_pools;
{
- SpinRLockGuard guard(simple_thread_pool_list_lock_);
- for (int i = 0; i < simple_thread_pool_list_.count(); i++) {
- ObSimpleThreadPoolStat &pool_stat = simple_thread_pool_list_.at(i);
- ObSimpleDynamicThreadPool *pool = pool_stat.pool_;
- ObDIActionGuard ag1(pool->name_);
- int64_t current_time = ObTimeUtility::current_time();
- if (OB_LIKELY(pool_stat.is_valid())) {
- int64_t interval = current_time - pool_stat.last_check_time_;
- int64_t idle = pool->get_threads_idle_time() - pool_stat.last_idle_time_;
- if (idle > interval
- && current_time - pool_stat.last_shrink_time_ > SHRINK_INTERVAL_US
- && pool->get_thread_count() == pool_stat.last_thread_count_
- && pool_stat.last_thread_count_ > pool->get_min_thread_cnt()) {
- int64_t queue_size = pool->get_queue_num();
- int tmp_ret = OB_SUCCESS;
- pool_stat.last_shrink_time_ = current_time;
- if (OB_TMP_FAIL(dec_pools.push_back(pool))) {
- COMMON_LOG_RET(ERROR, tmp_ret, "push dec pool failed", KP(pool));
- } else {
- pool->inc_ref();
- }
- current_time = ObTimeUtility::current_time();
- }
- }
- pool_stat.last_check_time_ = current_time;
- pool_stat.last_idle_time_ = pool->get_threads_idle_time();
- pool_stat.last_thread_count_ = pool->get_thread_count();
- pool->try_expand_thread_count();
- if (OB_UNLIKELY(loop_cnt % 100 == 0)) { // print each 20s
- int64_t qsize = pool->get_queue_num();
- int64_t current_idle_time = pool->get_threads_idle_time();
- SHARE_LOG(INFO, "[thread pool stat]", K(pool_stat), K(*pool), K(qsize), K(current_idle_time));
- }
+ SpinRLockGuard guard(pool_list_lock_);
+ for (int i = 0; i < pool_list_.count(); i++) {
+ ObSimpleDynamicThreadPool *pool = pool_list_.at(i);
+ pool->reap_workers();
}
}
- for (int i = 0; i < dec_pools.count(); i++) {
- dec_pools[i]->try_inc_thread_count(-1);
- dec_pools[i]->dec_ref();
- }
- ++loop_cnt;
ob_usleep(CHECK_INTERVAL_US, true);
}
}
@@ -643,14 +424,16 @@ void ObSimpleThreadPoolDynamicMgr::run1()
int ObSimpleThreadPoolDynamicMgr::bind(ObSimpleDynamicThreadPool *pool)
{
int ret = OB_SUCCESS;
- ObSimpleThreadPoolStat pool_stat(pool);
-
- SpinWLockGuard guard(simple_thread_pool_list_lock_);
- if (OB_FAIL(simple_thread_pool_list_.push_back(pool_stat))) {
- COMMON_LOG(WARN, "bind simple thread pool faild", KP(pool));
+ if (pool->has_bind_) {
+ // already bound
} else {
- pool->has_bind_ = true;
- COMMON_LOG(INFO, "bind simple thread pool success", K(*pool));
+ SpinWLockGuard guard(pool_list_lock_);
+ if (OB_FAIL(pool_list_.push_back(pool))) {
+ COMMON_LOG(WARN, "bind simple thread pool failed", KP(pool));
+ } else {
+ pool->has_bind_ = true;
+ COMMON_LOG(INFO, "bind simple thread pool success", K(*pool));
+ }
}
return ret;
}
@@ -660,32 +443,26 @@ int ObSimpleThreadPoolDynamicMgr::unbind(ObSimpleDynamicThreadPool *pool)
int ret = OB_SUCCESS;
if (OB_UNLIKELY(NULL == pool)) {
ret = OB_INVALID_ARGUMENT;
- COMMON_LOG(WARN, "unbind pool failed");
+ COMMON_LOG(WARN, "unbind pool failed");
} else if (!pool->has_bind_) {
// do-nothing
} else {
- SpinWLockGuard guard(simple_thread_pool_list_lock_);
+ SpinWLockGuard guard(pool_list_lock_);
int64_t idx = -1;
- for (int64_t i = 0; (i < simple_thread_pool_list_.count()) && (-1 == idx); ++i) {
- ObSimpleThreadPoolStat &pool_stat = simple_thread_pool_list_.at(i);
- if (pool_stat.pool_ == pool) {
+ for (int64_t i = 0; (i < pool_list_.count()) && (-1 == idx); ++i) {
+ if (pool_list_.at(i) == pool) {
idx = i;
}
}
- if ((-1 != idx) && OB_FAIL(simple_thread_pool_list_.remove(idx))) {
- COMMON_LOG(WARN, "failed to remove simple_thread_pool", K(ret), K(idx), KP(pool));
+ if ((-1 != idx) && OB_FAIL(pool_list_.remove(idx))) {
+ COMMON_LOG(WARN, "failed to remove pool from mgr list", K(ret), K(idx), KP(pool));
} else {
pool->has_bind_ = false;
- COMMON_LOG(INFO, "try to unbind simple thread pool", K(*pool), K(idx));
+ COMMON_LOG(INFO, "unbind simple thread pool", K(*pool), K(idx));
}
}
return ret;
}
-int64_t ObSimpleThreadPoolDynamicMgr::get_pool_num() const
-{
- return simple_thread_pool_list_.size();
-}
-
} // namespace common
} // namespace oceanbase
diff --git a/deps/oblib/src/lib/thread/ob_dynamic_thread_pool.h b/deps/oblib/src/lib/thread/ob_dynamic_thread_pool.h
index 2719adcee..1998ff348 100644
--- a/deps/oblib/src/lib/thread/ob_dynamic_thread_pool.h
+++ b/deps/oblib/src/lib/thread/ob_dynamic_thread_pool.h
@@ -94,61 +94,40 @@ class ObDynamicThreadPool: public lib::ThreadPool
DISALLOW_COPY_AND_ASSIGN(ObDynamicThreadPool);
};
class ObSimpleDynamicThreadPool
- : public lib::ThreadPool
{
friend class ObSimpleThreadPoolDynamicMgr;
public:
static const int64_t MAX_THREAD_NUM = 1024;
ObSimpleDynamicThreadPool()
- : has_bind_(false), min_thread_cnt_(OB_INVALID_COUNT), max_thread_cnt_(OB_INVALID_COUNT),
- running_thread_cnt_(0), threads_idle_time_(0), update_threads_lock_(), ref_cnt_(0), name_("unknown"), tenant_id_(OB_SERVER_TENANT_ID)
+ : has_bind_(false), min_thread_cnt_(-1), max_thread_cnt_(-1),
+ name_("unknown"), tenant_id_(OB_SERVER_TENANT_ID), ref_cnt_(0)
{}
virtual ~ObSimpleDynamicThreadPool();
- int init(const int64_t thread_num, const char* name, const int64_t tenant_id);
- virtual void stop() override;
- void destroy();
- int set_adaptive_thread(int64_t min_thread_num, int64_t max_thread_num);
- virtual int64_t get_queue_num() const = 0;
- void try_expand_thread_count();
- void try_inc_thread_count(int64_t cnt = 1);
- int64_t get_threads_idle_time() const { return ATOMIC_LOAD(&threads_idle_time_); }
- int64_t get_min_thread_cnt() const { return min_thread_cnt_; }
- int64_t get_max_thread_cnt() const { return max_thread_cnt_; }
- int64_t get_running_thread_cnt() const { return running_thread_cnt_; }
- int set_thread_count(int64_t n_threads);
- int set_max_thread_count(int64_t max_thread_cnt);
- virtual void notify_stop() {}
void inc_ref() { ATOMIC_INC(&ref_cnt_); }
void dec_ref() { ATOMIC_SAF(&ref_cnt_, 1); }
int64_t get_ref_cnt() { return ATOMIC_LOAD(&ref_cnt_); }
- TO_STRING_KV(KCSTRING_(name), KP(this), K_(min_thread_cnt), K_(max_thread_cnt), K_(running_thread_cnt), K_(threads_idle_time), K_(tenant_id));
-protected:
- int64_t inc_thread_idle_time(int64_t time_us)
- {
- return ATOMIC_AAF(&threads_idle_time_, time_us);
- }
- int64_t inc_running_thread_cnt(int64_t cnt)
- {
- return ATOMIC_AAF(&running_thread_cnt_, cnt);
- }
- int set_thread_count_and_try_recycle(int64_t cnt);
-private:
+
+ // Mgr interface — implemented by ObSimpleThreadPoolBase
+ virtual int64_t get_queue_num() const = 0;
+ virtual void reap_workers() = 0;
+ virtual int64_t worker_count() const = 0;
+ virtual void notify_stop() {}
+
+ TO_STRING_KV(KCSTRING_(name), KP(this), K_(min_thread_cnt), K_(max_thread_cnt), K_(tenant_id));
+
bool has_bind_;
int64_t min_thread_cnt_;
int64_t max_thread_cnt_;
- int64_t running_thread_cnt_;
- int64_t threads_idle_time_;
- lib::ObMutex update_threads_lock_;
- int64_t ref_cnt_;
const char* name_;
int64_t tenant_id_;
+private:
+ int64_t ref_cnt_;
};
class ObSimpleThreadPoolDynamicMgr : public lib::TGRunnable {
public:
- static const int64_t SHRINK_INTERVAL_US = 1 * 1000 * 1000;
static const int64_t CHECK_INTERVAL_US = 200 * 1000;
- ObSimpleThreadPoolDynamicMgr() : simple_thread_pool_list_(), simple_thread_pool_list_lock_(), is_inited_(false) {}
+ ObSimpleThreadPoolDynamicMgr() : pool_list_(), pool_list_lock_(), is_inited_(false) {}
virtual ~ObSimpleThreadPoolDynamicMgr();
int init();
void stop();
@@ -157,35 +136,10 @@ class ObSimpleThreadPoolDynamicMgr : public lib::TGRunnable {
void run1();
int bind(ObSimpleDynamicThreadPool *pool);
int unbind(ObSimpleDynamicThreadPool *pool);
- int64_t get_pool_num() const;
static ObSimpleThreadPoolDynamicMgr &get_instance();
- struct ObSimpleThreadPoolStat {
- ObSimpleThreadPoolStat()
- : pool_(NULL),
- last_check_time_(OB_INVALID_TIMESTAMP),
- last_idle_time_(OB_INVALID_TIMESTAMP),
- last_thread_count_(OB_INVALID_COUNT),
- last_shrink_time_(OB_INVALID_TIMESTAMP) {}
- ObSimpleThreadPoolStat(ObSimpleDynamicThreadPool *pool)
- : pool_(pool),
- last_check_time_(OB_INVALID_TIMESTAMP),
- last_idle_time_(OB_INVALID_TIMESTAMP),
- last_thread_count_(OB_INVALID_COUNT),
- last_shrink_time_(OB_INVALID_TIMESTAMP) {}
- int is_valid()
- {
- return last_check_time_ >= 0 && last_idle_time_ >= 0 && last_thread_count_ > 0;
- }
- TO_STRING_KV(KP_(pool), K_(last_check_time), K_(last_idle_time), K_(last_thread_count));
- ObSimpleDynamicThreadPool *pool_;
- int64_t last_check_time_;
- int64_t last_idle_time_;
- int64_t last_thread_count_;
- int64_t last_shrink_time_;
- };
private:
- ObArray simple_thread_pool_list_;
- common::SpinRWLock simple_thread_pool_list_lock_;
+ ObArray pool_list_;
+ common::SpinRWLock pool_list_lock_;
int is_inited_;
};
diff --git a/deps/oblib/src/lib/thread/ob_simple_thread_pool.h b/deps/oblib/src/lib/thread/ob_simple_thread_pool.h
index a7a61ec47..a535095db 100644
--- a/deps/oblib/src/lib/thread/ob_simple_thread_pool.h
+++ b/deps/oblib/src/lib/thread/ob_simple_thread_pool.h
@@ -21,6 +21,10 @@
#include "lib/queue/ob_priority_queue.h"
#include "lib/thread/thread_pool.h"
#include "lib/thread/ob_dynamic_thread_pool.h"
+#include "lib/thread/ob_adaptive_worker_pool.h"
+#include "lib/list/ob_dlist.h"
+#include "lib/lock/ob_mutex.h"
+#include "lib/thread/threads.h"
#include "common/ob_queue_thread.h"
namespace oceanbase
@@ -42,39 +46,83 @@ struct QueueTypeMap {
template
class ObSimpleThreadPoolBase
: public ObSimpleDynamicThreadPool
+ , public lib::ObAdaptiveWorkerPool>
{
using TaskType = typename QueueTypeMap::TaskType;
using QElemType = typename QueueTypeMap::QElemType;
static const int64_t QUEUE_WAIT_TIME = 1000 * 1000;
+ static const int64_t SHRINK_TIMEOUT_US = 1 * 1000 * 1000;
+
+ struct Worker;
+ using WorkerNode = common::ObDLinkNode;
+ using WorkerList = common::ObDList;
+
+ struct Worker : public lib::Threads {
+ ObSimpleThreadPoolBase *pool_;
+ int64_t idx_;
+ WorkerNode worker_node_;
+ Worker(ObSimpleThreadPoolBase *pool, int64_t idx)
+ : lib::Threads(1), pool_(pool), idx_(idx) {
+ worker_node_.get_data() = this;
+ }
+ void run1() override;
+ };
+ friend struct Worker;
+
public:
ObSimpleThreadPoolBase();
virtual ~ObSimpleThreadPoolBase();
- int init(const int64_t thread_num, const int64_t task_num_limit, const char *name = "unknown", const uint64_t tenant_id = OB_SERVER_TENANT_ID);
+ int init(const int64_t thread_num, const int64_t task_num_limit,
+ const char *name = "unknown",
+ const uint64_t tenant_id = OB_SERVER_TENANT_ID);
void destroy();
int push(TaskType *task);
- virtual int64_t get_queue_num() const override
- {
- return queue_.size();
- }
- virtual void notify_stop() override
- {
- queue_.wake_all();
+ virtual int64_t get_queue_num() const override { return queue_.size(); }
+ virtual void notify_stop() override { queue_.wake_all(); }
+ int64_t worker_count() const override {
+ return lib::ObAdaptiveWorkerPool>::worker_count();
}
-private:
+
+ // Pool-level control
+ void stop();
+ void wait();
+ bool has_set_stop() const { return stop_; }
+
+ // RunWrapper
+ void set_run_wrapper(lib::IRunWrapper *rw) { run_wrapper_ = rw; }
+ lib::IRunWrapper *get_run_wrapper() const { return run_wrapper_; }
+
+ // Thread count queries (TG handler compatibility)
+ int64_t get_thread_count() const { return worker_count(); }
+ uint64_t get_thread_idx() const { return cur_worker_idx_; }
+
+ // Thread count configuration
+ int set_thread_count(int64_t n_threads);
+ int set_adaptive_thread(int64_t min_thread_num, int64_t max_thread_num);
+
+ // CRTP for lib::ObAdaptiveWorkerPool
+ bool do_add_worker();
+ int64_t queue_size() const { return queue_.size(); }
+
+ // Reap stopped workers (called by Mgr background thread only)
+ void reap_workers() override;
+
+protected:
virtual void handle(TaskType *task) = 0;
virtual void handle_drop(TaskType *task) {
- // when thread set stop left task will be process by handle_drop (default impl is handle)
- // users should define it's behaviour to manage task memory or some what
handle(task);
}
-protected:
- void run1();
+ virtual void run1();
private:
- const char* name_;
bool is_inited_;
+ bool stop_;
T queue_;
+ lib::IRunWrapper *run_wrapper_;
+ WorkerList workers_;
+ lib::ObMutex workers_lock_;
+ uint64_t cur_worker_idx_;
};
using ObSimpleThreadPool = ObSimpleThreadPoolBase;
diff --git a/deps/oblib/src/lib/thread/ob_simple_thread_pool.ipp b/deps/oblib/src/lib/thread/ob_simple_thread_pool.ipp
index 8131f1422..254d9ad35 100644
--- a/deps/oblib/src/lib/thread/ob_simple_thread_pool.ipp
+++ b/deps/oblib/src/lib/thread/ob_simple_thread_pool.ipp
@@ -17,7 +17,6 @@
#include "lib/thread/ob_thread_name.h"
#include "lib/thread/ob_simple_thread_pool.h"
#include "lib/ash/ob_active_session_guard.h"
-#include "lib/thread/ob_dynamic_thread_pool.h"
#ifdef __APPLE__
#include
#elif defined(_WIN32)
@@ -28,10 +27,13 @@ namespace oceanbase
{
namespace common
{
+
template
ObSimpleThreadPoolBase::ObSimpleThreadPoolBase()
: ObSimpleDynamicThreadPool(),
- name_("unknown"), is_inited_(false)
+ is_inited_(false), stop_(false),
+ run_wrapper_(nullptr),
+ cur_worker_idx_(0)
{
}
@@ -44,28 +46,37 @@ ObSimpleThreadPoolBase::~ObSimpleThreadPoolBase()
}
template
-int ObSimpleThreadPoolBase::init(const int64_t thread_num, const int64_t task_num_limit, const char* name, const uint64_t tenant_id)
+int ObSimpleThreadPoolBase::init(
+ const int64_t thread_num, const int64_t task_num_limit,
+ const char *name, const uint64_t tenant_id)
{
int ret = OB_SUCCESS;
if (is_inited_) {
ret = OB_INIT_TWICE;
- } else if (thread_num <= 0 || task_num_limit <= 0 || thread_num > MAX_THREAD_NUM || OB_ISNULL(name)) {
+ } else if (thread_num <= 0 || task_num_limit <= 0 || OB_ISNULL(name)) {
ret = OB_INVALID_ARGUMENT;
} else if (OB_FAIL(queue_.init(task_num_limit, name, tenant_id))) {
COMMON_LOG(WARN, "task queue init failed", K(ret), K(task_num_limit));
} else {
is_inited_ = true;
+ stop_ = false;
name_ = name;
- if (OB_FAIL(ObSimpleDynamicThreadPool::init(thread_num, name, tenant_id))) {
- COMMON_LOG(WARN, "dyna,ic thread pool init fail", K(ret));
- } else if (OB_FAIL(lib::ThreadPool::start())) {
- COMMON_LOG(WARN, "start thread pool fail", K(ret));
+ tenant_id_ = tenant_id;
+ if (max_thread_cnt_ < 0) {
+ max_thread_cnt_ = thread_num;
+ }
+ if (min_thread_cnt_ < 0) {
+ min_thread_cnt_ = 0;
+ }
+ if (OB_FAIL(ObSimpleThreadPoolDynamicMgr::get_instance().bind(this))) {
+ COMMON_LOG(WARN, "bind dynamic mgr failed", K(ret));
}
}
if (OB_FAIL(ret)) {
destroy();
} else {
- COMMON_LOG(INFO, "simple thread pool init success", KCSTRING(name), K(thread_num), K(task_num_limit));
+ COMMON_LOG(INFO, "simple thread pool init success",
+ KCSTRING(name), K(thread_num), K(task_num_limit));
}
return ret;
}
@@ -74,7 +85,26 @@ template
void ObSimpleThreadPoolBase::destroy()
{
is_inited_ = false;
- ObSimpleDynamicThreadPool::destroy();
+ stop_ = true;
+ // Stop all workers
+ {
+ lib::ObMutexGuard g(workers_lock_);
+ DLIST_FOREACH_NORET(wnode, workers_) {
+ wnode->get_data()->stop();
+ }
+ }
+ // Wait and delete all workers
+ {
+ lib::ObMutexGuard g(workers_lock_);
+ DLIST_FOREACH_REMOVESAFE_NORET(wnode, workers_) {
+ Worker *w = wnode->get_data();
+ workers_.remove(wnode);
+ w->wait();
+ w->destroy();
+ OB_DELETE(Worker, "QThWker", w);
+ }
+ }
+ IGNORE_RETURN ObSimpleThreadPoolDynamicMgr::get_instance().unbind(this);
queue_.destroy();
}
@@ -86,44 +116,183 @@ int ObSimpleThreadPoolBase::push(TaskType *task)
ret = OB_NOT_INIT;
} else if (NULL == task) {
ret = OB_INVALID_ARGUMENT;
- } else if (has_set_stop()) {
+ } else if (stop_) {
ret = OB_IN_STOP_STATE;
} else {
ret = queue_.push(task);
if (OB_SIZE_OVERFLOW == ret) {
ret = OB_EAGAIN;
}
- try_expand_thread_count();
+ if (this->idle_count() == 0) {
+ this->try_expand_one(max_thread_cnt_);
+ }
}
return ret;
}
template
-void ObSimpleThreadPoolBase::run1()
+bool ObSimpleThreadPoolBase::do_add_worker()
+{
+ if (stop_) {
+ return false;
+ }
+ Worker *w = nullptr;
+ {
+ lib::ObMutexGuard g(workers_lock_);
+
+ w = OB_NEW(Worker, "QThWker", this, static_cast(cur_worker_idx_++));
+ if (OB_ISNULL(w)) {
+ return false;
+ }
+ if (run_wrapper_ != nullptr) {
+ w->set_run_wrapper(run_wrapper_);
+ }
+ if (OB_SUCCESS != w->init()) {
+ OB_DELETE(Worker, "QThWker", w);
+ return false;
+ }
+ // Prevent reap_workers() from deleting this worker before start():
+ // the Threads constructor sets stop_=true, but this worker is not
+ // actually stopping — it just hasn't started yet.
+ w->has_set_stop() = false;
+ if (!workers_.add_last(&w->worker_node_)) {
+ OB_DELETE(Worker, "QThWker", w);
+ return false;
+ }
+ }
+ // Start outside the lock: the new worker may call try_expand_one →
+ // do_add_worker, which would deadlock if we held workers_lock_
+ if (OB_SUCCESS != w->start()) {
+ lib::ObMutexGuard g(workers_lock_);
+ workers_.remove(&w->worker_node_);
+ OB_DELETE(Worker, "QThWker", w);
+ return false;
+ }
+ return true;
+}
+
+template
+int ObSimpleThreadPoolBase::set_adaptive_thread(
+ int64_t min_thread_num, int64_t max_thread_num)
{
int ret = OB_SUCCESS;
- const int64_t thread_idx = get_thread_idx();
- if (NULL != name_) {
- lib::set_thread_name(name_, thread_idx);
+ if (min_thread_num < 0 || max_thread_num <= 0
+ || min_thread_num > max_thread_num) {
+ ret = OB_INVALID_ARGUMENT;
+ COMMON_LOG(WARN, "set_adaptive_thread failed",
+ KP(this), K(min_thread_num), K(max_thread_num));
+ } else {
+ min_thread_cnt_ = min_thread_num;
+ max_thread_cnt_ = max_thread_num;
+ if (OB_FAIL(ObSimpleThreadPoolDynamicMgr::get_instance().bind(this))) {
+ COMMON_LOG(WARN, "bind dynamic mgr failed", K(ret));
+ }
+ COMMON_LOG(INFO, "set adaptive thread success",
+ KP(this), K(min_thread_num), K(max_thread_num));
}
- while (!has_set_stop() && !(OB_NOT_NULL(&lib::Thread::current()) ? lib::Thread::current().has_set_stop() : false)) {
+ return ret;
+}
+
+template
+int ObSimpleThreadPoolBase::set_thread_count(int64_t n_threads)
+{
+ int ret = OB_SUCCESS;
+ if (n_threads <= 0) {
+ ret = OB_INVALID_ARGUMENT;
+ } else {
+ max_thread_cnt_ = n_threads;
+ if (min_thread_cnt_ > max_thread_cnt_) {
+ min_thread_cnt_ = max_thread_cnt_;
+ }
+ if (OB_FAIL(ObSimpleThreadPoolDynamicMgr::get_instance().bind(this))) {
+ COMMON_LOG(WARN, "bind dynamic mgr failed", K(ret));
+ }
+ }
+ return ret;
+}
+
+template
+void ObSimpleThreadPoolBase::stop()
+{
+ stop_ = true;
+ notify_stop();
+ lib::ObMutexGuard g(workers_lock_);
+ DLIST_FOREACH_NORET(wnode, workers_) {
+ wnode->get_data()->stop();
+ }
+}
+
+template
+void ObSimpleThreadPoolBase::wait()
+{
+ lib::ObMutexGuard g(workers_lock_);
+ DLIST_FOREACH_NORET(wnode, workers_) {
+ wnode->get_data()->wait();
+ }
+}
+
+template
+void ObSimpleThreadPoolBase::reap_workers()
+{
+ // Called only from Mgr background thread
+ lib::ObMutexGuard g(workers_lock_);
+ DLIST_FOREACH_REMOVESAFE_NORET(wnode, workers_) {
+ Worker *w = wnode->get_data();
+ if (w->has_set_stop()) {
+ workers_.remove(wnode);
+ w->wait();
+ w->destroy();
+ OB_DELETE(Worker, "QThWker", w);
+ }
+ }
+}
+
+template
+void ObSimpleThreadPoolBase::Worker::run1()
+{
+ if (NULL != pool_->name_) {
+ lib::set_thread_name(pool_->name_, idx_);
+ }
+ pool_->cur_worker_idx_ = idx_;
+ pool_->run1();
+ // Worker is exiting — mark stopped for Mgr to reap
+ this->has_set_stop() = true;
+}
+
+template
+void ObSimpleThreadPoolBase::run1()
+{
+ int64_t idle_since = 0;
+ while (!stop_ && !lib::Thread::current().has_set_stop()) {
QElemType *task = NULL;
- const int64_t curr_thread_num = get_thread_count();
- int64_t pop_before_ts = ObTimeUtility::current_time();
- if (OB_SUCC(queue_.pop(task, QUEUE_WAIT_TIME))) {
- IGNORE_RETURN inc_thread_idle_time(ObTimeUtility::current_time() - pop_before_ts);
- int64_t running_thread_cnt = inc_running_thread_cnt(1);
- if (running_thread_cnt == curr_thread_num
- && get_queue_num() > 0) {
- try_inc_thread_count(1);
+ bool expand = false;
+ int ret = this->pop_with_idle([&]() {
+ return queue_.pop(task, QUEUE_WAIT_TIME);
+ }, expand);
+
+ if (OB_SUCC(ret)) {
+ if (OB_NOT_NULL(task)) {
+ idle_since = 0;
+ if (expand) {
+ this->try_expand_one(max_thread_cnt_);
+ }
+ handle(static_cast(task));
+ }
+ } else if (OB_ENTRY_NOT_EXIST == ret) {
+ int64_t now = ObTimeUtility::current_time();
+ if (idle_since == 0) {
+ idle_since = now;
+ } else if (now - idle_since >= SHRINK_TIMEOUT_US) {
+ if (this->try_shrink_one(min_thread_cnt_)) {
+ return; // Worker::run1 will set has_set_stop
+ }
+ idle_since = 0;
}
- handle(static_cast(task));
- IGNORE_RETURN inc_running_thread_cnt(-1);
- } else {
- IGNORE_RETURN inc_thread_idle_time(ObTimeUtility::current_time() - pop_before_ts);
}
}
- if (has_set_stop()) {
+
+ if (stop_) {
+ int ret = OB_SUCCESS;
QElemType *task = NULL;
while (OB_SUCC(queue_.pop(task))) {
handle_drop(static_cast(task));
diff --git a/deps/oblib/src/lib/thread/thread.cpp b/deps/oblib/src/lib/thread/thread.cpp
index cfd1e1560..46835e85f 100644
--- a/deps/oblib/src/lib/thread/thread.cpp
+++ b/deps/oblib/src/lib/thread/thread.cpp
@@ -97,8 +97,8 @@ int Thread::start()
} else if (stack_size_ <= 0) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("invalid stack_size", K(ret), K(stack_size_));
-#if !defined(OB_USE_ASAN) && !defined(_WIN32)
- } else if (OB_ISNULL(stack_addr_ = g_stack_allocer.alloc(0 == GET_TENANT_ID() ? OB_SERVER_TENANT_ID : GET_TENANT_ID(), stack_size_ + SIG_STACK_SIZE))) {
+#if !defined(OB_USE_ASAN) && !defined(__APPLE__) && !defined(__ANDROID__) && !defined(_WIN32)
+ } else if (OB_ISNULL(stack_addr_ = g_stack_allocer.alloc(0 == GET_TENANT_ID() ? OB_SERVER_TENANT_ID : GET_TENANT_ID(), stack_size_))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR("alloc stack memory failed", K(stack_size_));
#endif
@@ -120,7 +120,7 @@ int Thread::start()
// On macOS/Android, pthread_attr_setstack often fails with EINVAL if address/size
// are not perfectly aligned or if the memory is already managed in a way
// that pthread doesn't like. Use setstacksize instead and let the system
- // allocate the stack, while keeping our stack_addr_ for stack_header logic.
+ // allocate the stack.
pret = pthread_attr_setstacksize(&attr, stack_size_);
if (pret != 0) {
// Fallback to default if setstacksize fails
@@ -156,7 +156,7 @@ int Thread::start()
}
}
} else {
- int64_t total_size = stack_size_ + SIG_STACK_SIZE;
+ int64_t total_size = stack_size_;
LOG_ERROR("pthread_attr_setstack failed", K(pret), K(total_size), K_(stack_size), KP(stack_addr_));
}
if (0 != pret) {
@@ -364,13 +364,13 @@ void Thread::destroy_stack()
{
#ifdef _WIN32
pth_ = pthread_null();
-#elif !defined(OB_USE_ASAN)
+#else
+#if !defined(OB_USE_ASAN)
if (stack_addr_ != nullptr) {
g_stack_allocer.dealloc(stack_addr_);
stack_addr_ = nullptr;
- pth_ = 0;
}
-#else
+#endif
pth_ = 0;
#endif
}
@@ -391,32 +391,16 @@ void* Thread::__th_start(void *arg)
current_thread_ = th;
th->tid_ = gettid();
-#if !defined(OB_USE_ASAN) && !defined(_WIN32)
- ObStackHeader *stack_header = ProtectedStackAllocator::stack_header(th->stack_addr_);
- abort_unless(stack_header->check_magic());
-
- #ifndef OB_USE_ASAN
- /**
- signal handler stack
- */
- #if !defined(__APPLE__)
- stack_t nss;
- stack_t oss;
- bzero(&nss, sizeof(nss));
- bzero(&oss, sizeof(oss));
- nss.ss_sp = &((char*)th->stack_addr_)[th->stack_size_];
- nss.ss_size = SIG_STACK_SIZE;
- bool restore_sigstack = false;
- if (-1 == sigaltstack(&nss, &oss)) {
- LOG_WARN_RET(OB_ERR_SYS, "sigaltstack failed, ignore it", K(errno));
- } else {
- restore_sigstack = true;
+#if !defined(OB_USE_ASAN) && !defined(_WIN32) && !defined(__APPLE__) && !defined(__ANDROID__)
+ ObStackHeader *stack_header = nullptr;
+ if (th->stack_addr_ != nullptr) {
+ stack_header = ProtectedStackAllocator::stack_header(th->stack_addr_);
+ abort_unless(stack_header->check_magic());
}
- DEFER(if (restore_sigstack) { sigaltstack(&oss, nullptr); });
- #endif // !__APPLE__
- #endif
- stack_header->pth_ = (uint64_t)pthread_self();
+ if (stack_header != nullptr) {
+ stack_header->pth_ = (uint64_t)pthread_self();
+ }
#endif
int ret = OB_SUCCESS;
diff --git a/deps/oblib/src/lib/utility/ob_macro_utils.h b/deps/oblib/src/lib/utility/ob_macro_utils.h
index 4d4d4e309..6794216b5 100644
--- a/deps/oblib/src/lib/utility/ob_macro_utils.h
+++ b/deps/oblib/src/lib/utility/ob_macro_utils.h
@@ -17,7 +17,10 @@
#ifndef _OB_MACRO_UTILS_H_
#define _OB_MACRO_UTILS_H_
-#ifdef _WIN32
+/* Windows MSVC (cl.exe) has no weak symbols; stubs in ob_log.cpp etc. rely on OB_WEAK_SYMBOL.
+ * clang-cl + lld-link support __attribute__((weak)) on COFF — use it so strong defs in
+ * src/share/ob_errno.cpp override stubs (same as ELF). Plain _WIN32 must leave this empty. */
+#if defined(_WIN32) && !(defined(__clang__) || defined(__GNUC__))
#define OB_WEAK_SYMBOL
#else
#define OB_WEAK_SYMBOL __attribute__((weak))
diff --git a/deps/oblib/src/lib/vector/ob_vector_util.cpp b/deps/oblib/src/lib/vector/ob_vector_util.cpp
index b5050d6fb..b2fd90e17 100644
--- a/deps/oblib/src/lib/vector/ob_vector_util.cpp
+++ b/deps/oblib/src/lib/vector/ob_vector_util.cpp
@@ -69,7 +69,7 @@ int init_vasg_logger(void* logger)
if (!check_vsag_init()) {
return -4016;
} else {
-#ifdef OB_BUILD_CDC_DISABLE_VSAG
+#if defined(OB_BUILD_CDC_DISABLE_VSAG) && !defined(OB_BUILD_EMBED_MODE)
#else
obvsag::set_logger(logger);
obvsag::set_log_level(static_cast(OB_LOGGER.get_log_level()));
@@ -82,7 +82,7 @@ int init_vasg_logger(void* logger)
bool check_vsag_init()
{
INIT_SUCC(ret);
-#ifdef OB_BUILD_CDC_DISABLE_VSAG
+#if defined(OB_BUILD_CDC_DISABLE_VSAG) && !defined(OB_BUILD_EMBED_MODE)
return true;
#else
return obvsag::is_init();
@@ -98,7 +98,7 @@ int create_index(obvsag::VectorIndexPtr& index_handler, int index_type,
bool bq_use_fht /*= false*/)
{
INIT_SUCC(ret);
-#ifdef OB_BUILD_CDC_DISABLE_VSAG
+#if defined(OB_BUILD_CDC_DISABLE_VSAG) && !defined(OB_BUILD_EMBED_MODE)
return ret;
#else
obvsag::set_block_size_limit(2*1024*1024);
@@ -123,7 +123,7 @@ int create_index(obvsag::VectorIndexPtr &index_handler, int index_type, const ch
bool use_reorder, float doc_prune_ratio, int window_size, void *allocator, int extra_info_size /* 0 */)
{
INIT_SUCC(ret);
-#ifdef OB_BUILD_CDC_DISABLE_VSAG
+#if defined(OB_BUILD_CDC_DISABLE_VSAG) && !defined(OB_BUILD_EMBED_MODE)
return ret;
#else
obvsag::set_block_size_limit(2*1024*1024);
@@ -137,7 +137,7 @@ int create_index(obvsag::VectorIndexPtr &index_handler, int index_type, const ch
int build_index(obvsag::VectorIndexPtr index_handler, float* vector_list, int64_t* ids, int dim, int size, char* extra_info /*= nullptr*/)
{
INIT_SUCC(ret);
-#ifdef OB_BUILD_CDC_DISABLE_VSAG
+#if defined(OB_BUILD_CDC_DISABLE_VSAG) && !defined(OB_BUILD_EMBED_MODE)
return ret;
#else
return obvsag::build_index(index_handler, vector_list, ids, dim, size, extra_info);
@@ -149,7 +149,7 @@ int build_index(obvsag::VectorIndexPtr &index_handler, uint32_t *lens, uint32_t
int size, char *extra_infos /*= nullptr*/)
{
INIT_SUCC(ret);
-#ifdef OB_BUILD_CDC_DISABLE_VSAG
+#if defined(OB_BUILD_CDC_DISABLE_VSAG) && !defined(OB_BUILD_EMBED_MODE)
return ret;
#else
return obvsag::build_index(index_handler, lens, dims, vals, ids, size, extra_infos);
@@ -159,7 +159,7 @@ int build_index(obvsag::VectorIndexPtr &index_handler, uint32_t *lens, uint32_t
int add_index(obvsag::VectorIndexPtr index_handler, float* vector_list, int64_t* ids, int dim, char *extra_info, int size)
{
INIT_SUCC(ret);
-#ifdef OB_BUILD_CDC_DISABLE_VSAG
+#if defined(OB_BUILD_CDC_DISABLE_VSAG) && !defined(OB_BUILD_EMBED_MODE)
return ret;
#else
return obvsag::add_index(index_handler, vector_list, ids, dim, size, extra_info);
@@ -170,7 +170,7 @@ int add_index(obvsag::VectorIndexPtr &index_handler, uint32_t *lens, uint32_t *d
char *extra_infos)
{
INIT_SUCC(ret);
-#ifdef OB_BUILD_CDC_DISABLE_VSAG
+#if defined(OB_BUILD_CDC_DISABLE_VSAG) && !defined(OB_BUILD_EMBED_MODE)
return ret;
#else
return obvsag::add_index(index_handler, lens, dims, vals, ids, size, extra_infos);
@@ -180,7 +180,7 @@ int add_index(obvsag::VectorIndexPtr &index_handler, uint32_t *lens, uint32_t *d
int get_index_number(obvsag::VectorIndexPtr index_handler, int64_t &size)
{
INIT_SUCC(ret);
-#ifdef OB_BUILD_CDC_DISABLE_VSAG
+#if defined(OB_BUILD_CDC_DISABLE_VSAG) && !defined(OB_BUILD_EMBED_MODE)
return ret;
#else
return obvsag::get_index_number(index_handler, size);
@@ -190,7 +190,7 @@ int get_index_number(obvsag::VectorIndexPtr index_handler, int64_t &size)
int get_index_type(obvsag::VectorIndexPtr index_handler)
{
INIT_SUCC(ret);
-#ifdef OB_BUILD_CDC_DISABLE_VSAG
+#if defined(OB_BUILD_CDC_DISABLE_VSAG) && !defined(OB_BUILD_EMBED_MODE)
return ret;
#else
return obvsag::get_index_type(index_handler);
@@ -204,7 +204,7 @@ int cal_distance_by_id(obvsag::VectorIndexPtr index_handler,
const float *&distances)
{
INIT_SUCC(ret);
-#ifdef OB_BUILD_CDC_DISABLE_VSAG
+#if defined(OB_BUILD_CDC_DISABLE_VSAG) && !defined(OB_BUILD_EMBED_MODE)
return ret;
#else
return obvsag::cal_distance_by_id(index_handler, vector, ids, count, distances);
@@ -218,7 +218,7 @@ int cal_distance_by_id(obvsag::VectorIndexPtr index_handler,
const float *&distances)
{
INIT_SUCC(ret);
-#ifdef OB_BUILD_CDC_DISABLE_VSAG
+#if defined(OB_BUILD_CDC_DISABLE_VSAG) && !defined(OB_BUILD_EMBED_MODE)
return ret;
#else
return obvsag::cal_distance_by_id(index_handler, len, dims, vals, ids, count, distances);
@@ -228,7 +228,7 @@ int cal_distance_by_id(obvsag::VectorIndexPtr index_handler,
int get_vid_bound(obvsag::VectorIndexPtr index_handler, int64_t &min_vid, int64_t &max_vid)
{
INIT_SUCC(ret);
-#ifdef OB_BUILD_CDC_DISABLE_VSAG
+#if defined(OB_BUILD_CDC_DISABLE_VSAG) && !defined(OB_BUILD_EMBED_MODE)
return ret;
#else
return obvsag::get_vid_bound(index_handler, min_vid, max_vid);
@@ -240,7 +240,7 @@ int get_extra_info_by_ids(obvsag::VectorIndexPtr& index_handler,
int64_t count,
char *extra_infos) {
INIT_SUCC(ret);
-#ifdef OB_BUILD_CDC_DISABLE_VSAG
+#if defined(OB_BUILD_CDC_DISABLE_VSAG) && !defined(OB_BUILD_EMBED_MODE)
return ret;
#else
return obvsag::get_extra_info_by_ids(index_handler, ids, count, extra_infos);
@@ -254,7 +254,7 @@ int knn_search(obvsag::VectorIndexPtr index_handler, float* query_vector,int dim
float distance_threshold)
{
INIT_SUCC(ret);
-#ifdef OB_BUILD_CDC_DISABLE_VSAG
+#if defined(OB_BUILD_CDC_DISABLE_VSAG) && !defined(OB_BUILD_EMBED_MODE)
return ret;
#else
return obvsag::knn_search(index_handler, query_vector, dim, topk,
@@ -271,7 +271,7 @@ int knn_search(obvsag::VectorIndexPtr index_handler, float* query_vector,int dim
bool need_extra_info, void *&iter_ctx, bool is_last_search)
{
INIT_SUCC(ret);
-#ifdef OB_BUILD_CDC_DISABLE_VSAG
+#if defined(OB_BUILD_CDC_DISABLE_VSAG) && !defined(OB_BUILD_EMBED_MODE)
return ret;
#else
return obvsag::knn_search(index_handler, query_vector, dim, topk,
@@ -288,7 +288,7 @@ int knn_search(obvsag::VectorIndexPtr index_handler, uint32_t len, uint32_t *dim
bool is_extra_info_filter, float valid_ratio, void *allocator, bool need_extra_info)
{
INIT_SUCC(ret);
-#ifdef OB_BUILD_CDC_DISABLE_VSAG
+#if defined(OB_BUILD_CDC_DISABLE_VSAG) && !defined(OB_BUILD_EMBED_MODE)
return ret;
#else
return obvsag::knn_search(index_handler, len, dims, vals, topk,
@@ -302,7 +302,7 @@ int knn_search(obvsag::VectorIndexPtr index_handler, uint32_t len, uint32_t *dim
int fserialize(obvsag::VectorIndexPtr index_handler, std::ostream& out_stream)
{
INIT_SUCC(ret);
-#ifdef OB_BUILD_CDC_DISABLE_VSAG
+#if defined(OB_BUILD_CDC_DISABLE_VSAG) && !defined(OB_BUILD_EMBED_MODE)
return ret;
#else
return obvsag::fserialize(index_handler, out_stream);
@@ -312,7 +312,7 @@ int fserialize(obvsag::VectorIndexPtr index_handler, std::ostream& out_stream)
int fdeserialize(obvsag::VectorIndexPtr& index_handler, std::istream& in_stream)
{
INIT_SUCC(ret);
-#ifdef OB_BUILD_CDC_DISABLE_VSAG
+#if defined(OB_BUILD_CDC_DISABLE_VSAG) && !defined(OB_BUILD_EMBED_MODE)
return ret;
#else
return obvsag::fdeserialize(index_handler,in_stream);
@@ -322,7 +322,7 @@ int fdeserialize(obvsag::VectorIndexPtr& index_handler, std::istream& in_stream)
int delete_index(obvsag::VectorIndexPtr& index_handler)
{
INIT_SUCC(ret);
-#ifdef OB_BUILD_CDC_DISABLE_VSAG
+#if defined(OB_BUILD_CDC_DISABLE_VSAG) && !defined(OB_BUILD_EMBED_MODE)
return ret;
#else
return obvsag::delete_index(index_handler);
@@ -331,7 +331,7 @@ int delete_index(obvsag::VectorIndexPtr& index_handler)
void delete_iter_ctx(void *iter_ctx)
{
-#ifdef OB_BUILD_CDC_DISABLE_VSAG
+#if defined(OB_BUILD_CDC_DISABLE_VSAG) && !defined(OB_BUILD_EMBED_MODE)
#else
obvsag::delete_iter_ctx(iter_ctx);
#endif
@@ -341,7 +341,7 @@ void delete_iter_ctx(void *iter_ctx)
uint64_t estimate_memory(obvsag::VectorIndexPtr& index_handler, const uint64_t row_count, const bool is_build)
{
INIT_SUCC(ret);
-#ifdef OB_BUILD_CDC_DISABLE_VSAG
+#if defined(OB_BUILD_CDC_DISABLE_VSAG) && !defined(OB_BUILD_EMBED_MODE)
return ret;
#else
return obvsag::estimate_memory(index_handler, row_count, is_build);
@@ -352,7 +352,7 @@ uint64_t estimate_memory(obvsag::VectorIndexPtr& index_handler, const uint64_t r
int immutable_optimize(obvsag::VectorIndexPtr& index_handler)
{
int ret = OB_SUCCESS;
-#ifdef OB_BUILD_CDC_DISABLE_VSAG
+#if defined(OB_BUILD_CDC_DISABLE_VSAG) && !defined(OB_BUILD_EMBED_MODE)
return ret;
#else
return obvsag::immutable_optimize(index_handler);
diff --git a/deps/oblib/src/lib/vector/ob_vsag_adaptor.cpp b/deps/oblib/src/lib/vector/ob_vsag_adaptor.cpp
index 16b27d369..e9bb05fb1 100644
--- a/deps/oblib/src/lib/vector/ob_vsag_adaptor.cpp
+++ b/deps/oblib/src/lib/vector/ob_vsag_adaptor.cpp
@@ -18,7 +18,7 @@
#include "ob_vsag_adaptor.h"
#include