From 7aaa87eac4fd51a6d723147e0b3655d2eac76433 Mon Sep 17 00:00:00 2001 From: efrain robles Date: Fri, 17 Jul 2026 17:32:10 -0600 Subject: [PATCH 1/3] [.NET] Add Databricks Zerobus Ingest SDK for .NET Initial implementation of the .NET SDK wrapping the Rust C FFI (rust/ffi/) via P/Invoke. - JSON, Protocol Buffers, and Arrow Flight ingestion streams - Fluent StreamBuilder API with typed sub-builders - Thread-safe streams with ReaderWriterLockSlim - IAsyncDisposable support (.NET 8+) - HeadersProviderBridge for custom authentication callbacks - ProtoSchema for Unity Catalog schema generation - Integration tests with real gRPC mock server (zerobus_service.proto) - 51 tests: 30 unit + 21 integration (4 self-test, 4 smoke, 17 end-to-end) - CI/CD workflows integrated into push.yml and release-dotnet.yml - Cross-platform NuGet package (win-x64, linux-x64, linux-arm64, osx-x64, osx-arm64) Signed-off-by: Efrain Robles Gonzalez --- .github/workflows/ci-dotnet.yml | 109 ++++ .github/workflows/push.yml | 28 +- .github/workflows/release-dotnet.yml | 199 +++++++ .gitignore | 18 + CLAUDE.md | 10 +- README.md | 3 +- dotnet/CHANGELOG.md | 22 + dotnet/CONTRIBUTING.md | 123 +++++ dotnet/Directory.Build.props | 10 + dotnet/Directory.Packages.props | 24 + dotnet/LICENSE | 201 +++++++ dotnet/Makefile | 68 +++ dotnet/NEXT_CHANGELOG.md | 3 + dotnet/NuGet.Config | 7 + dotnet/README.md | 444 +++++++++++++++ dotnet/Zerobus-DotNet.slnx | 17 + dotnet/build_native.sh | 122 ++++ .../ArrowIngestion/ArrowIngestion.csproj | 15 + .../ArrowIngestion/ArrowIngestionExample.cs | 55 ++ .../JsonIngestion/BatchIngestionExample.cs | 46 ++ .../JsonIngestion/JsonIngestion.csproj | 14 + .../JsonIngestion/SingleRecordExample.cs | 51 ++ .../ProtoIngestion/BatchIngestionExample.cs | 44 ++ .../ProtoIngestion/ProtoIngestion.csproj | 15 + .../ProtoIngestion/SingleRecordExample.cs | 46 ++ dotnet/src/Databricks.Zerobus/AckCallback.cs | 17 + .../ArrowStreamConfigurationOptions.cs | 204 +++++++ .../Databricks.Zerobus/BaseZerobusStream.cs | 294 ++++++++++ dotnet/src/Databricks.Zerobus/CLAUDE.md | 114 ++++ .../Databricks.Zerobus.csproj | 67 +++ dotnet/src/Databricks.Zerobus/EncodedBatch.cs | 27 + .../src/Databricks.Zerobus/HeadersProvider.cs | 9 + .../Databricks.Zerobus/IPCCompressionType.cs | 16 + .../Native/HeadersProviderBridge.cs | 110 ++++ .../Native/InteropStructs.cs | 96 ++++ .../Native/NativeLibraryResolver.cs | 184 +++++++ .../Native/NativeMethods.cs | 272 +++++++++ .../SafeHandles/ZerobusArrowStreamHandle.cs | 21 + .../SafeHandles/ZerobusProtoSchemaHandle.cs | 21 + .../Native/SafeHandles/ZerobusSdkHandle.cs | 21 + .../Native/SafeHandles/ZerobusStreamHandle.cs | 21 + .../NonRetriableException.cs | 23 + dotnet/src/Databricks.Zerobus/ProtoSchema.cs | 120 ++++ .../src/Databricks.Zerobus/StreamBuilder.cs | 396 +++++++++++++ .../StreamConfigurationOptions.cs | 267 +++++++++ .../src/Databricks.Zerobus/TableProperties.cs | 56 ++ .../Databricks.Zerobus/ZerobusArrowStream.cs | 202 +++++++ .../Databricks.Zerobus/ZerobusException.cs | 37 ++ .../Databricks.Zerobus/ZerobusJsonStream.cs | 167 ++++++ .../Databricks.Zerobus/ZerobusProtoStream.cs | 203 +++++++ dotnet/src/Databricks.Zerobus/ZerobusSdk.cs | 521 ++++++++++++++++++ .../build/Databricks.Zerobus.props | 5 + .../build/Databricks.Zerobus.targets | 43 ++ ...Databricks.Zerobus.IntegrationTests.csproj | 36 ++ .../IntegrationTests.cs | 376 +++++++++++++ .../MockServerSelfTest.cs | 177 ++++++ .../MockZerobusServer.cs | 109 ++++ .../MockZerobusService.cs | 202 +++++++ .../NativeLibraryHelper.cs | 52 ++ .../NativeLoadSmokeTest.cs | 90 +++ .../Protos/zerobus_service.proto | 234 ++++++++ .../ArrowStreamConfigurationOptionsTests.cs | 53 ++ .../Databricks.Zerobus.Tests.csproj | 24 + .../ExceptionTests.cs | 44 ++ .../StreamBuilderTests.cs | 74 +++ .../StreamConfigurationOptionsTests.cs | 106 ++++ .../tools/GenerateProto/GenerateProto.csproj | 16 + dotnet/tools/GenerateProto/Program.cs | 56 ++ 68 files changed, 6871 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/ci-dotnet.yml create mode 100644 .github/workflows/release-dotnet.yml create mode 100644 dotnet/CHANGELOG.md create mode 100644 dotnet/CONTRIBUTING.md create mode 100644 dotnet/Directory.Build.props create mode 100644 dotnet/Directory.Packages.props create mode 100644 dotnet/LICENSE create mode 100644 dotnet/Makefile create mode 100644 dotnet/NEXT_CHANGELOG.md create mode 100644 dotnet/NuGet.Config create mode 100644 dotnet/README.md create mode 100644 dotnet/Zerobus-DotNet.slnx create mode 100644 dotnet/build_native.sh create mode 100644 dotnet/examples/ArrowIngestion/ArrowIngestion.csproj create mode 100644 dotnet/examples/ArrowIngestion/ArrowIngestionExample.cs create mode 100644 dotnet/examples/JsonIngestion/BatchIngestionExample.cs create mode 100644 dotnet/examples/JsonIngestion/JsonIngestion.csproj create mode 100644 dotnet/examples/JsonIngestion/SingleRecordExample.cs create mode 100644 dotnet/examples/ProtoIngestion/BatchIngestionExample.cs create mode 100644 dotnet/examples/ProtoIngestion/ProtoIngestion.csproj create mode 100644 dotnet/examples/ProtoIngestion/SingleRecordExample.cs create mode 100644 dotnet/src/Databricks.Zerobus/AckCallback.cs create mode 100644 dotnet/src/Databricks.Zerobus/ArrowStreamConfigurationOptions.cs create mode 100644 dotnet/src/Databricks.Zerobus/BaseZerobusStream.cs create mode 100644 dotnet/src/Databricks.Zerobus/CLAUDE.md create mode 100644 dotnet/src/Databricks.Zerobus/Databricks.Zerobus.csproj create mode 100644 dotnet/src/Databricks.Zerobus/EncodedBatch.cs create mode 100644 dotnet/src/Databricks.Zerobus/HeadersProvider.cs create mode 100644 dotnet/src/Databricks.Zerobus/IPCCompressionType.cs create mode 100644 dotnet/src/Databricks.Zerobus/Native/HeadersProviderBridge.cs create mode 100644 dotnet/src/Databricks.Zerobus/Native/InteropStructs.cs create mode 100644 dotnet/src/Databricks.Zerobus/Native/NativeLibraryResolver.cs create mode 100644 dotnet/src/Databricks.Zerobus/Native/NativeMethods.cs create mode 100644 dotnet/src/Databricks.Zerobus/Native/SafeHandles/ZerobusArrowStreamHandle.cs create mode 100644 dotnet/src/Databricks.Zerobus/Native/SafeHandles/ZerobusProtoSchemaHandle.cs create mode 100644 dotnet/src/Databricks.Zerobus/Native/SafeHandles/ZerobusSdkHandle.cs create mode 100644 dotnet/src/Databricks.Zerobus/Native/SafeHandles/ZerobusStreamHandle.cs create mode 100644 dotnet/src/Databricks.Zerobus/NonRetriableException.cs create mode 100644 dotnet/src/Databricks.Zerobus/ProtoSchema.cs create mode 100644 dotnet/src/Databricks.Zerobus/StreamBuilder.cs create mode 100644 dotnet/src/Databricks.Zerobus/StreamConfigurationOptions.cs create mode 100644 dotnet/src/Databricks.Zerobus/TableProperties.cs create mode 100644 dotnet/src/Databricks.Zerobus/ZerobusArrowStream.cs create mode 100644 dotnet/src/Databricks.Zerobus/ZerobusException.cs create mode 100644 dotnet/src/Databricks.Zerobus/ZerobusJsonStream.cs create mode 100644 dotnet/src/Databricks.Zerobus/ZerobusProtoStream.cs create mode 100644 dotnet/src/Databricks.Zerobus/ZerobusSdk.cs create mode 100644 dotnet/src/Databricks.Zerobus/build/Databricks.Zerobus.props create mode 100644 dotnet/src/Databricks.Zerobus/build/Databricks.Zerobus.targets create mode 100644 dotnet/tests/Databricks.Zerobus.IntegrationTests/Databricks.Zerobus.IntegrationTests.csproj create mode 100644 dotnet/tests/Databricks.Zerobus.IntegrationTests/IntegrationTests.cs create mode 100644 dotnet/tests/Databricks.Zerobus.IntegrationTests/MockServerSelfTest.cs create mode 100644 dotnet/tests/Databricks.Zerobus.IntegrationTests/MockZerobusServer.cs create mode 100644 dotnet/tests/Databricks.Zerobus.IntegrationTests/MockZerobusService.cs create mode 100644 dotnet/tests/Databricks.Zerobus.IntegrationTests/NativeLibraryHelper.cs create mode 100644 dotnet/tests/Databricks.Zerobus.IntegrationTests/NativeLoadSmokeTest.cs create mode 100644 dotnet/tests/Databricks.Zerobus.IntegrationTests/Protos/zerobus_service.proto create mode 100644 dotnet/tests/Databricks.Zerobus.Tests/ArrowStreamConfigurationOptionsTests.cs create mode 100644 dotnet/tests/Databricks.Zerobus.Tests/Databricks.Zerobus.Tests.csproj create mode 100644 dotnet/tests/Databricks.Zerobus.Tests/ExceptionTests.cs create mode 100644 dotnet/tests/Databricks.Zerobus.Tests/StreamBuilderTests.cs create mode 100644 dotnet/tests/Databricks.Zerobus.Tests/StreamConfigurationOptionsTests.cs create mode 100644 dotnet/tools/GenerateProto/GenerateProto.csproj create mode 100644 dotnet/tools/GenerateProto/Program.cs diff --git a/.github/workflows/ci-dotnet.yml b/.github/workflows/ci-dotnet.yml new file mode 100644 index 00000000..9efc6755 --- /dev/null +++ b/.github/workflows/ci-dotnet.yml @@ -0,0 +1,109 @@ +# Build .NET SDK — CI +# Called from push.yml (path-filtered) and cross-sdk (on rust/** changes). +# Also invocable manually via workflow_dispatch. +name: Build .NET + +on: + workflow_call: + inputs: + use_local_sdk: + description: 'Ignored — .NET uses pre-built FFI binaries, not local Rust compilation.' + type: boolean + default: false + artifact_suffix: + description: 'Suffix appended to artifact names for uniqueness' + type: string + default: '' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + DOTNET_CLI_TELEMETRY_OPTOUT: '1' + DOTNET_NOLOGO: '1' + +jobs: + # ─── Format check ──────────────────────────────────────────────── + fmt: + name: Format Check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + - uses: actions/setup-dotnet@4d6c8fcf3c8baa263966f8c6d72e6f47f76c9559 # v5 + with: + dotnet-version: '8.0.x' + - name: Check formatting + working-directory: dotnet + run: | + dotnet format --verify-no-changes --verbosity detailed || { + echo "::error::Code formatting violations found. Run 'dotnet format' locally to fix." + exit 1 + } + + # ─── Build & Test (matrix) ────────────────────────────────────── + test: + name: Test (${{ matrix.os }}) + needs: fmt + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + + - uses: actions/setup-dotnet@4d6c8fcf3c8baa263966f8c6d72e6f47f76c9559 # v5 + with: + dotnet-version: '8.0.x' + + - name: Restore dependencies + working-directory: dotnet + run: dotnet restore --configfile NuGet.Config + + - name: Build + working-directory: dotnet + run: dotnet build --no-restore --configuration Release + + - name: Test + working-directory: dotnet + run: dotnet test --no-build --configuration Release --logger "trx;LogFileName=test-results.trx" + + - name: Upload test results + if: always() + uses: actions/upload-artifact@b4b15ac8a8a6516bb370cc420f053ba1b865e6e4 # v4 + with: + name: test-results-${{ matrix.os }}-dotnet${{ inputs.artifact_suffix }} + path: | + dotnet/tests/**/TestResults/*.trx + dotnet/tests/**/TestResults/*.xml + retention-days: 7 + if-no-files-found: ignore + + # ─── Pack (Linux only — single platform for NuGet) ────────────── + pack: + name: Pack NuGet + needs: test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + + - uses: actions/setup-dotnet@4d6c8fcf3c8baa263966f8c6d72e6f47f76c9559 # v5 + with: + dotnet-version: '8.0.x' + + - name: Restore + working-directory: dotnet + run: dotnet restore --configfile NuGet.Config + + - name: Pack + working-directory: dotnet + run: dotnet pack src/Databricks.Zerobus --no-restore --configuration Release -o artifacts + + - name: Upload NuGet package + uses: actions/upload-artifact@b4b15ac8a8a6516bb370cc420f053ba1b865e6e4 # v4 + with: + name: dotnet-nuget${{ inputs.artifact_suffix }} + path: dotnet/artifacts/*.nupkg + if-no-files-found: error diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 60a0ba35..baad7674 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -11,7 +11,7 @@ jobs: runs-on: group: databricks-protected-runner-group labels: linux-ubuntu-latest - needs: [changes, rust, java, typescript, python, go, purego, cpp] + needs: [changes, rust, java, typescript, python, go, purego, cpp, dotnet] if: always() steps: - run: | @@ -22,7 +22,8 @@ jobs: "${{ needs.python.result }}" \ "${{ needs.go.result }}" \ "${{ needs.purego.result }}" \ - "${{ needs.cpp.result }}"; do + "${{ needs.cpp.result }}" \ + "${{ needs.dotnet.result }}"; do if [[ "$result" != "success" && "$result" != "skipped" ]]; then echo "Job failed or was cancelled: $result" exit 1 @@ -41,6 +42,7 @@ jobs: go: ${{ steps.filter.outputs.go }} purego: ${{ steps.filter.outputs.purego }} cpp: ${{ steps.filter.outputs.cpp }} + dotnet: ${{ steps.filter.outputs.dotnet }} steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 @@ -71,6 +73,9 @@ jobs: cpp: - 'cpp/**' - '.github/workflows/ci-cpp.yml' + dotnet: + - 'dotnet/**' + - '.github/workflows/ci-dotnet.yml' # Blocking jobs — required for merge rust: @@ -129,6 +134,14 @@ jobs: contents: read uses: ./.github/workflows/ci-cpp.yml + dotnet: + needs: changes + if: needs.changes.outputs.dotnet == 'true' + permissions: + id-token: write + contents: read + uses: ./.github/workflows/ci-dotnet.yml + # Non-blocking cross-SDK jobs — run all SDK tests against the local Rust # source when rust/** changes, to catch breakage before a new FFI release. # These are intentionally excluded from `gate` so they don't block merging. @@ -181,3 +194,14 @@ jobs: with: use_local_sdk: true artifact_suffix: '-local-sdk' + + cross-sdk-dotnet: + needs: changes + if: needs.changes.outputs.rust == 'true' + permissions: + id-token: write + contents: read + uses: ./.github/workflows/ci-dotnet.yml + with: + use_local_sdk: true + artifact_suffix: '-local-sdk' diff --git a/.github/workflows/release-dotnet.yml b/.github/workflows/release-dotnet.yml new file mode 100644 index 00000000..e135c857 --- /dev/null +++ b/.github/workflows/release-dotnet.yml @@ -0,0 +1,199 @@ +# Release .NET SDK +# Triggered by tag dotnet/v* or manually with an FFI build run ID. +# Downloads pre-built native FFI libraries, bundles them into the NuGet package, +# and optionally publishes to NuGet.org. +name: Release .NET + +on: + push: + tags: + - 'dotnet/v*' + workflow_dispatch: + inputs: + ffi-run-id: + description: 'Run ID of the release-ffi.yml build (required for manual runs)' + type: string + required: false + default: '' + publish-to-nuget: + description: 'Publish to NuGet.org (requires NUGET_API_KEY secret)' + type: boolean + default: false + +run-name: Release .NET SDK ${{ github.ref_name }} + +env: + DOTNET_CLI_TELEMETRY_OPTOUT: '1' + DOTNET_NOLOGO: '1' + +jobs: + # ─── Resolve FFI run ID (auto-detect on tag, use input on manual) ─ + resolve-ffi: + name: Resolve FFI artifacts + runs-on: ubuntu-latest + outputs: + ffi-run-id: ${{ steps.resolve.outputs.ffi-run-id }} + package-version: ${{ steps.resolve.outputs.package-version }} + steps: + - name: Resolve FFI run ID and version + id: resolve + env: + GH_TOKEN: ${{ github.token }} + run: | + # Determine FFI run ID + if [[ -n "${{ inputs.ffi-run-id }}" ]]; then + FFI_RUN_ID="${{ inputs.ffi-run-id }}" + else + # Auto-detect: latest successful release-ffi workflow run + echo "Auto-detecting latest successful release-ffi run..." + FFI_RUN_ID=$(gh run list \ + --repo "${{ github.repository }}" \ + --workflow release-ffi.yml \ + --status success \ + --limit 1 \ + --json databaseId \ + --jq '.[0].databaseId') + if [[ -z "$FFI_RUN_ID" || "$FFI_RUN_ID" == "null" ]]; then + echo "::error::No successful release-ffi run found. Run it first or provide ffi-run-id." + exit 1 + fi + fi + echo "ffi-run-id=$FFI_RUN_ID" >> "$GITHUB_OUTPUT" + echo "Using FFI run: $FFI_RUN_ID" + + # Determine package version + if [[ "${{ github.ref_type }}" == "tag" ]]; then + # Extract version from tag: dotnet/v1.2.3 → 1.2.3 + VERSION="${{ github.ref_name }}" + VERSION="${VERSION#dotnet/v}" + else + # Use csproj version for manual runs + VERSION="" + fi + echo "package-version=$VERSION" >> "$GITHUB_OUTPUT" + echo "Package version: ${VERSION:-}" + + # ─── Download FFI native libraries from release-ffi workflow ───── + download-ffi: + name: Download FFI artifacts + needs: resolve-ffi + runs-on: ubuntu-latest + outputs: + ffi-artifacts-dir: ${{ steps.download.outputs.dir }} + steps: + - name: Download all FFI artifacts + id: download + env: + GH_TOKEN: ${{ github.token }} + run: | + mkdir -p ffi-artifacts + gh run download "${{ needs.resolve-ffi.outputs.ffi-run-id }}" \ + --repo "${{ github.repository }}" \ + --pattern 'ffi-*' \ + --dir ffi-artifacts + echo "dir=$(pwd)/ffi-artifacts" >> "$GITHUB_OUTPUT" + echo "Downloaded artifacts:" + find ffi-artifacts -type f -exec ls -lh {} \; + + # ─── Build package with native libs ───────────────────────────── + build: + name: Build & Pack NuGet + needs: [resolve-ffi, download-ffi] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + + - uses: actions/setup-dotnet@4d6c8fcf3c8baa263966f8c6d72e6f47f76c9559 # v5 + with: + dotnet-version: '8.0.x' + + # Map FFI artifact directories to NuGet runtime identifiers + # FFI naming: linux-x86_64, linux-aarch64, darwin-x86_64, darwin-aarch64, windows-x86_64 + # NuGet RIDs: linux-x64, linux-arm64, osx-x64, osx-arm64, win-x64 + - name: Place native libraries in runtimes/ + working-directory: dotnet + run: | + set -euo pipefail + FFI_DIR="${{ needs.download-ffi.outputs.ffi-artifacts-dir }}" + RUNTIMES="src/Databricks.Zerobus/runtimes" + + # Mapping: FFI artifact_dir → NuGet RID → native file name + declare -A MAP=( + ['linux-x86_64']='linux-x64:libzerobus_ffi.so' + ['linux-aarch64']='linux-arm64:libzerobus_ffi.so' + ['darwin-x86_64']='osx-x64:libzerobus_ffi.dylib' + ['darwin-aarch64']='osx-arm64:libzerobus_ffi.dylib' + ['windows-x86_64']='win-x64:zerobus_ffi.dll' + ) + + for ffi_dir in "${!MAP[@]}"; do + IFS=':' read -r rid libname <<< "${MAP[$ffi_dir]}" + src="$FFI_DIR/ffi-$ffi_dir/$libname" + dest="$RUNTIMES/$rid/native/$libname" + + if [[ -f "$src" ]]; then + mkdir -p "$(dirname "$dest")" + cp -v "$src" "$dest" + else + echo "::warning::Native lib not found at $src (FFI dir: ffi-$ffi_dir)." + echo "::warning::macOS FFI builds currently produce only static libs (.a)." + echo "::warning::Dynamic libs for macOS require zigbuild with cdylib crate type." + fi + done + + echo "" + echo "Final runtimes tree:" + find "$RUNTIMES" -type f -exec ls -lh {} \; + + # Set version from tag, or keep csproj version for manual runs + - name: Set package version + if: needs.resolve-ffi.outputs.package-version != '' + working-directory: dotnet + run: | + csproj="src/Databricks.Zerobus/Databricks.Zerobus.csproj" + sed -i "s|[0-9.]*|${{ needs.resolve-ffi.outputs.package-version }}|" "$csproj" + grep '' "$csproj" + + - name: Restore + working-directory: dotnet + run: dotnet restore --configfile NuGet.Config + + - name: Build + working-directory: dotnet + run: dotnet build --no-restore --configuration Release + + - name: Test + working-directory: dotnet + run: dotnet test --no-build --configuration Release + + - name: Pack + working-directory: dotnet + run: dotnet pack src/Databricks.Zerobus --no-build --configuration Release -o artifacts + + - name: Upload NuGet package + uses: actions/upload-artifact@b4b15ac8a8a6516bb370cc420f053ba1b865e6e4 # v4 + with: + name: dotnet-nuget-release + path: dotnet/artifacts/*.nupkg + if-no-files-found: error + + # TODO: Uncomment when NuGet.org publishing is configured + # ─── Publish to NuGet.org ─────────────────────────────────────── + # publish: + # name: Publish to NuGet.org + # if: inputs.publish-to-nuget + # needs: build + # runs-on: ubuntu-latest + # steps: + # - name: Download package + # uses: actions/download-artifact@fa0bad59f5c22a1081e21447fd2f06f1a42bb61b # v4 + # with: + # name: dotnet-nuget-release + # path: packages/ + # + # - name: Push to NuGet.org + # run: | + # dotnet nuget push packages/*.nupkg \ + # --api-key "${{ secrets.NUGET_API_KEY }}" \ + # --source https://api.nuget.org/v3/index.json \ + # --skip-duplicate diff --git a/.gitignore b/.gitignore index e59b4a2a..05b85f1c 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,21 @@ Thumbs.db # Databricks .databricks .cargo/ + +# .NET +**/bin/ +**/obj/ +*.nupkg +*.snupkg +*.DotSettings.user +*.user +*.suo +TestResults/ +artifacts/ +dotnet/.dotnet/ + + +# .NET native libraries +dotnet/src/Databricks.Zerobus/runtimes/*/native/*.dll +dotnet/src/Databricks.Zerobus/runtimes/*/native/*.so +dotnet/src/Databricks.Zerobus/runtimes/*/native/*.dylib diff --git a/CLAUDE.md b/CLAUDE.md index 9b7ff843..3ab8a944 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,7 +2,7 @@ ## Architecture -This is a polyglot SDK monorepo. A single Rust core (`rust/sdk/`) implements gRPC streaming, OAuth, recovery, and ingestion logic. Five wrapper SDKs expose it to other languages: +This is a polyglot SDK monorepo. A single Rust core (`rust/sdk/`) implements gRPC streaming, OAuth, recovery, and ingestion logic. Seven wrapper SDKs expose it to other languages: | SDK | Directory | Binding mechanism | Build system | |------------|----------------|--------------------------|------------------| @@ -12,10 +12,11 @@ This is a polyglot SDK monorepo. A single Rust core (`rust/sdk/`) implements gRP | Java | `java/` | JNI (`rust/jni/`) | Maven | | Go | `go/` | cgo + static FFI lib | Go modules | | C++ | `cpp/` | C FFI (`rust/ffi/`) | CMake | +| .NET | `dotnet/` | P/Invoke (`rust/ffi/`) | MSBuild / NuGet | The data flow is: **User code → Wrapper API → FFI boundary → Rust core → gRPC → Zerobus service**. -Go and C++ use the C FFI layer (`rust/ffi/`, header auto-generated by cbindgen). Java has its own JNI crate (`rust/jni/`). Python and TypeScript bind Rust directly via PyO3 and NAPI-RS respectively. +Go, C++, and .NET use the C FFI layer (`rust/ffi/`, header auto-generated by cbindgen). Java has its own JNI crate (`rust/jni/`). Python and TypeScript bind Rust directly via PyO3 and NAPI-RS respectively. ## Writing SDK client code — performance rules @@ -45,7 +46,7 @@ When writing or reviewing a README, doc comment, or example, the **first** inges **Strict semver.** Breaking changes are only allowed in major version bumps. - Removing or renaming a public API element is a breaking change. Mark it `#[deprecated]` (or language equivalent) in a minor release first, then remove in the next major. -- Changing the FFI C header (`rust/ffi/zerobus.h`) in a non-additive way breaks Go and Java. Treat FFI signature changes with the same rigor as public API changes. +- Changing the FFI C header (`rust/ffi/zerobus.h`) in a non-additive way breaks Go, C++, and .NET. Treat FFI signature changes with the same rigor as public API changes. - Each SDK is versioned independently. A Rust core bump does not automatically require wrapper version bumps unless the wrapper's public API changes. - Release tags follow `/v` (e.g., `rust/v1.1.0`, `python/v1.0.0`). @@ -64,6 +65,7 @@ Because every non-Rust SDK crosses a foreign-function boundary, keep these in mi - Go uses `runtime.Pinner` to prevent GC from relocating memory while Rust holds a pointer. Never remove pinner calls. - Go's `cgo.Handle` registry keeps `HeadersProvider` callbacks alive; leaking these handles leaks Go objects. - Java has no finalizers on streams — users must call `close()` or use try-with-resources. Forgetting this leaks JNI resources. +- .NET has no finalizers on streams — users must call `Dispose()` or use `using` statements. `SafeHandle` provides critical-finalization as a backstop. - Python and TypeScript rely on GC-triggered cleanup via PyO3/NAPI-RS reference counting. ### Thread safety @@ -72,6 +74,7 @@ Because every non-Rust SDK crosses a foreign-function boundary, keep these in mi - Python: async streams are safe; sync streams are single-threaded per instance. - TypeScript: async-safe via Node.js event loop. - Java: **not thread-safe** — external synchronization required for concurrent access. +- .NET: thread-safe — uses `ReaderWriterLockSlim` internally for concurrent ingest. ## Build and Test Commands @@ -85,6 +88,7 @@ Each SDK has its own build/test commands. Run from the SDK directory: | Java | `mvn compile` | `mvn test` | `mvn spotless:check` | `mvn spotless:apply` | | Go | `make build` | `make test` | `make lint` | `make fmt` | | C++ | `make build` | `make test` | `make lint` | `make fmt` | +| .NET | `dotnet build` | `dotnet test` | `dotnet format --verify-no-changes` | `dotnet format` | ## CI/CD diff --git a/README.md b/README.md index 09d69b39..94b65b1a 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Zerobus is a high-throughput streaming service for direct data ingestion into Da | TypeScript | [`typescript/`](typescript/) | [`@databricks/zerobus-ingest-sdk`](https://www.npmjs.com/package/@databricks/zerobus-ingest-sdk) | | Java | [`java/`](java/) | [`com.databricks:zerobus-ingest-sdk`](https://central.sonatype.com/artifact/com.databricks/zerobus-ingest-sdk) | | C++ | [`cpp/`](cpp/) | Source / CMake (`zerobus::zerobus`) | +| .NET | [`dotnet/`](dotnet/) | [`Databricks.Zerobus.Ingest`](https://www.nuget.org/packages/Databricks.Zerobus.Ingest) | ## Platform Support @@ -133,7 +134,7 @@ Instead of writing `.proto` files by hand, each SDK ships a tool to generate pro ### Arrow Flight ingestion (Beta) -Available in the Rust, Python, Go, TypeScript, and Java SDKs starting from their 2.0.0 releases, and in the C++ SDK from its initial `0.1.0` release. Currently in Beta — the API is stabilising but may still change before reaching GA. A third record format option alongside JSON and Protocol Buffers: send Apache Arrow `RecordBatch` data directly to Zerobus over the Arrow Flight protocol, on the same gRPC connection. Best fit when: +Available in the Rust, Python, Go, TypeScript, Java, and .NET SDKs starting from their 2.0.0 releases, and in the C++ SDK from its initial `0.1.0` release. Currently in Beta — the API is stabilising but may still change before reaching GA. A third record format option alongside JSON and Protocol Buffers: send Apache Arrow `RecordBatch` data directly to Zerobus over the Arrow Flight protocol, on the same gRPC connection. Best fit when: - Your workload is naturally columnar or batched — analytics pipelines, gateways aggregating short windows of rows, wide/numeric schemas where row-by-row serialization adds noticeable CPU overhead. - Your application already produces Arrow data — pyarrow, the [arrow-rs](https://github.com/apache/arrow-rs) crates, DataFusion, Polars, or other libraries built on Arrow. diff --git a/dotnet/CHANGELOG.md b/dotnet/CHANGELOG.md new file mode 100644 index 00000000..bd1052f7 --- /dev/null +++ b/dotnet/CHANGELOG.md @@ -0,0 +1,22 @@ +# Changelog + +All notable changes to the Databricks Zerobus Ingest SDK for .NET. + +## [0.1.0] - 2026-07-17 + +### Added +- Initial release of the .NET SDK +- JSON ingestion stream (`ZerobusJsonStream`) with single and batch ingestion +- Protocol Buffers ingestion stream (`ZerobusProtoStream`) with typed and pre-serialized ingestion +- Arrow Flight ingestion stream (`ZerobusArrowStream`) — Beta +- `ZerobusSdk` with `CreateBuilder()` for advanced configuration +- `StreamBuilder` fluent API with JsonStreamBuilder, ProtoStreamBuilder, ArrowStreamBuilder +- `ProtoSchema` for generating protobuf descriptors from Unity Catalog table JSON +- `StreamConfigurationOptions` and `ArrowStreamConfigurationOptions` with builder pattern +- Acknowledgment callback support via `AckOnAckDelegate` and `AckOnErrorDelegate` +- Native library auto-resolution via `DllImportResolver` (.NET 8+) and `LoadLibrary`/`dlopen` (netstandard2.0) +- NuGet package with bundled native libraries for win-x64, linux-x64, linux-arm64, osx-x64, osx-arm64 +- Exception hierarchy: `ZerobusException`, `NonRetriableException` +- Support for .NET 8.0 and .NET Standard 2.0 +- Examples for JSON, protobuf, and Arrow ingestion +- `generate-proto` CLI tool for schema generation diff --git a/dotnet/CONTRIBUTING.md b/dotnet/CONTRIBUTING.md new file mode 100644 index 00000000..69c78eb6 --- /dev/null +++ b/dotnet/CONTRIBUTING.md @@ -0,0 +1,123 @@ +# Contributing to the Zerobus .NET SDK + +See the top-level [CONTRIBUTING.md](https://github.com/databricks/zerobus-sdk/blob/main/CONTRIBUTING.md) for general contribution guidelines, pull request process, and commit requirements. This document covers .NET-specific development setup and workflow. + +## Development Setup + +### Prerequisites + +- [Git](https://git-scm.com/downloads) +- [.NET 8.0 SDK or later](https://dotnet.microsoft.com/download/dotnet/8.0) + +### Setting Up Your Development Environment + +1. **Clone the repository** + + ```bash + git clone https://github.com/databricks/zerobus-sdk.git + cd zerobus-sdk/dotnet + ``` + +2. **Build** + + ```bash + dotnet restore --configfile NuGet.Config + dotnet build + ``` + +3. **Run tests** + + ```bash + dotnet test + ``` + +> **Note:** The `NuGet.Config` file in `dotnet/` clears external package sources and uses only nuget.org. If you have corporate feeds configured in a parent `NuGet.Config`, adjust accordingly. + +## Coding Style + +Style is enforced by EditorConfig (`.editorconfig`). Use `dotnet format` for code formatting. + +### Running the Formatter + +```bash +dotnet format +``` + +This formats: +- C# code (whitespace, style, and analyzers) +- Imports organization and unused import removal +- XML doc comment formatting + +### Checking Formatting + +```bash +dotnet format --verify-no-changes +``` + +### Running Tests + +```bash +dotnet test # All tests +dotnet test --filter "FullyQualifiedName~StreamBuilderTests" # Specific test class +dotnet test --filter "FullyQualifiedName~Default_ReturnsOptions" # Specific test method +``` + +## Continuous Integration + +All pull requests must pass CI checks: + +- **fmt** — `dotnet format --verify-no-changes` +- **test** — `dotnet build` + `dotnet test` across Ubuntu, Windows, and macOS runners + +Check the GitHub Actions tab of the pull request for results. + +## Build Commands + +| Command | Description | +|---|---| +| `dotnet clean` | Clean build outputs | +| `dotnet build` | Compile code | +| `dotnet test` | Run tests | +| `dotnet format` | Format code | +| `dotnet format --verify-no-changes` | Check formatting | +| `dotnet pack src/Databricks.Zerobus -c Release` | Create NuGet package | +| `dotnet restore --configfile NuGet.Config` | Restore with clean package sources | + +## Project Structure + +``` +dotnet/ +├── src/Databricks.Zerobus/ # Main library +│ ├── Native/ # P/Invoke declarations and interop +│ │ ├── NativeMethods.cs # [DllImport] extern methods +│ │ ├── NativeLibraryResolver.cs # Cross-platform lib loading +│ │ ├── InteropStructs.cs # C struct marshaling +│ │ └── SafeHandles/ # SafeHandle subclasses +│ ├── ZerobusSdk.cs # Main SDK class +│ ├── StreamBuilder.cs # Fluent builder API +│ ├── BaseZerobusStream.cs # Abstract stream base class +│ ├── ZerobusProtoStream.cs # Protobuf ingestion stream +│ ├── ZerobusJsonStream.cs # JSON ingestion stream +│ ├── ZerobusArrowStream.cs # Arrow Flight stream (Beta) + +│ ├── StreamConfigurationOptions.cs # gRPC stream config +│ ├── ArrowStreamConfigurationOptions.cs # Arrow stream config +│ ├── ProtoSchema.cs # Unity Catalog → proto schema +│ ├── ZerobusException.cs # Base exception type +│ ├── NonRetriableException.cs +│ ├── AckCallback.cs # Delegate types +│ ├── EncodedBatch.cs # Arrow batch wrapper +│ ├── HeadersProvider.cs # Custom auth headers delegate +│ ├── IPCCompressionType.cs # Compression enum +│ └── TableProperties.cs # Table metadata +├── tests/Databricks.Zerobus.Tests/ # xUnit test suite +├── examples/ # Runnable examples +│ ├── JsonIngestion/ # JSON single + batch +│ ├── ProtoIngestion/ # Protobuf single + batch +│ └── ArrowIngestion/ # Arrow Flight (Beta) +├── tools/GenerateProto/ # Schema generation CLI +├── .github/workflows/ # CI/CD +│ ├── ci-dotnet.yml # Push/PR: build, test, pack +│ └── release-dotnet.yml # Manual release to NuGet.org +└── Zerobus-DotNet.sln # Solution file +``` diff --git a/dotnet/Directory.Build.props b/dotnet/Directory.Build.props new file mode 100644 index 00000000..947693e3 --- /dev/null +++ b/dotnet/Directory.Build.props @@ -0,0 +1,10 @@ + + + enable + enable + 12 + true + true + CS1591 + + diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props new file mode 100644 index 00000000..ff73e97d --- /dev/null +++ b/dotnet/Directory.Packages.props @@ -0,0 +1,24 @@ + + + + false + + + + + + + + + + + + + + + + + + diff --git a/dotnet/LICENSE b/dotnet/LICENSE new file mode 100644 index 00000000..95325de5 --- /dev/null +++ b/dotnet/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright (2026) Databricks, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/dotnet/Makefile b/dotnet/Makefile new file mode 100644 index 00000000..ab16914f --- /dev/null +++ b/dotnet/Makefile @@ -0,0 +1,68 @@ +.PHONY: all build test clean pack restore fmt check restore-native native help + +# Default target +all: restore build test + +# Restore NuGet packages (use local NuGet.Config to avoid external feeds) +restore: + dotnet restore --configfile NuGet.Config + +# Build all projects +build: + dotnet build + +# Run all tests +test: + dotnet test + +# Run only unit tests (skip integration) +test-unit: + dotnet test --filter "Category!=Integration" + +# Run only integration tests (requires native library) +test-integration: + dotnet test --filter "Category=Integration" + +# Clean build outputs +clean: + dotnet clean + +# Create NuGet package +pack: + dotnet pack src/Databricks.Zerobus -c Release -o artifacts + +# Format code +fmt: + dotnet format + +# Check formatting (CI) +fmt-check: + dotnet format --verify-no-changes + +# Restore + build native (requires Rust toolchain) +native: + ./build_native.sh + +# Cross-compile native for all platforms +native-all: + ./build_native.sh --all + +# Full release build (native + managed + pack) +release: native build test pack + +help: + @echo "Zerobus .NET SDK — Makefile targets" + @echo "" + @echo " all Restore, build, and test (default)" + @echo " restore Restore NuGet packages" + @echo " build Build all projects" + @echo " test Run all tests" + @echo " test-unit Run unit tests only" + @echo " test-integration Run integration tests (requires native lib)" + @echo " clean Clean build outputs" + @echo " pack Create NuGet package" + @echo " fmt Auto-format code" + @echo " fmt-check Check formatting (CI)" + @echo " native Build native FFI for current platform" + @echo " native-all Cross-compile native for all platforms" + @echo " release Full release: native + build + test + pack" diff --git a/dotnet/NEXT_CHANGELOG.md b/dotnet/NEXT_CHANGELOG.md new file mode 100644 index 00000000..f88aedf6 --- /dev/null +++ b/dotnet/NEXT_CHANGELOG.md @@ -0,0 +1,3 @@ +# Next Changelog + + diff --git a/dotnet/NuGet.Config b/dotnet/NuGet.Config new file mode 100644 index 00000000..765346e5 --- /dev/null +++ b/dotnet/NuGet.Config @@ -0,0 +1,7 @@ + + + + + + + diff --git a/dotnet/README.md b/dotnet/README.md new file mode 100644 index 00000000..f6009512 --- /dev/null +++ b/dotnet/README.md @@ -0,0 +1,444 @@ +# Databricks Zerobus Ingest SDK for .NET + +High-throughput streaming ingestion SDK for Databricks Delta tables over gRPC. + +## Features + +- **High throughput** — Native Rust backend with P/Invoke bindings for maximum performance +- **Offset-based API** — Minimal overhead; no per-record object allocation +- **Built-in retry and recovery** — Configurable automatic recovery from stream failures +- **Flexible configuration** — Fine-grained control over batching, timeouts, and callbacks +- **Protocol Buffers support** — Type-safe, compact wire format with schema compilation +- **JSON ingestion** — Schema-free JSON ingestion without protobuf schemas +- **Arrow Flight (Beta)** — Columnar, batched ingestion for analytics workloads +- **OAuth 2.0 authentication** — Service principal support via Databricks OAuth +- **Cross-platform** — Windows (x64), Linux (glibc x64, arm64), macOS (x64, Apple Silicon) + +## Architecture + +``` +┌──────────────────────────────────────────┐ +│ .NET Application │ +│ ZerobusSdk / ZerobusProtoStream / │ +│ ZerobusJsonStream / ZerobusArrowStream │ +├──────────────────────────────────────────┤ +│ BaseZerobusStream (P/Invoke) │ +├──────────────────────────────────────────┤ +│ Native Rust SDK (zerobus_ffi) │ +│ Tokio runtime · gRPC · HTTP/2 · Arrow │ +└──────────────────────────────────────────┘ +``` + +Direct native calls avoid .NET gRPC overhead. The offset-based API eliminates per-record `Task` allocation. + +## Requirements + +**Runtime:** .NET 8.0+ or .NET Standard 2.0 compatible runtime. A Databricks workspace with Zerobus access enabled. + +**Supported Platforms:** + +| Platform | Architecture | Notes | +|---|---|---| +| Windows | x86_64 | | +| Linux | x86_64, aarch64 | glibc 2.26+ (Amazon Linux 2 compatible) | +| macOS | x86_64, Apple Silicon | | + +**Dependencies:** +- `Google.Protobuf` 3.28.2 (for protobuf streams) +- Apache Arrow library (optional — only needed for Arrow Flight ingestion) + +**Build requirements:** .NET 8.0 SDK or later. + +## Quick Start User Guide + +### Prerequisites + +You need: +- A [Databricks workspace](https://databricks.com) URL and workspace ID +- A Delta table created with `USING DELTA` +- A service principal with OAuth client ID, client secret, and SQL-level permissions (`USE CATALOG`, `USE SCHEMA`, `SELECT`, `MODIFY`) on the target table + +### Installation + +**Option 1: NuGet (Recommended)** + +```bash +dotnet add package Databricks.Zerobus.Ingest +``` + +**Option 2: Build from Source** + +```bash +git clone https://github.com/databricks/zerobus-sdk.git +cd zerobus-sdk/dotnet +dotnet restore --configfile NuGet.Config +dotnet build +``` + +Place the native libraries in `src/Databricks.Zerobus/runtimes/{rid}/native/` or set the `ZEROBUS_NATIVE_LIB_PATH` environment variable. + +### Write Client Code + +The idiomatic flow is ingest in a loop, then `Flush()` once at the end. + +```csharp +using Databricks.Zerobus; + +const string serverEndpoint = "https://my-workspace.databricks.com"; +const string unityCatalogUrl = "https://my-workspace.databricks.com/api/2.1/unity-catalog"; +const string tableName = "my_catalog.my_schema.my_table"; + +using var sdk = new ZerobusSdk(serverEndpoint, unityCatalogUrl); + +await using var stream = await sdk.StreamBuilder() + .Table(tableName) + .OAuth("client-id", "client-secret") + .Json() + .BuildAsync(); + +// Ingest records — returns as soon as the record is queued +for (int i = 0; i < 100; i++) +{ + stream.IngestRecord($"{{\"id\": {i}, \"name\": \"item-{i}\"}}"); +} + +// Confirm durability — acks are ordered, so the last offset confirms all +stream.Flush(); + +Console.WriteLine("Successfully ingested 100 records!"); +``` + +### Acknowledgments and Throughput + +`IngestRecord()` returns immediately after queuing; the SDK handles sending and acknowledgment tracking in the background. To confirm durability, call `Flush()`, which returns once everything queued so far is acknowledged. + +The idiomatic pattern is ingest in a loop, then `Flush()` for bounded batches (or periodically for long-running streams). Alternatively, register ack callbacks via `AckCallback(onAck, onError)`. + +Each `IngestRecord()` returns the record's offset. `WaitForOffset(offset)` blocks until that offset is acknowledged. Because acks are ordered, waiting on the last offset confirms the whole run. + +> **Avoid calling `WaitForOffset()` after every record in a tight loop** — that limits throughput to one record per round-trip. + +## Usage Examples + +The `examples/` directory is organized by stream type: `JsonIngestion/`, `ProtoIngestion/`, `ArrowIngestion/` (Beta). + +### Creating Streams (Stream Builder) + +`ZerobusSdk.StreamBuilder()` is the recommended way to create a stream, exposing a single fluent API for all stream types. + +**Proto:** + +```csharp +byte[] descriptorBytes = ProtoSchema.FromUnityCatalogJson(ucTableJson).GetDescriptorBytes(); + +ZerobusProtoStream protoStream = await sdk.StreamBuilder() + .Table("catalog.schema.table") + .Oauth(clientId, clientSecret) + .CompiledProto(descriptorBytes) + .BuildAsync(); +``` + +**JSON:** + +```csharp +ZerobusJsonStream jsonStream = await sdk.StreamBuilder() + .Table("catalog.schema.table") + .Oauth(clientId, clientSecret) + .Json() + .BuildAsync(); +``` + +**Arrow (Beta):** + +```csharp +byte[] schemaIpcBytes = GetArrowSchemaIpcBytes(); // Use Apache.Arrow or any Arrow library + +ZerobusArrowStream arrowStream = await sdk.StreamBuilder() + .Table("catalog.schema.table") + .Oauth(clientId, clientSecret) + .Arrow(schemaIpcBytes) + .IpcCompression(IPCCompressionType.Zstd) + .BuildAsync(); +``` + +Configuration is set directly on the builder (e.g., `.MaxInflightRecords(50000)`, `.Recovery(true)`). Arrow-specific options like `.MaxInflightBatches(...)` and `.IpcCompression(...)` are available after calling `.Arrow(...)`. + +### Protocol Buffers Examples + +Proto is best for production systems — type-safe, compact, fast. + +```csharp +// Generate the proto descriptor from Unity Catalog +using var protoSchema = ProtoSchema.FromUnityCatalogJson(ucTableJson); +byte[] descriptorBytes = protoSchema.GetDescriptorBytes(); + +var stream = await sdk.StreamBuilder() + .Table(tableName) + .Oauth(clientId, clientSecret) + .CompiledProto(descriptorBytes) + .BuildAsync(); + +// Single record +var record = new MyProtoMessage { Id = 1, Name = "test" }; +long offset = stream.IngestRecord(record); + +// Batch +var records = new List { /* ... */ }; +long? lastOffset = stream.IngestRecords(records); +stream.Flush(); +``` + +### JSON Examples + +JSON is best for rapid prototyping and flexible schemas. No protobuf types required. + +```csharp +var stream = await sdk.StreamBuilder() + .Table(tableName) + .Oauth(clientId, clientSecret) + .Json() + .BuildAsync(); + +// Single record +stream.IngestRecord("{\"id\": 1, \"name\": \"test\"}"); + +// Batch +var jsonRecords = new[] { "{\"a\": 1}", "{\"a\": 2}" }; +stream.IngestRecords(jsonRecords); +stream.Flush(); +``` + +### Arrow Flight Examples (Beta) + +Arrow is best for columnar data, wide/numeric schemas, or applications that already produce Apache Arrow RecordBatches. + +> **Beta disclaimer:** Arrow Flight ingestion is in Beta. The API is stabilizing but may still change before reaching GA. + +```csharp +// Serialize schema via Apache.Arrow +byte[] schemaIpcBytes = SerializeSchema(schema); + +var stream = await sdk.StreamBuilder() + .Table(tableName) + .Oauth(clientId, clientSecret) + .Arrow(schemaIpcBytes) + .MaxInflightBatches(10_000) + .IpcCompression(IPCCompressionType.Zstd) + .BuildAsync(); + +long offset = stream.IngestBatch(recordBatchIpcBytes); +stream.WaitForOffset(offset); +``` + +## API Style + +The SDK uses an **offset-based API** — every `IngestRecord()` / `IngestRecords()` returns a `long` offset immediately after queuing. The offset is a lightweight handle you can wait on later via `WaitForOffset()` or `Flush()`. No per-record object allocation. + +```csharp +long lastOffset = 0; +for (int i = 0; i < 1_000_000; i++) +{ + lastOffset = stream.IngestRecord($"{{}}"); +} +stream.WaitForOffset(lastOffset); // Ingest in a loop, then confirm durability once +``` + +## Choose Your Serialization Format + +| Format | Best For | Pros | Cons | +|---|---|---|---| +| Protocol Buffers | Production systems | Type-safe, compact, fast | Requires schema compilation | +| JSON | Prototyping, flexible schemas | Human-readable, no compilation | Larger payload, slower | +| Arrow Flight (Beta) | Columnar/analytics, wide/numeric schemas, Arrow-native apps | High throughput, native Arrow types, optional IPC compression | Extra deps, API may change | + +## Configuration + +### Stream Configuration Options + +| Option | Default | Description | +|---|---|---| +| `MaxInflightRecords` | 1000000 | Max unacknowledged records | +| `Recovery` | true | Enable auto recovery | +| `RecoveryTimeoutMs` | 15000 | Recovery timeout (ms) | +| `RecoveryBackoffMs` | 2000 | Delay between recovery attempts (ms) | +| `RecoveryRetries` | 4 | Max recovery attempts | +| `FlushTimeoutMs` | 300000 | Flush timeout (ms) | +| `ServerLackOfAckTimeoutMs` | 60000 | Server ack timeout (ms) | +| `OnAck` / `OnError` | None | Callbacks for record acknowledgment | + +### Arrow Stream Configuration Options (Beta) + +| Option | Default | Description | +|---|---|---| +| `MaxInflightBatches` | 10000 | Max unacknowledged Arrow batches | +| `Recovery` | true | Enable auto recovery | +| `RecoveryTimeoutMs` | 15000 | Recovery timeout (ms) | +| `RecoveryBackoffMs` | 2000 | Delay between recovery attempts (ms) | +| `RecoveryRetries` | 4 | Max recovery attempts | +| `ServerLackOfAckTimeoutMs` | 60000 | Server ack timeout (ms) | +| `FlushTimeoutMs` | 300000 | Flush timeout (ms) | +| `ConnectionTimeoutMs` | 30000 | gRPC connection timeout (ms) | +| `IpcCompression` | None | IPC compression codec | +| `StreamPausedMaxWaitTimeMs` | 0 | Max wait during paused state on graceful close. Negative = wait full server duration | + +## Logging + +The .NET SDK has minimal managed-side logging. Most detailed logging (token generation, gRPC, retries) is handled internally by the native Rust SDK. Control Rust-side logging via the `RUST_LOG` environment variable: + +```bash +export RUST_LOG=debug +``` + +## Error Handling + +Two exception types: + +- **`ZerobusException`** — Base exception. May be retryable (`IsRetryable = true`) for network issues or temporary server errors. +- **`NonRetriableException`** — Fatal errors (invalid credentials, missing table). No retry. + +```csharp +try +{ + stream.IngestRecord(json); +} +catch (NonRetriableException ex) +{ + // Fatal — don't retry + Console.Error.WriteLine($"Non-retriable error: {ex.Message}"); +} +catch (ZerobusException ex) when (ex.IsRetryable) +{ + // Retry with backoff +} +``` + +## API Reference + +### ZerobusSdk + +**Constructors:** +- `ZerobusSdk(string serverEndpoint, string unityCatalogEndpoint)` +- `ZerobusSdk(string serverEndpoint, string unityCatalogEndpoint, string applicationName)` — applicationName is appended to the HTTP user-agent header. Conventionally `"/"`. + +**Static factories:** +- `ZerobusSdk.CreateBuilder(string serverEndpoint, string unityCatalogEndpoint)` — returns `SdkBuilder` for advanced configuration + +**Methods:** +- `StreamBuilder()` — Returns `StreamBuilder`. The recommended way to create streams. +- `Dispose()` — Closes the SDK and frees native resources. + +### StreamBuilder + +Fluent builder returned by `ZerobusSdk.StreamBuilder()`. Shared methods: + +| Method | Description | +|---|---| +| `.Table(name)` | Full UC table name (catalog.schema.table) | +| `.Oauth(clientId, secret)` | Service principal credentials | +| `.MaxInflightRecords(n)` | Buffer size (default: 1M) | +| `.Recovery(bool)` | Auto-recovery on failure (default: true) | +| `.RecoveryTimeoutMs(n)` | Recovery timeout (default: 15s) | +| `.RecoveryBackoffMs(n)` | Recovery backoff (default: 2s) | +| `.RecoveryRetries(n)` | Max recovery attempts (default: 4) | +| `.ServerLackOfAckTimeoutMs(n)` | Server ack timeout (default: 60s) | +| `.FlushTimeoutMs(n)` | Flush timeout (default: 5 min) | +| `.AckCallback(onAck, onError)` | Acknowledgment callbacks | + +Format selection methods that return typed sub-builders: + +- `.Json()` → `JsonStreamBuilder` with `BuildAsync()` returning `Task` +- `.CompiledProto(byte[])` → `ProtoStreamBuilder` with `BuildAsync()` returning `Task>` +- `.Arrow(byte[])` → `ArrowStreamBuilder` (Beta) with `BuildAsync()` returning `Task`, plus additional methods: `MaxInflightBatches()`, `ConnectionTimeoutMs()`, `IpcCompression()`, `StreamPausedMaxWaitTimeMs()` + +### ZerobusProtoStream\ + +**Single Record:** +- `long IngestRecord(T record)` — ingests protobuf message, returns offset immediately +- `long IngestRecord(byte[] encodedBytes)` — ingests pre-encoded bytes + +**Batch:** +- `long? IngestRecords(IEnumerable records)` +- `long? IngestRecords(IReadOnlyList encodedRecords)` + +**Recovery:** +- `IReadOnlyList GetUnackedRecords()` +- `IReadOnlyList GetUnackedRecords(MessageParser parser)` + +**Lifecycle:** `WaitForOffset()`, `Flush()`, `Close()`, `IsClosed`, `Dispose()`, `TableName`, `Options`, `ClientId`, `ClientSecret` + +### ZerobusJsonStream + +**Single Record:** +- `long IngestRecord(string json)` + +**Batch:** +- `long? IngestRecords(IReadOnlyList jsonRecords)` +- `long? IngestRecords(IEnumerable jsonRecords)` + +**Recovery:** +- `IReadOnlyList GetUnackedRecords()` +- `IReadOnlyList GetUnackedRecordBytes()` + +**Lifecycle:** Same as ZerobusProtoStream. + +### ZerobusArrowStream (Beta) + +- `long IngestBatch(byte[] ipcBytes)` — serializes to Arrow IPC, queues it, returns offset +- `IReadOnlyList GetUnackedBatches()` +- Same lifecycle methods: `WaitForOffset()`, `Flush()`, `Close()`, `IsClosed`, `Dispose()` + +### StreamConfigurationOptions + +Built via `StreamConfigurationOptions.NewBuilder()`. Chainable setters: `SetMaxInflightRecords`, `SetRecovery`, `SetRecoveryTimeoutMs`, `SetRecoveryBackoffMs`, `SetRecoveryRetries`, `SetFlushTimeoutMs`, `SetServerLackOfAckTimeoutMs`, `SetAckCallback(onAck, onError)`, `SetStreamPausedMaxWaitTimeMs`, `SetCallbackMaxWaitTimeMs`. Static `Default` and `NewBuilder()` methods. + +### ArrowStreamConfigurationOptions (Beta) + +Built via `ArrowStreamConfigurationOptions.NewBuilder()`. Chainable setters: `SetMaxInflightBatches`, `SetRecovery`, `SetRecoveryTimeoutMs`, `SetRecoveryBackoffMs`, `SetRecoveryRetries`, `SetServerLackOfAckTimeoutMs`, `SetFlushTimeoutMs`, `SetConnectionTimeoutMs`, `SetIpcCompression`, `SetStreamPausedMaxWaitTimeMs`. Static `Default` and `NewBuilder()` methods. + +### IPCCompressionType + +Three values: `None` (default, -1), `Lz4Frame` (fast, modest ratio, 0), `Zstd` (higher ratio at higher CPU cost, 1). Enable compression only when network bandwidth limits throughput. + +### ZerobusException + +Base exception. Constructors: `(string message)`, `(string message, bool isRetryable)`, `(string message, bool isRetryable, Exception? innerException)`. Property: `bool IsRetryable`. + +### NonRetriableException + +Extends `ZerobusException`. Always `IsRetryable = false`. + +### AckOnAckDelegate / AckOnErrorDelegate + +```csharp +delegate void AckOnAckDelegate(long offsetId); +delegate void AckOnErrorDelegate(long offsetId, string errorMessage); +``` + +Observe acknowledgments as they arrive on a background thread while you keep ingesting. Callbacks must be thread-safe and lightweight. + +## Best Practices + +1. **Reuse one `ZerobusSdk` per application** +2. **Always close streams** in `using` statements or `finally` blocks +3. **Use offset-based API for high throughput** — avoids `Task` overhead +4. **Ingest in a loop then flush** — confirm durability once. Per-record waits only when a specific record must be confirmed before continuing +5. **Batch records** with `IngestRecords()` +6. **Configure `maxInflightRecords`** based on throughput/memory needs +7. **Distinguish retryable vs non-retryable** errors via `ZerobusException.IsRetryable` +8. **Use `AckCallback`** for non-blocking durability monitoring +9. **Use `ProtoSchema.FromUnityCatalogJson()`** to generate proto descriptors from table schemas +10. **Choose the right API**: `IngestRecord()` + final `Flush()` for high throughput; `IngestRecord()` + per-record `WaitForOffset()` for specific record confirmation +11. **Recovery pattern**: use `sdk.RecreateStreamAsync(closedStream)` for automatic re-ingestion of unacknowledged records, or manually use `GetUnackedRecords()` after close + +## Community and Contributing + +- [Contributing Guide](CONTRIBUTING.md) — .NET-specific development setup and workflow +- [General Contributing Guide](https://github.com/databricks/zerobus-sdk/blob/main/CONTRIBUTING.md) — PR process, commit requirements, policies +- [Changelog](CHANGELOG.md) +- [Security Policy](https://github.com/databricks/zerobus-sdk/blob/main/SECURITY.md) +- Developer Certificate of Origin (DCO) — all commits must be signed off (`git commit -s`) +- Open Source Attributions — NOTICE file + +## License + +Apache License 2.0 — see [LICENSE](LICENSE). diff --git a/dotnet/Zerobus-DotNet.slnx b/dotnet/Zerobus-DotNet.slnx new file mode 100644 index 00000000..c8d8bd5f --- /dev/null +++ b/dotnet/Zerobus-DotNet.slnx @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/dotnet/build_native.sh b/dotnet/build_native.sh new file mode 100644 index 00000000..bba01006 --- /dev/null +++ b/dotnet/build_native.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# build_native.sh — Compile the Zerobus Rust FFI library and copy it to the +# .NET runtimes directory for local development. +# +# Usage: +# ./build_native.sh # Build for current platform only +# ./build_native.sh --all # Cross-compile for all supported platforms +# ./build_native.sh --release # Build in release mode (default) +# ./build_native.sh --debug # Build in debug mode +# +# Prerequisites: +# - Rust toolchain (rustup, cargo) +# - For cross-compilation: zig (via setup-zig or cargo-zigbuild) +# - This script must be run from the monorepo root (zerobus-sdk/) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +MONOREPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +FFI_CRATE="$MONOREPO_ROOT/rust/ffi" +RUNTIMES_DIR="$SCRIPT_DIR/src/Databricks.Zerobus/runtimes" + +BUILD_MODE="--release" +TARGET="" +RUST_DIR="$MONOREPO_ROOT/rust" + +# ─── Parse args ──────────────────────────────────────────────────── +ALL_PLATFORMS=false +while [[ $# -gt 0 ]]; do + case "$1" in + --all) ALL_PLATFORMS=true; shift ;; + --release) BUILD_MODE="--release"; shift ;; + --debug) BUILD_MODE=""; shift ;; + *) echo "Unknown option: $1"; exit 1 ;; + esac +done + +# ─── Detect platform ─────────────────────────────────────────────── +detect_target() { + case "$(uname -s)" in + Linux) + case "$(uname -m)" in + x86_64) echo "x86_64-unknown-linux-gnu" ;; + aarch64) echo "aarch64-unknown-linux-gnu" ;; + esac ;; + Darwin) + case "$(uname -m)" in + x86_64) echo "x86_64-apple-darwin" ;; + arm64) echo "aarch64-apple-darwin" ;; + esac ;; + MINGW*|MSYS*|CYGWIN*) + echo "x86_64-pc-windows-msvc" ;; + esac +} + +# ─── Map Rust target → NuGet RID + native file name ───────────────── +map_artifact() { + case "$1" in + x86_64-unknown-linux-gnu) echo "linux-x64:libzerobus_ffi.so" ;; + aarch64-unknown-linux-gnu) echo "linux-arm64:libzerobus_ffi.so" ;; + x86_64-apple-darwin) echo "osx-x64:libzerobus_ffi.dylib" ;; + aarch64-apple-darwin) echo "osx-arm64:libzerobus_ffi.dylib" ;; + x86_64-pc-windows-msvc) echo "win-x64:zerobus_ffi.dll" ;; + esac +} + +ALL_TARGETS=( + "x86_64-unknown-linux-gnu" + "aarch64-unknown-linux-gnu" + "x86_64-apple-darwin" + "aarch64-apple-darwin" + "x86_64-pc-windows-msvc" +) + +# ─── Build single target ─────────────────────────────────────────── +build_target() { + local target="$1" + local mapping + mapping="$(map_artifact "$target")" + IFS=':' read -r rid libname <<< "$mapping" + + echo " → Building for $target ($rid)..." + + cd "$RUST_DIR" + cargo build $BUILD_MODE -p zerobus-ffi --target "$target" + + local src + src="target/$target/release/$libname" + if [[ -z "$BUILD_MODE" ]]; then + src="target/$target/debug/$libname" + fi + + local dest="$RUNTIMES_DIR/$rid/native/$libname" + mkdir -p "$(dirname "$dest")" + cp -v "$src" "$dest" + echo " ✔ $rid done" +} + +# ─── Main ────────────────────────────────────────────────────────── +echo "==> Building Zerobus FFI native libraries" +echo " FFI crate: $FFI_CRATE" +echo " Runtimes: $RUNTIMES_DIR" +echo "" + +if $ALL_PLATFORMS; then + echo "==> Cross-compiling for all $(( ${#ALL_TARGETS[@]} )) platforms..." + for target in "${ALL_TARGETS[@]}"; do + build_target "$target" + done +else + TARGET="$(detect_target)" + if [[ -z "$TARGET" ]]; then + echo "ERROR: Could not detect current platform. Use --all to cross-compile." + exit 1 + fi + echo "==> Building for current platform: $TARGET" + build_target "$TARGET" +fi + +echo "" +echo "==> Done. Native libraries placed in:" +find "$RUNTIMES_DIR" -type f -exec ls -lh {} \; diff --git a/dotnet/examples/ArrowIngestion/ArrowIngestion.csproj b/dotnet/examples/ArrowIngestion/ArrowIngestion.csproj new file mode 100644 index 00000000..357f2555 --- /dev/null +++ b/dotnet/examples/ArrowIngestion/ArrowIngestion.csproj @@ -0,0 +1,15 @@ + + + + Exe + net8.0 + Databricks.Zerobus.Examples.ArrowIngestion + false + + + + + + + + diff --git a/dotnet/examples/ArrowIngestion/ArrowIngestionExample.cs b/dotnet/examples/ArrowIngestion/ArrowIngestionExample.cs new file mode 100644 index 00000000..1fc5f997 --- /dev/null +++ b/dotnet/examples/ArrowIngestion/ArrowIngestionExample.cs @@ -0,0 +1,55 @@ +using Databricks.Zerobus; + +// =================================================================== +// Arrow Flight Ingestion Example (Beta) +// =================================================================== +// This example demonstrates ingesting Apache Arrow RecordBatches +// using the Arrow Flight protocol via the Zerobus SDK. +// +// Since the SDK accepts raw Arrow IPC bytes, you can use any +// Arrow library (Apache.Arrow, ADBC, etc.) to produce them. +// =================================================================== + +const string serverEndpoint = "https://YOUR_WORKSPACE.databricks.com"; +const string unityCatalogUrl = "https://YOUR_WORKSPACE.databricks.com/api/2.1/unity-catalog"; + +Console.WriteLine("Zerobus Arrow Flight Ingestion Example (Beta)"); +Console.WriteLine("============================================="); + +// Step 1: Serialize your Arrow schema to IPC format bytes +// Using Apache.Arrow: +// +// var schema = new Schema.Builder() +// .Field(f => f.Name("id").DataType(Int64Type.Default).Nullable(false)) +// .Field(f => f.Name("name").DataType(StringType.Default).Nullable(true)) +// .Build(); +// +// byte[] schemaIpcBytes; +// using (var ms = new MemoryStream()) +// { +// var writer = new Apache.Arrow.Ipc.ArrowStreamWriter(ms, schema); +// writer.WriteStart(); +// writer.WriteEnd(); +// schemaIpcBytes = ms.ToArray(); +// } + +byte[] schemaIpcBytes = Array.Empty(); // Replace with real schema IPC bytes + +using var sdk = new ZerobusSdk(serverEndpoint, unityCatalogUrl); + +// await using var stream = await sdk.StreamBuilder() +// .Table(tableName) +// .OAuth(clientId, clientSecret) +// .Arrow(schemaIpcBytes) +// .MaxInflightBatches(10_000) +// .IpcCompression(IPCCompressionType.Zstd) +// .BuildAsync(); +// +// // Serialize Arrow RecordBatch to IPC bytes +// byte[] batchIpcBytes = SerializeRecordBatch(recordBatch); +// +// long offset = stream.IngestBatch(batchIpcBytes); +// stream.WaitForOffset(offset); + +Console.WriteLine("See code comments for Arrow ingestion setup."); +Console.WriteLine("Done."); diff --git a/dotnet/examples/JsonIngestion/BatchIngestionExample.cs b/dotnet/examples/JsonIngestion/BatchIngestionExample.cs new file mode 100644 index 00000000..cd1859eb --- /dev/null +++ b/dotnet/examples/JsonIngestion/BatchIngestionExample.cs @@ -0,0 +1,46 @@ +using Databricks.Zerobus; + +namespace Databricks.Zerobus.Examples.JsonIngestion; + +public static class BatchIngestionExample +{ + public static async Task Run() + // =================================================================== + // Batch JSON Ingestion Example + // =================================================================== + { + const string serverEndpoint = "https://YOUR_WORKSPACE.databricks.com"; + const string unityCatalogUrl = "https://YOUR_WORKSPACE.databricks.com/api/2.1/unity-catalog"; + const string tableName = "my_catalog.my_schema.my_table"; + const string clientId = "YOUR_SERVICE_PRINCIPAL_CLIENT_ID"; + const string clientSecret = "YOUR_SERVICE_PRINCIPAL_CLIENT_SECRET"; + + Console.WriteLine("Zerobus JSON Batch Ingestion Example"); + Console.WriteLine("====================================="); + + using var sdk = new ZerobusSdk(serverEndpoint, unityCatalogUrl); + + using var stream = await sdk.StreamBuilder() + .Table(tableName) + .OAuth(clientId, clientSecret) + .MaxInflightRecords(500_000) + .Recovery(true) + .Json() + .BuildAsync(); + + Console.WriteLine($"Stream created for table: {stream.TableName}"); + + var records = new List(); + for (int i = 1; i <= 1000; i++) + { + records.Add($"{{\"id\": {i}, \"name\": \"Batch Item {i}\", \"value\": {i * 1.5}}}"); + } + + var lastOffset = stream.IngestRecords(records); + Console.WriteLine($"Ingested {records.Count} records. Last offset: {lastOffset}"); + + stream.Flush(); + Console.WriteLine("All records flushed and durably acknowledged."); + Console.WriteLine("Done."); + } +} diff --git a/dotnet/examples/JsonIngestion/JsonIngestion.csproj b/dotnet/examples/JsonIngestion/JsonIngestion.csproj new file mode 100644 index 00000000..2d955bbc --- /dev/null +++ b/dotnet/examples/JsonIngestion/JsonIngestion.csproj @@ -0,0 +1,14 @@ + + + + Exe + net8.0 + Databricks.Zerobus.Examples.JsonIngestion + false + + + + + + + diff --git a/dotnet/examples/JsonIngestion/SingleRecordExample.cs b/dotnet/examples/JsonIngestion/SingleRecordExample.cs new file mode 100644 index 00000000..86b373ca --- /dev/null +++ b/dotnet/examples/JsonIngestion/SingleRecordExample.cs @@ -0,0 +1,51 @@ +using Databricks.Zerobus; + +// =================================================================== +// Single Record JSON Ingestion Example +// =================================================================== +// This example demonstrates ingesting JSON records one at a time +// into a Databricks Delta table using the Zerobus SDK. +// +// Prerequisites: +// - A Databricks workspace with Unity Catalog +// - A Delta table created with USING DELTA +// - A service principal with OAuth credentials and table permissions +// - The Zerobus native library (zerobus_ffi.dll/.so/.dylib) +// available in your runtimes folder or system path +// =================================================================== + +const string serverEndpoint = "https://YOUR_WORKSPACE.databricks.com"; +const string unityCatalogUrl = "https://YOUR_WORKSPACE.databricks.com/api/2.1/unity-catalog"; +const string tableName = "my_catalog.my_schema.my_table"; +const string clientId = "YOUR_SERVICE_PRINCIPAL_CLIENT_ID"; +const string clientSecret = "YOUR_SERVICE_PRINCIPAL_CLIENT_SECRET"; + +Console.WriteLine("Zerobus JSON Single Record Ingestion Example"); +Console.WriteLine("============================================="); + +using var sdk = new ZerobusSdk(serverEndpoint, unityCatalogUrl); + +using var stream = await sdk.StreamBuilder() + .Table(tableName) + .OAuth(clientId, clientSecret) + .MaxInflightRecords(100_000) + .Json() + .BuildAsync(); + +Console.WriteLine($"Stream created for table: {stream.TableName}"); + +// Ingest single records +var offset1 = stream.IngestRecord("{\"id\": 1, \"name\": \"Product A\", \"price\": 29.99}"); +Console.WriteLine($"Ingested record at offset: {offset1}"); + +var offset2 = stream.IngestRecord("{\"id\": 2, \"name\": \"Product B\", \"price\": 49.99}"); +Console.WriteLine($"Ingested record at offset: {offset2}"); + +// Wait for the last offset to ensure all records are durably stored +stream.WaitForOffset(offset2); +Console.WriteLine("All records durably acknowledged."); + +// Or, flush all pending records +// stream.Flush(); + +Console.WriteLine("Done."); diff --git a/dotnet/examples/ProtoIngestion/BatchIngestionExample.cs b/dotnet/examples/ProtoIngestion/BatchIngestionExample.cs new file mode 100644 index 00000000..8191d512 --- /dev/null +++ b/dotnet/examples/ProtoIngestion/BatchIngestionExample.cs @@ -0,0 +1,44 @@ +using Databricks.Zerobus; + +namespace Databricks.Zerobus.Examples.ProtoIngestion; + +public static class BatchIngestionExample +{ + public static void Run() + // =================================================================== + // Batch Protocol Buffers Ingestion Example + // =================================================================== + { + Console.WriteLine("Zerobus Protocol Buffers Batch Ingestion Example"); + Console.WriteLine("================================================"); + + // Replace with real values: + // const string serverEndpoint = "https://YOUR_WORKSPACE.databricks.com"; + // const string unityCatalogUrl = "https://YOUR_WORKSPACE.databricks.com/api/2.1/unity-catalog"; + // const string tableName = "my_catalog.my_schema.my_table"; + // const string clientId = "YOUR_SERVICE_PRINCIPAL_CLIENT_ID"; + // const string clientSecret = "YOUR_SERVICE_PRINCIPAL_CLIENT_SECRET"; + + // byte[] descriptorBytes = Array.Empty(); + // using var sdk = new ZerobusSdk(serverEndpoint, unityCatalogUrl); + // + // await using var stream = await sdk.StreamBuilder() + // .Table(tableName) + // .OAuth(clientId, clientSecret) + // .CompiledProto(descriptorBytes) + // .BuildAsync(); + // + // var records = new List(); + // for (int i = 0; i < 1000; i++) + // { + // var msg = new MyProtoMessage { Id = i, Name = $"Item {i}" }; + // records.Add(msg.ToByteArray()); + // } + // + // var lastOffset = stream.IngestRecords(records); + // stream.Flush(); + + Console.WriteLine("See code comments for protobuf batch ingestion setup."); + Console.WriteLine("Done."); + } +} diff --git a/dotnet/examples/ProtoIngestion/ProtoIngestion.csproj b/dotnet/examples/ProtoIngestion/ProtoIngestion.csproj new file mode 100644 index 00000000..4a4771a2 --- /dev/null +++ b/dotnet/examples/ProtoIngestion/ProtoIngestion.csproj @@ -0,0 +1,15 @@ + + + + Exe + net8.0 + Databricks.Zerobus.Examples.ProtoIngestion + false + + + + + + + + diff --git a/dotnet/examples/ProtoIngestion/SingleRecordExample.cs b/dotnet/examples/ProtoIngestion/SingleRecordExample.cs new file mode 100644 index 00000000..05415781 --- /dev/null +++ b/dotnet/examples/ProtoIngestion/SingleRecordExample.cs @@ -0,0 +1,46 @@ +using Databricks.Zerobus; +using Google.Protobuf; + +// =================================================================== +// Single Record Protocol Buffers Ingestion Example +// =================================================================== +// This example demonstrates ingesting protobuf records using the +// Zerobus SDK. You need: +// 1. A compiled .proto schema generated from your Unity Catalog table +// (use the generate-proto tool or ProtoSchema.FromUnityCatalogJson) +// 2. The compiled descriptor proto bytes +// 3. Your protobuf message type (here using a manual example) +// =================================================================== + +const string serverEndpoint = "https://YOUR_WORKSPACE.databricks.com"; +const string unityCatalogUrl = "https://YOUR_WORKSPACE.databricks.com/api/2.1/unity-catalog"; + +Console.WriteLine("Zerobus Protocol Buffers Single Record Ingestion Example"); +Console.WriteLine("======================================================="); + +// Step 1: Generate proto schema from Unity Catalog +// This would normally be done via the Unity Catalog API response JSON +// string ucTableJson = await FetchTableFromUnityCatalog(tableName); +// var protoSchema = ProtoSchema.FromUnityCatalogJson(ucTableJson); +// byte[] descriptorBytes = protoSchema.GetDescriptorBytes(); +// protoSchema.Dispose(); + +// For this example, use pre-existing descriptor bytes +byte[] descriptorBytes = Array.Empty(); // Replace with real descriptor bytes + +using var sdk = new ZerobusSdk(serverEndpoint, unityCatalogUrl); + +// Step 2: Create a protobuf stream. The generic type T must be your protobuf message type. +// Replace 'MyProtoMessage' with your actual generated message class. +// +// await using var stream = await sdk.StreamBuilder() +// .Table(tableName) +// .OAuth(clientId, clientSecret) +// .CompiledProto(descriptorBytes) +// .BuildAsync(); +// +// var record = new MyProtoMessage { Id = 1, Name = "Test", Price = 29.99 }; +// var offset = stream.IngestRecord(record); + +Console.WriteLine("See code comments for protobuf ingestion setup."); +Console.WriteLine("Done."); diff --git a/dotnet/src/Databricks.Zerobus/AckCallback.cs b/dotnet/src/Databricks.Zerobus/AckCallback.cs new file mode 100644 index 00000000..6b71091c --- /dev/null +++ b/dotnet/src/Databricks.Zerobus/AckCallback.cs @@ -0,0 +1,17 @@ +namespace Databricks.Zerobus; + +/// +/// Callback delegate for successful acknowledgment of records up to the given offset. +/// +/// +/// The offset ID that has been durably acknowledged by the server. +/// All records with offset IDs less than or equal to this value have been durably stored. +/// +public delegate void AckOnAckDelegate(long offsetId); + +/// +/// Callback delegate for errors affecting a specific offset. +/// +/// The offset ID that encountered an error. +/// A description of the error that occurred. +public delegate void AckOnErrorDelegate(long offsetId, string errorMessage); diff --git a/dotnet/src/Databricks.Zerobus/ArrowStreamConfigurationOptions.cs b/dotnet/src/Databricks.Zerobus/ArrowStreamConfigurationOptions.cs new file mode 100644 index 00000000..e2e72305 --- /dev/null +++ b/dotnet/src/Databricks.Zerobus/ArrowStreamConfigurationOptions.cs @@ -0,0 +1,204 @@ +namespace Databricks.Zerobus; + +/// +/// Configuration options for Zerobus Arrow Flight streams (Beta). +/// Extends gRPC stream options with Arrow-specific settings. +/// +public sealed class ArrowStreamConfigurationOptions +{ + /// + /// Default: 10,000 batches. + /// + public const int DefaultMaxInflightBatches = 10_000; + + /// + /// Default: 30,000 ms (30 seconds). + /// + public const int DefaultConnectionTimeoutMs = 30_000; + + /// + /// Maximum number of Arrow batches that can be in flight. + /// + public int MaxInflightBatches { get; } + + /// + /// Whether automatic recovery is enabled. Default: true. + /// + public bool Recovery { get; } + + /// + /// Recovery timeout in milliseconds. + /// + public int RecoveryTimeoutMs { get; } + + /// + /// Recovery backoff delay in milliseconds. + /// + public int RecoveryBackoffMs { get; } + + /// + /// Maximum number of recovery attempts. + /// + public int RecoveryRetries { get; } + + /// + /// Server lack-of-ack timeout in milliseconds. + /// + public int ServerLackOfAckTimeoutMs { get; } + + /// + /// Flush timeout in milliseconds. + /// + public int FlushTimeoutMs { get; } + + /// + /// Connection timeout in milliseconds. + /// + public int ConnectionTimeoutMs { get; } + + /// + /// IPC compression type. Default: None. + /// + public IPCCompressionType IpcCompression { get; } + + /// + /// Maximum wait time when the stream is paused, in milliseconds. + /// A negative value means "wait the full server-specified duration." + /// + public long StreamPausedMaxWaitTimeMs { get; } + + private ArrowStreamConfigurationOptions(Builder builder) + { + MaxInflightBatches = builder.MaxInflightBatches; + Recovery = builder.Recovery; + RecoveryTimeoutMs = builder.RecoveryTimeoutMs; + RecoveryBackoffMs = builder.RecoveryBackoffMs; + RecoveryRetries = builder.RecoveryRetries; + ServerLackOfAckTimeoutMs = builder.ServerLackOfAckTimeoutMs; + FlushTimeoutMs = builder.FlushTimeoutMs; + ConnectionTimeoutMs = builder.ConnectionTimeoutMs; + IpcCompression = builder.IpcCompression; + StreamPausedMaxWaitTimeMs = builder.StreamPausedMaxWaitTimeMs; + } + + /// + /// Returns the default Arrow stream configuration options. + /// + public static ArrowStreamConfigurationOptions Default => new(new Builder()); + + /// + /// Creates a new builder for ArrowStreamConfigurationOptions. + /// + public static Builder NewBuilder() => new(); + + /// + /// Builder for creating instances. + /// + public sealed class Builder + { + internal int MaxInflightBatches { get; set; } = DefaultMaxInflightBatches; + internal bool Recovery { get; set; } = StreamConfigurationOptions.DefaultRecovery; + internal int RecoveryTimeoutMs { get; set; } = StreamConfigurationOptions.DefaultRecoveryTimeoutMs; + internal int RecoveryBackoffMs { get; set; } = StreamConfigurationOptions.DefaultRecoveryBackoffMs; + internal int RecoveryRetries { get; set; } = StreamConfigurationOptions.DefaultRecoveryRetries; + internal int ServerLackOfAckTimeoutMs { get; set; } = StreamConfigurationOptions.DefaultServerLackOfAckTimeoutMs; + internal int FlushTimeoutMs { get; set; } = StreamConfigurationOptions.DefaultFlushTimeoutMs; + internal int ConnectionTimeoutMs { get; set; } = DefaultConnectionTimeoutMs; + internal IPCCompressionType IpcCompression { get; set; } = IPCCompressionType.None; + internal long StreamPausedMaxWaitTimeMs { get; set; } + + internal Builder() { } + + /// + /// Sets the maximum number of in-flight Arrow batches. Must be positive. + /// + public Builder SetMaxInflightBatches(int maxInflightBatches) + { + if (maxInflightBatches <= 0) + throw new ArgumentException("maxInflightBatches must be positive", nameof(maxInflightBatches)); + MaxInflightBatches = maxInflightBatches; + return this; + } + + /// + /// Enables or disables automatic recovery. + /// + public Builder SetRecovery(bool recovery) { Recovery = recovery; return this; } + + /// + /// Sets the recovery timeout in milliseconds. Must be non-negative. + /// + public Builder SetRecoveryTimeoutMs(int ms) + { + if (ms < 0) throw new ArgumentException("Must be non-negative", nameof(ms)); + RecoveryTimeoutMs = ms; return this; + } + + /// + /// Sets the recovery backoff in milliseconds. Must be non-negative. + /// + public Builder SetRecoveryBackoffMs(int ms) + { + if (ms < 0) throw new ArgumentException("Must be non-negative", nameof(ms)); + RecoveryBackoffMs = ms; return this; + } + + /// + /// Sets the maximum number of recovery retries. Must be non-negative. + /// + public Builder SetRecoveryRetries(int retries) + { + if (retries < 0) throw new ArgumentException("Must be non-negative", nameof(retries)); + RecoveryRetries = retries; return this; + } + + /// + /// Sets the server lack-of-ack timeout in milliseconds. Must be positive. + /// + public Builder SetServerLackOfAckTimeoutMs(int ms) + { + if (ms <= 0) throw new ArgumentException("Must be positive", nameof(ms)); + ServerLackOfAckTimeoutMs = ms; return this; + } + + /// + /// Sets the flush timeout in milliseconds. Must be positive. + /// + public Builder SetFlushTimeoutMs(int ms) + { + if (ms <= 0) throw new ArgumentException("Must be positive", nameof(ms)); + FlushTimeoutMs = ms; return this; + } + + /// + /// Sets the connection timeout in milliseconds. Must be positive. + /// + public Builder SetConnectionTimeoutMs(int ms) + { + if (ms <= 0) throw new ArgumentException("Must be positive", nameof(ms)); + ConnectionTimeoutMs = ms; return this; + } + + /// + /// Sets the IPC compression type. + /// + public Builder SetIpcCompression(IPCCompressionType compression) + { + IpcCompression = compression; return this; + } + + /// + /// Sets the maximum wait time when the stream is paused. + /// A negative value means "wait the full server-specified duration." + /// + public Builder SetStreamPausedMaxWaitTimeMs(long ms) + { + StreamPausedMaxWaitTimeMs = ms; return this; + } + + /// + /// Builds the instance. + /// + public ArrowStreamConfigurationOptions Build() => new(this); + } +} diff --git a/dotnet/src/Databricks.Zerobus/BaseZerobusStream.cs b/dotnet/src/Databricks.Zerobus/BaseZerobusStream.cs new file mode 100644 index 00000000..e82d87e8 --- /dev/null +++ b/dotnet/src/Databricks.Zerobus/BaseZerobusStream.cs @@ -0,0 +1,294 @@ +using Databricks.Zerobus.Native; +using System.Runtime.InteropServices; + +namespace Databricks.Zerobus; + +/// +/// Base class for Zerobus ingestion streams providing common native method +/// declarations, lifecycle management, and offset-based acknowledgment tracking. +/// +/// +/// Streams are thread-safe. Multiple threads can ingest records concurrently; +/// the SDK serializes access to the underlying native handle via a reader-writer lock. +/// Streams implement IDisposable and (on .NET 8+) IAsyncDisposable. +/// Always use using or await using statements to ensure native resources are freed. +/// +public abstract class BaseZerobusStream : IDisposable +#if NET8_0_OR_GREATER + , IAsyncDisposable +#endif + +{ + private readonly ReaderWriterLockSlim _lock = new(LockRecursionPolicy.NoRecursion); + private ZerobusStreamHandle _handle; + internal int _disposed; + + /// + /// The fully qualified Unity Catalog table name. + /// + public string TableName { get; } + + /// + /// The stream configuration options. + /// + public StreamConfigurationOptions Options { get; } + + /// + /// Whether this is a JSON-mode stream (as opposed to protobuf mode). + /// + protected bool IsJsonMode { get; } + + /// + /// Cached unacknowledged records available after stream closure. + /// + protected List? CachedUnackedRecords { get; private set; } + + /// + /// Whether the stream has been closed or disposed. + /// + public bool IsClosed => _disposed != 0 || _handle.IsClosed || _handle.IsInvalid; + + internal IntPtr NativeHandle => _handle.DangerousGetHandle(); + + /// + /// Creates a new base stream with the given native handle and configuration. + /// + internal protected BaseZerobusStream( + IntPtr nativeHandle, + string tableName, + StreamConfigurationOptions options, + bool isJsonMode) + { + _handle = new ZerobusStreamHandle(nativeHandle); + TableName = tableName ?? throw new ArgumentNullException(nameof(tableName)); + Options = options ?? throw new ArgumentNullException(nameof(options)); + IsJsonMode = isJsonMode; + } + + // ==================== Lifecycle (sync) ==================== + + /// + /// Waits until the record at the given offset has been durably acknowledged. + /// + public void WaitForOffset(long offset) + { + EnsureOpen(); + WithReadLock(() => + { + CResult result; + byte ok = NativeMethods.zerobus_stream_wait_for_offset(NativeHandle, offset, out result); + if (ok == 0) + { + string msg = Marshal.PtrToStringAnsi(result.ErrorMessage) ?? "Wait for offset failed"; + SafeFreeErrorMessage(result.ErrorMessage); + throw new ZerobusException(msg, isRetryable: result.IsRetryable); + } + }); + } + + /// + /// Blocks until all currently queued records have been durably acknowledged. + /// + public void Flush() + { + EnsureOpen(); + WithReadLock(() => + { + byte ok = NativeMethods.zerobus_stream_flush(NativeHandle); + if (ok == 0) + throw new ZerobusException("Flush failed. Check logs for details.", isRetryable: true); + }); + } + + /// + /// Gracefully closes the stream, flushing all pending records first. + /// After calling this method, unacked records can be retrieved via + /// . + /// + public void Close() + { + if (_disposed != 0 || _handle.IsClosed || _handle.IsInvalid) + return; + + WithWriteLock(() => + { + if (_disposed != 0 || _handle.IsClosed || _handle.IsInvalid) + return; + + byte ok = NativeMethods.zerobus_stream_close(NativeHandle); + if (ok == 1) + { + CacheUnackedData(); + } + }); + + DisposeNativeHandle(); + } + + // ==================== Lifecycle (async) ==================== + + /// + public Task WaitForOffsetAsync(long offset) + { + return Task.Run(() => WaitForOffset(offset)); + } + + /// + public Task FlushAsync() + { + return Task.Run(() => Flush()); + } + + /// + public Task CloseAsync() + { + return Task.Run(() => Close()); + } + + // ==================== Unacked records ==================== + + /// + /// Returns any records that were not acknowledged when the stream was closed. + /// Returns an empty list if called before the stream is closed. + /// + protected IReadOnlyList GetCachedUnackedRecords() + { + return CachedUnackedRecords ?? (IReadOnlyList)Array.Empty(); + } + + /// + /// Retrieves unacknowledged records from the live native stream. + /// + protected List GetNativeUnackedRecords() + { + EnsureOpen(); + CRecordArray array = NativeMethods.zerobus_stream_get_unacked_records(NativeHandle); + var result = new List((int)(ulong)array.Len); + + if (array.Records != IntPtr.Zero && array.Len != UIntPtr.Zero) + { + int recordSize = Marshal.SizeOf(typeof(CRecord)); + for (ulong i = 0; i < (ulong)array.Len; i++) + { + IntPtr recordPtr = array.Records + (int)(i * (ulong)recordSize); + var record = (CRecord)Marshal.PtrToStructure(recordPtr, typeof(CRecord))!; + if (record.Data != IntPtr.Zero && record.DataLen != UIntPtr.Zero) + { + int len = (int)(ulong)record.DataLen; + var bytes = new byte[len]; + Marshal.Copy(record.Data, bytes, 0, len); + result.Add(bytes); + } + } + } + + NativeMethods.zerobus_free_record_array(array); + return result; + } + + /// + /// Caches unacknowledged records before the native handle is destroyed. + /// + protected void CacheUnackedData() + { + try + { + CachedUnackedRecords = GetNativeUnackedRecords(); + } + catch + { + CachedUnackedRecords = new List(); + } + } + + // ==================== Guard / helpers ==================== + + /// + /// Throws if the stream is closed or disposed. + /// + protected void EnsureOpen() + { + if (IsClosed) + throw new ZerobusException("Stream is closed or disposed.", isRetryable: false); + } + + /// + /// Executes an action under the read lock. Used for ingest, flush, wait operations. + /// + protected void WithReadLock(Action action) + { + _lock.EnterReadLock(); + try { action(); } + finally { _lock.ExitReadLock(); } + } + + /// + /// Executes an action under the write lock. Used for close/dispose operations. + /// + protected void WithWriteLock(Action action) + { + _lock.EnterWriteLock(); + try { action(); } + finally { _lock.ExitWriteLock(); } + } + + /// + /// Executes a function under the read lock, returning its result. + /// + protected T WithReadLock(Func func) + { + _lock.EnterReadLock(); + try { return func(); } + finally { _lock.ExitReadLock(); } + } + + /// + /// Disposes native resources. + /// + protected void DisposeNativeHandle() + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) + { + _handle.Dispose(); + _lock.Dispose(); + } + } + + // ==================== IDisposable / IAsyncDisposable ==================== + + /// + /// Releases all resources used by the stream. + /// + public void Dispose() + { + if (_disposed != 0) return; + + if (!_handle.IsClosed && !_handle.IsInvalid) + { + try { CacheUnackedData(); } + catch { /* best effort */ } + } + + DisposeNativeHandle(); + } + +#if NET8_0_OR_GREATER + /// + /// Asynchronously releases all resources used by the stream. + /// + public ValueTask DisposeAsync() + { + Dispose(); + return ValueTask.CompletedTask; + } +#endif + + private static void SafeFreeErrorMessage(IntPtr msg) + { + if (msg != IntPtr.Zero) + { + try { NativeMethods.zerobus_free_error_message(msg); } + catch { /* best effort */ } + } + } +} diff --git a/dotnet/src/Databricks.Zerobus/CLAUDE.md b/dotnet/src/Databricks.Zerobus/CLAUDE.md new file mode 100644 index 00000000..a5f2dc17 --- /dev/null +++ b/dotnet/src/Databricks.Zerobus/CLAUDE.md @@ -0,0 +1,114 @@ +# .NET SDK + +A .NET wrapper around the Rust core via P/Invoke (C FFI). + +## Client Code Patterns (Read Before Writing or Reviewing Examples/Docs) + +Ingestion is asynchronous. The methods `IngestRecord()` and `IngestRecords()` return control the moment a record or batch enters a queue; the SDK handles transmission and acknowledgment tracking in the background. + +Four distinct patterns: + +1. **Idiomatic flow** — Ingest repeatedly in a loop, then issue a single `Flush()` call after a bounded batch (or periodically for long-running streams) to confirm durability. Alternatively, call `WaitForOffset()` on the last received offset, since acknowledgments are ordered and confirming the last one confirms everything prior. + +2. **Async monitoring** — Register an ack callback (`AckOnAckDelegate` / `AckOnErrorDelegate`) to observe progress without blocking the ingest loop. + +3. **Per-record `WaitForOffset()`** — Use this when a specific record must be acknowledged before proceeding. Avoid calling it after every record in a tight loop, as that limits throughput to one record per round-trip. + +4. **Stream class preference** — Use `ZerobusProtoStream` for protobuf ingestion or `ZerobusJsonStream` for JSON ingestion. + +## Structure + +``` +dotnet/ +├── src/Databricks.Zerobus/ # Main library +│ ├── Native/ # P/Invoke layer +│ │ ├── NativeMethods.cs # [DllImport] declarations +│ │ ├── NativeLibraryResolver.cs # Platform lib loading +│ │ ├── InteropStructs.cs # C struct marshaling +│ │ └── SafeHandles/ # SafeHandle subclasses +│ ├── ZerobusSdk.cs # Main SDK class + +│ ├── ZerobusProtoStream.cs # Proto ingestion stream +│ ├── ZerobusJsonStream.cs # JSON ingestion stream +│ ├── ZerobusArrowStream.cs # Arrow Flight stream (Beta) +│ ├── StreamBuilder.cs # Fluent builder API +│ ├── StreamConfigurationOptions.cs # gRPC stream config +│ ├── ArrowStreamConfigurationOptions.cs # Arrow stream config +│ ├── ProtoSchema.cs # UC schema → proto +│ └── ... # Exceptions, callbacks, enums +├── tests/ # xUnit test suite +├── examples/ # Runnable examples (JsonIngestion, ProtoIngestion, ArrowIngestion) +├── tools/ # generate-proto CLI +└── Zerobus-DotNet.sln # Solution file +``` + +The native library is built from `rust/ffi/` and bundled in the NuGet package for all supported platforms under `runtimes/{rid}/native/`. + +## Build Commands + +All commands are run from the `dotnet/` directory: + +- `dotnet build` — Compile +- `dotnet test` — Run all tests +- `dotnet test --filter "FullyQualifiedName~ClassName"` — Run specific tests +- `dotnet format` — Auto-format code (whitespace + style) +- `dotnet format --verify-no-changes` — Check formatting (CI) +- `dotnet pack src/Databricks.Zerobus -c Release` — Create NuGet package +- `dotnet restore --configfile NuGet.Config` — Restore with clean package sources + +## FFI Boundary: P/Invoke + +The .NET SDK uses the C FFI from `rust/ffi/`: + +1. **Handle pattern** — .NET holds an `IntPtr` wrapped in `SafeHandle` subclasses. Every native method receives this handle. If the handle is invalid after close, Rust panics or returns an error. + +2. **No finalizers** — Unlike Go, Python, or TypeScript, .NET streams do not have GC-triggered cleanup. Users must call `Dispose()` explicitly or use `using` statements. Forgetting this causes native memory leakage. `SafeHandle` provides critical-finalization as a backstop. + +3. **IDisposable** — Both SDK and stream classes implement `IDisposable`. Always prefer `using` statements. + +4. **Async bridge** — `Task` bridges Rust futures to .NET. Stream creation returns `Task`, `Task>`, or `Task`. `BuildAsync()` wraps the synchronous native stream creation in `Task.Run()` for non-blocking behavior. + +5. **Thread safety** — The SDK is not thread-safe. Do not share SDK or stream instances across threads without external synchronization. + +6. **Native library loading** — `NativeLibraryResolver.cs` detects the OS and architecture, resolves the correct `.so`, `.dylib`, or `.dll` from NuGet `runtimes/{rid}/native/`, development paths, or the `ZEROBUS_NATIVE_LIB_PATH` environment variable. On .NET 8+, it uses `NativeLibrary.SetDllImportResolver`. On netstandard2.0, it falls back to `LoadLibrary` / `dlopen`. + +7. **Native method stability** — `[DllImport]` declarations in `NativeMethods.cs` are internal but must stay in sync with `rust/ffi/zerobus.h`. Changing one without the other causes `DllNotFoundException` or `EntryPointNotFoundException` at runtime. + +## Breaking Change Rules + +The public API is everything in the `Databricks.Zerobus` namespace with `public` visibility: + +- Removing or renaming public classes, methods, or properties is breaking +- Changing method signatures (parameter types, return types) is breaking +- Deprecation requires `[Obsolete("message")]` with XML doc explaining the replacement +- Native method signatures (`internal static extern`) must stay synchronized with `rust/ffi/` — modifying one without the other causes runtime errors + +## Performance Notes + +- P/Invoke calls have non-trivial overhead (~50-100ns per crossing). Batch APIs reduce this cost. +- Proto descriptors are transmitted as `byte[]` with a copy at the boundary, a one-time cost at stream creation. +- Record payloads (`byte[]` for proto, strings for JSON) are copied across P/Invoke. For high throughput, prefer proto with batch ingestion. +- `GCHandle.Alloc` is used to pin managed buffers during native calls — buffers are unpinned immediately after each call. +- Native library resolution runs once at class initialization time. + +## Changelog and Documentation + +- Every PR that changes user-facing behavior must update `dotnet/NEXT_CHANGELOG.md` under the appropriate section +- Update `dotnet/README.md` if the change impacts usage, setup, or API surface +- Add or update examples in `dotnet/examples/` for new or modified APIs +- Add XML doc comments for all new public classes and methods + +## Release + +- The version source is `dotnet/src/Databricks.Zerobus/Databricks.Zerobus.csproj` (`x.y.z`) +- Tag format: `dotnet/v` — triggers `release-dotnet.yml` which downloads FFI native libs from `release-ffi.yml`, places them into `runtimes/{rid}/native/`, builds the NuGet package, and optionally publishes to NuGet.org +- The NuGet package bundles native libraries for all 5 platforms. The release depends on `rust/ffi/` — if Rust FFI code changed, both sides must be coordinated +- On a version bump PR: update the version in `Databricks.Zerobus.csproj`, move `NEXT_CHANGELOG.md` contents into `CHANGELOG.md`, and reset `NEXT_CHANGELOG.md` + +## Config + +- .NET 8.0+ and .NET Standard 2.0 compatibility +- Implicit usings + nullable reference types enabled +- EditorConfig-based formatting (`dotnet format`) +- Dependency: `Google.Protobuf` 3.28.2 (for protobuf streams) +- Apache.Arrow is optional — Arrow schema IPC bytes are passed as `byte[]`, so any Arrow library can be used diff --git a/dotnet/src/Databricks.Zerobus/Databricks.Zerobus.csproj b/dotnet/src/Databricks.Zerobus/Databricks.Zerobus.csproj new file mode 100644 index 00000000..97d9d2e5 --- /dev/null +++ b/dotnet/src/Databricks.Zerobus/Databricks.Zerobus.csproj @@ -0,0 +1,67 @@ + + + + net8.0;netstandard2.0 + Databricks.Zerobus + Databricks.Zerobus + + + Databricks.Zerobus.Ingest + Databricks Zerobus Ingest SDK + High-throughput streaming ingestion SDK for Databricks Delta tables. Provides JSON, Protocol Buffers, and Apache Arrow Flight ingestion over gRPC. + Databricks + Apache-2.0 + https://github.com/databricks/zerobus-sdk + https://github.com/databricks/zerobus-sdk + databricks;zerobus;streaming;ingestion;grpc;arrow;protobuf;delta + 0.1.0 + true + + + true + true + true + snupkg + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dotnet/src/Databricks.Zerobus/EncodedBatch.cs b/dotnet/src/Databricks.Zerobus/EncodedBatch.cs new file mode 100644 index 00000000..6ebefe0d --- /dev/null +++ b/dotnet/src/Databricks.Zerobus/EncodedBatch.cs @@ -0,0 +1,27 @@ +namespace Databricks.Zerobus; + +/// +/// Represents an encoded Arrow Flight batch with its IPC bytes and individual record lengths. +/// Returned by . +/// +public sealed class EncodedBatch +{ + /// + /// The raw IPC-encoded batch bytes. + /// + public byte[] Data { get; } + + /// + /// The lengths of individual records within the batch. + /// + public int[] Lengths { get; } + + /// + /// Creates a new EncodedBatch. + /// + public EncodedBatch(byte[] data, int[] lengths) + { + Data = data ?? throw new ArgumentNullException(nameof(data)); + Lengths = lengths ?? throw new ArgumentNullException(nameof(lengths)); + } +} diff --git a/dotnet/src/Databricks.Zerobus/HeadersProvider.cs b/dotnet/src/Databricks.Zerobus/HeadersProvider.cs new file mode 100644 index 00000000..4ce0d05d --- /dev/null +++ b/dotnet/src/Databricks.Zerobus/HeadersProvider.cs @@ -0,0 +1,9 @@ +namespace Databricks.Zerobus; + +/// +/// Delegate for providing custom authentication headers for stream creation. +/// The returned dictionary should include keys like "authorization" and +/// "x-databricks-zerobus-table-name". +/// +/// A dictionary of header name to header value. +public delegate IReadOnlyDictionary HeadersProviderDelegate(); diff --git a/dotnet/src/Databricks.Zerobus/IPCCompressionType.cs b/dotnet/src/Databricks.Zerobus/IPCCompressionType.cs new file mode 100644 index 00000000..80a1481a --- /dev/null +++ b/dotnet/src/Databricks.Zerobus/IPCCompressionType.cs @@ -0,0 +1,16 @@ +namespace Databricks.Zerobus; + +/// +/// Compression type for Apache Arrow Flight IPC messages. +/// +public enum IPCCompressionType +{ + /// No compression (default). + None = -1, + + /// LZ4 Frame compression. + Lz4Frame = 0, + + /// Zstandard compression. + Zstd = 1 +} diff --git a/dotnet/src/Databricks.Zerobus/Native/HeadersProviderBridge.cs b/dotnet/src/Databricks.Zerobus/Native/HeadersProviderBridge.cs new file mode 100644 index 00000000..0ad204f6 --- /dev/null +++ b/dotnet/src/Databricks.Zerobus/Native/HeadersProviderBridge.cs @@ -0,0 +1,110 @@ +using System.Runtime.InteropServices; +using System.Collections.Concurrent; + +namespace Databricks.Zerobus.Native; + +/// +/// Managed-to-native callback bridge for custom headers providers. +/// Keeps delegates alive via GCHandle and routes to a C# IHeadersProvider. +/// +internal sealed class HeadersProviderBridge : IDisposable +{ + private readonly HeadersProviderDelegate _provider; + private readonly NativeMethods.HeadersProviderNativeCallback _nativeCallback; + private GCHandle _handle; + private volatile int _disposed; + + /// + /// Creates a bridge from a managed headers provider. + /// + public HeadersProviderBridge(HeadersProviderDelegate provider) + { + _provider = provider ?? throw new ArgumentNullException(nameof(provider)); + _nativeCallback = OnGetHeaders; + _handle = GCHandle.Alloc(this); // Keep this bridge alive for the native callback + } + + /// + /// Native callback delegate pointer. Pass this to native SDK functions + /// that accept a HeadersProviderCallback. + /// + public IntPtr NativeCallbackPtr => + Marshal.GetFunctionPointerForDelegate(_nativeCallback); + + /// + /// User data pointer. Pass this alongside the callback pointer. + /// + public IntPtr UserDataPtr => + GCHandle.ToIntPtr(_handle); + + /// + /// The native callback implementation. Called from Rust across FFI. + /// + private CHeaders OnGetHeaders(IntPtr userData) + { + try + { + var headers = _provider(); + return BuildNativeHeaders(headers); + } + catch (Exception ex) + { + return new CHeaders + { + Headers = IntPtr.Zero, + Count = 0, + ErrorMessage = Marshal.StringToHGlobalAnsi(ex.Message) + }; + } + } + + private static unsafe CHeaders BuildNativeHeaders(IReadOnlyDictionary headers) + { + if (headers == null || headers.Count == 0) + return new CHeaders { Headers = IntPtr.Zero, Count = 0, ErrorMessage = IntPtr.Zero }; + + nuint count = (nuint)headers.Count; + IntPtr array = NativeMethods.zerobus_alloc_header_array(count); + if (array == IntPtr.Zero) + return new CHeaders + { + Headers = IntPtr.Zero, + Count = 0, + ErrorMessage = Marshal.StringToHGlobalAnsi("Failed to allocate header array") + }; + + int headerSize = Marshal.SizeOf(typeof(CHeader)); + int i = 0; + foreach (var kv in headers) + { + IntPtr headerPtr = array + (i * headerSize); + + IntPtr keyPtr = AllocUtf8(kv.Key); + IntPtr valuePtr = AllocUtf8(kv.Value); + + var header = new CHeader { Key = keyPtr, Value = valuePtr }; + Marshal.StructureToPtr(header, headerPtr, false); + i++; + } + + return new CHeaders { Headers = array, Count = count, ErrorMessage = IntPtr.Zero }; + } + + private static IntPtr AllocUtf8(string s) + { + byte[] bytes = System.Text.Encoding.UTF8.GetBytes(s); + IntPtr ptr = Marshal.AllocHGlobal(bytes.Length + 1); + Marshal.Copy(bytes, 0, ptr, bytes.Length); + Marshal.WriteByte(ptr, bytes.Length, 0); // null terminator + return ptr; + } + + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) + { + if (_handle.IsAllocated) + _handle.Free(); + } + } +} diff --git a/dotnet/src/Databricks.Zerobus/Native/InteropStructs.cs b/dotnet/src/Databricks.Zerobus/Native/InteropStructs.cs new file mode 100644 index 00000000..f3fd3d4c --- /dev/null +++ b/dotnet/src/Databricks.Zerobus/Native/InteropStructs.cs @@ -0,0 +1,96 @@ +using System.Runtime.InteropServices; + +namespace Databricks.Zerobus.Native; + +/// +/// C interop structs that mirror the Zerobus C FFI (zerobus.h). +/// Layout must match exactly — changing field types or order breaks native ABI. +/// + +[StructLayout(LayoutKind.Sequential)] +internal struct CHeader +{ + public IntPtr Key; // char* + public IntPtr Value; // char* +} + +[StructLayout(LayoutKind.Sequential)] +internal struct CHeaders +{ + public IntPtr Headers; // CHeader* + public nuint Count; + public IntPtr ErrorMessage; // char* +} + +[StructLayout(LayoutKind.Sequential)] +internal struct CResult +{ + [MarshalAs(UnmanagedType.U1)] + public bool Success; + public IntPtr ErrorMessage; // char* + [MarshalAs(UnmanagedType.U1)] + public bool IsRetryable; +} + +[StructLayout(LayoutKind.Sequential)] +internal struct CRecord +{ + [MarshalAs(UnmanagedType.U1)] + public bool IsJson; + public IntPtr Data; // uint8_t* + public nuint DataLen; +} + +[StructLayout(LayoutKind.Sequential)] +internal struct CRecordArray +{ + public IntPtr Records; // CRecord* + public nuint Len; +} + +[StructLayout(LayoutKind.Sequential)] +internal struct CArrowBatchArray +{ + public IntPtr Batches; // uint8_t** + public IntPtr Lengths; // uintptr_t* + public nuint Count; +} + +[StructLayout(LayoutKind.Sequential)] +internal struct CStreamConfigurationOptions +{ + public nuint MaxInflightRequests; // uintptr_t + [MarshalAs(UnmanagedType.U1)] + public bool Recovery; + public ulong RecoveryTimeoutMs; // uint64_t + public ulong RecoveryBackoffMs; // uint64_t + public uint RecoveryRetries; // uint32_t + public ulong ServerLackOfAckTimeoutMs; // uint64_t + public ulong FlushTimeoutMs; // uint64_t + public int RecordType; + public ulong StreamPausedMaxWaitTimeMs; // uint64_t + [MarshalAs(UnmanagedType.U1)] + public bool HasStreamPausedMaxWaitTimeMs; + public ulong CallbackMaxWaitTimeMs; // uint64_t + [MarshalAs(UnmanagedType.U1)] + public bool HasCallbackMaxWaitTimeMs; + public IntPtr AckOnAck; // void (*)(int64_t, void*) + public IntPtr AckOnError; // void (*)(int64_t, const char*, void*) + public IntPtr AckUserData; // void* +} + +[StructLayout(LayoutKind.Sequential)] +internal struct CArrowStreamConfigurationOptions +{ + public nuint MaxInflightBatches; // uintptr_t + [MarshalAs(UnmanagedType.U1)] + public bool Recovery; + public ulong RecoveryTimeoutMs; // uint64_t + public ulong RecoveryBackoffMs; // uint64_t + public uint RecoveryRetries; // uint32_t + public ulong ServerLackOfAckTimeoutMs; // uint64_t + public ulong FlushTimeoutMs; // uint64_t + public ulong ConnectionTimeoutMs; // uint64_t + public int IpcCompression; // -1=None, 0=LZ4_FRAME, 1=ZSTD + public ulong StreamPausedMaxWaitTimeMs; // uint64_t +} diff --git a/dotnet/src/Databricks.Zerobus/Native/NativeLibraryResolver.cs b/dotnet/src/Databricks.Zerobus/Native/NativeLibraryResolver.cs new file mode 100644 index 00000000..bad46dcc --- /dev/null +++ b/dotnet/src/Databricks.Zerobus/Native/NativeLibraryResolver.cs @@ -0,0 +1,184 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +namespace Databricks.Zerobus.Native; + +/// +/// Resolves the native Zerobus FFI library at runtime for the current platform. +/// Supports NuGet RID-based bundling, local development paths, and system-wide installations. +/// On .NET 8+, uses NativeLibrary.SetDllImportResolver. On netstandard2.0, pre-loads via LoadLibrary/dlopen. +/// +internal static class NativeLibraryResolver +{ + private static bool _initialized; + private static readonly object _lock = new(); + + /// + /// Ensures the native library is loaded and ready. Thread-safe, idempotent. + /// Call once at SDK initialization time. + /// + public static void EnsureLoaded() + { + if (_initialized) return; + + lock (_lock) + { + if (_initialized) return; + +#if NET8_0_OR_GREATER + var resolver = new DllImportResolver(ResolveNativeLibrary); + NativeLibrary.SetDllImportResolver( + typeof(NativeMethods).Assembly, resolver); +#else + // For netstandard2.0: pre-load the native library manually + LoadNativeLibrary(); +#endif + + _initialized = true; + } + } + +#if NET8_0_OR_GREATER + private static IntPtr ResolveNativeLibrary(string libraryName, Assembly assembly, DllImportSearchPath? searchPath) + { + if (libraryName != "zerobus_ffi") return IntPtr.Zero; + + foreach (var candidate in GetCandidatePaths()) + { + if (NativeLibrary.TryLoad(candidate, out var handle)) + { + return handle; + } + } + + if (NativeLibrary.TryLoad("zerobus_ffi", assembly, searchPath, out var defaultHandle)) + { + return defaultHandle; + } + + return IntPtr.Zero; + } +#else + private static void LoadNativeLibrary() + { + foreach (var candidate in GetCandidatePaths()) + { + if (TryLoadLibrary(candidate)) + { + return; + } + } + + // Try default search paths + TryLoadLibrary(GetLibraryFileName()); + } + + [DllImport("kernel32", SetLastError = true)] + private static extern IntPtr LoadLibrary(string lpFileName); + + [DllImport("libdl", SetLastError = true)] + private static extern IntPtr dlopen(string filename, int flags); + + private static bool TryLoadLibrary(string path) + { + try + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return LoadLibrary(path) != IntPtr.Zero; + } + else + { + return dlopen(path, 2 /* RTLD_NOW */) != IntPtr.Zero; + } + } + catch + { + return false; + } + } +#endif + + private static IEnumerable GetCandidatePaths() + { + string rid = GetRuntimeIdentifier(); + string libName = GetLibraryFileName(); + + string baseDir = AppContext.BaseDirectory; + yield return Path.Combine(baseDir, "runtimes", rid, "native", libName); + + string? devDir = FindDevRuntimesDir(); + if (devDir != null) + { + yield return Path.Combine(devDir, rid, "native", libName); + } + + string? envPath = Environment.GetEnvironmentVariable("ZEROBUS_NATIVE_LIB_PATH"); + if (!string.IsNullOrEmpty(envPath)) + { + yield return Path.Combine(envPath, libName); + yield return envPath; + } + + yield return Path.Combine(baseDir, libName); + yield return Path.Combine(baseDir, "../../../../runtimes", rid, "native", libName); + } + + private static string? FindDevRuntimesDir() + { + try + { + var asmDir = AppContext.BaseDirectory; + var dir = new DirectoryInfo(asmDir); + while (dir != null && dir.Parent != null) + { + var candidate = Path.Combine(dir.FullName, "runtimes"); + if (Directory.Exists(candidate)) + { + return candidate; + } + + var srcCandidate = Path.Combine(dir.Parent.FullName, "src", "Databricks.Zerobus", "runtimes"); + if (Directory.Exists(srcCandidate)) + { + return srcCandidate; + } + + dir = dir.Parent; + } + } + catch + { + // Ignore failures walking directories + } + return null; + } + + internal static string GetRuntimeIdentifier() + { + string os = GetOs(); + string arch = RuntimeInformation.ProcessArchitecture switch + { + Architecture.X64 => "x64", + Architecture.Arm64 => "arm64", + _ => "x64" + }; + return $"{os}-{arch}"; + } + + internal static string GetOs() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) return "win"; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) return "linux"; + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) return "osx"; + return "linux"; + } + + internal static string GetLibraryFileName() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) return "zerobus_ffi.dll"; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) return "libzerobus_ffi.so"; + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) return "libzerobus_ffi.dylib"; + return "libzerobus_ffi.so"; + } +} diff --git a/dotnet/src/Databricks.Zerobus/Native/NativeMethods.cs b/dotnet/src/Databricks.Zerobus/Native/NativeMethods.cs new file mode 100644 index 00000000..42d2feec --- /dev/null +++ b/dotnet/src/Databricks.Zerobus/Native/NativeMethods.cs @@ -0,0 +1,272 @@ +using System.Runtime.InteropServices; + +namespace Databricks.Zerobus.Native; + +/// +/// C FFI function declarations for the Zerobus native library. +/// All functions use Cdecl calling convention and map to zerobus_ffi.{dll,so,dylib}. +/// +internal static partial class NativeMethods +{ + // The library name is resolved at runtime by NativeLibraryResolver. + // This constant is used as a fallback and for DllImport decoration. + private const string LibraryName = "zerobus_ffi"; + + // ==================== SDK Builder API ==================== + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr zerobus_sdk_builder_new(); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern void zerobus_sdk_builder_endpoint(IntPtr builder, IntPtr endpoint); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern void zerobus_sdk_builder_unity_catalog_url(IntPtr builder, IntPtr url); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern void zerobus_sdk_builder_sdk_identifier(IntPtr builder, IntPtr identifier); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern void zerobus_sdk_builder_application_name(IntPtr builder, IntPtr name); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern void zerobus_sdk_builder_disable_tls(IntPtr builder); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr zerobus_sdk_builder_build(IntPtr builder); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern void zerobus_sdk_builder_free(IntPtr builder); + + // ==================== Legacy SDK API (ABI back-compat) ==================== + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr zerobus_sdk_new( + [MarshalAs(UnmanagedType.LPStr)] string endpoint, + [MarshalAs(UnmanagedType.LPStr)] string ucUrl, + out CResult result); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern void zerobus_sdk_free(IntPtr sdk); + + // ==================== Stream Creation (gRPC JSON/Proto) ==================== + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr zerobus_sdk_create_stream( + IntPtr sdk, + [MarshalAs(UnmanagedType.LPStr)] string tableName, + IntPtr descriptorProtoBytes, + UIntPtr descriptorProtoLen, + [MarshalAs(UnmanagedType.LPStr)] string clientId, + [MarshalAs(UnmanagedType.LPStr)] string clientSecret, + ref CStreamConfigurationOptions options, + out CResult result); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr zerobus_sdk_create_stream_with_headers_provider( + IntPtr sdk, + [MarshalAs(UnmanagedType.LPStr)] string tableName, + IntPtr descriptorProtoBytes, + UIntPtr descriptorProtoLen, + HeadersProviderNativeCallback headersProvider, + IntPtr userData, + ref CStreamConfigurationOptions options, + out CResult result); + + // ==================== Stream Lifecycle (Generic) ==================== + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern void zerobus_stream_free(IntPtr stream); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern byte zerobus_stream_flush(IntPtr stream); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern byte zerobus_stream_close(IntPtr stream); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern byte zerobus_stream_is_closed(IntPtr stream); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern byte zerobus_stream_wait_for_offset( + IntPtr stream, + long offset, + out CResult result); + + // ==================== Record Ingestion (Generic Stream) ==================== + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern long zerobus_stream_ingest_proto_record( + IntPtr stream, + IntPtr data, + UIntPtr dataLen); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern long zerobus_stream_ingest_json_record( + IntPtr stream, + IntPtr data, + UIntPtr dataLen); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern long zerobus_stream_ingest_proto_records( + IntPtr stream, + [MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 2)] + IntPtr[] records, + IntPtr[] lengths, + UIntPtr count); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern long zerobus_stream_ingest_json_records( + IntPtr stream, + [MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 2)] + IntPtr[] records, + IntPtr[] lengths, + UIntPtr count); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern void zerobus_stream_ingest_proto_record_nowait( + IntPtr stream, + IntPtr data, + UIntPtr dataLen); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern void zerobus_stream_ingest_json_record_nowait( + IntPtr stream, + IntPtr data, + UIntPtr dataLen); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern void zerobus_stream_ingest_proto_records_nowait( + IntPtr stream, + [MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 2)] + IntPtr[] records, + IntPtr[] lengths, + UIntPtr count); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern void zerobus_stream_ingest_json_records_nowait( + IntPtr stream, + [MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 2)] + IntPtr[] records, + IntPtr[] lengths, + UIntPtr count); + + // ==================== Unacked Records ==================== + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern CRecordArray zerobus_stream_get_unacked_records(IntPtr stream); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern void zerobus_free_record_array(CRecordArray array); + + // ==================== Arrow Stream API ==================== + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr zerobus_sdk_create_arrow_stream( + IntPtr sdk, + [MarshalAs(UnmanagedType.LPStr)] string tableName, + IntPtr schemaIpcBytes, + UIntPtr schemaIpcLen, + [MarshalAs(UnmanagedType.LPStr)] string clientId, + [MarshalAs(UnmanagedType.LPStr)] string clientSecret, + ref CArrowStreamConfigurationOptions options, + out CResult result); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr zerobus_sdk_create_arrow_stream_with_headers_provider( + IntPtr sdk, + [MarshalAs(UnmanagedType.LPStr)] string tableName, + IntPtr schemaIpcBytes, + UIntPtr schemaIpcLen, + HeadersProviderNativeCallback headersProvider, + IntPtr userData, + ref CArrowStreamConfigurationOptions options, + out CResult result); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern void zerobus_arrow_stream_free(IntPtr stream); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern long zerobus_arrow_stream_ingest_batch( + IntPtr stream, + IntPtr ipcBytes, + UIntPtr ipcLen, + out CResult result); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern byte zerobus_arrow_stream_wait_for_offset( + IntPtr stream, + long offset, + out CResult result); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern byte zerobus_arrow_stream_flush(IntPtr stream); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern byte zerobus_arrow_stream_close(IntPtr stream); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern CArrowBatchArray zerobus_arrow_stream_get_unacked_batches(IntPtr stream); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern void zerobus_arrow_free_batch_array(CArrowBatchArray array); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern byte zerobus_arrow_stream_is_closed(IntPtr stream); + + // ==================== Configuration Defaults ==================== + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern CStreamConfigurationOptions zerobus_get_default_config(); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern CArrowStreamConfigurationOptions zerobus_arrow_get_default_config(); + + // ==================== Protobuf Schema API ==================== + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr zerobus_proto_schema_from_uc_json( + [MarshalAs(UnmanagedType.LPStr)] string ucTableJson, + out CResult result); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr zerobus_proto_schema_descriptor_bytes( + IntPtr schema, + out UIntPtr outLen); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern byte zerobus_proto_schema_encode_json( + IntPtr schema, + [MarshalAs(UnmanagedType.LPStr)] string json, + out IntPtr outData, + out UIntPtr outLen); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern void zerobus_free_proto_bytes(IntPtr data, UIntPtr len); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern void zerobus_proto_schema_free(IntPtr schema); + + // ==================== Helper/Allocator Functions ==================== + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr zerobus_alloc_header_array(UIntPtr count); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr zerobus_alloc_cstring(IntPtr data, UIntPtr len); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern void zerobus_free_headers(CHeaders headers); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern void zerobus_free_error_message(IntPtr errorMessage); + + // ==================== Delegate Types ==================== + + /// + /// Native callback for custom headers provider. + /// Returns a CHeaders struct by value. + /// + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + internal delegate CHeaders HeadersProviderNativeCallback(IntPtr userData); +} diff --git a/dotnet/src/Databricks.Zerobus/Native/SafeHandles/ZerobusArrowStreamHandle.cs b/dotnet/src/Databricks.Zerobus/Native/SafeHandles/ZerobusArrowStreamHandle.cs new file mode 100644 index 00000000..4c3ca0c7 --- /dev/null +++ b/dotnet/src/Databricks.Zerobus/Native/SafeHandles/ZerobusArrowStreamHandle.cs @@ -0,0 +1,21 @@ +using Microsoft.Win32.SafeHandles; + +namespace Databricks.Zerobus.Native; + +/// +/// SafeHandle for a CArrowStream opaque pointer. +/// +internal sealed class ZerobusArrowStreamHandle : SafeHandleZeroOrMinusOneIsInvalid +{ + public ZerobusArrowStreamHandle() : base(true) { } + public ZerobusArrowStreamHandle(IntPtr handle) : base(true) { SetHandle(handle); } + + protected override bool ReleaseHandle() + { + if (!IsInvalid && !IsClosed) + { + NativeMethods.zerobus_arrow_stream_free(handle); + } + return true; + } +} diff --git a/dotnet/src/Databricks.Zerobus/Native/SafeHandles/ZerobusProtoSchemaHandle.cs b/dotnet/src/Databricks.Zerobus/Native/SafeHandles/ZerobusProtoSchemaHandle.cs new file mode 100644 index 00000000..09726ebc --- /dev/null +++ b/dotnet/src/Databricks.Zerobus/Native/SafeHandles/ZerobusProtoSchemaHandle.cs @@ -0,0 +1,21 @@ +using Microsoft.Win32.SafeHandles; + +namespace Databricks.Zerobus.Native; + +/// +/// SafeHandle for a CZerobusProtoSchema opaque pointer. +/// +internal sealed class ZerobusProtoSchemaHandle : SafeHandleZeroOrMinusOneIsInvalid +{ + public ZerobusProtoSchemaHandle() : base(true) { } + public ZerobusProtoSchemaHandle(IntPtr handle) : base(true) { SetHandle(handle); } + + protected override bool ReleaseHandle() + { + if (!IsInvalid && !IsClosed) + { + NativeMethods.zerobus_proto_schema_free(handle); + } + return true; + } +} diff --git a/dotnet/src/Databricks.Zerobus/Native/SafeHandles/ZerobusSdkHandle.cs b/dotnet/src/Databricks.Zerobus/Native/SafeHandles/ZerobusSdkHandle.cs new file mode 100644 index 00000000..65f881e4 --- /dev/null +++ b/dotnet/src/Databricks.Zerobus/Native/SafeHandles/ZerobusSdkHandle.cs @@ -0,0 +1,21 @@ +using Microsoft.Win32.SafeHandles; + +namespace Databricks.Zerobus.Native; + +/// +/// SafeHandle for a CZerobusSdk opaque pointer. +/// +internal sealed class ZerobusSdkHandle : SafeHandleZeroOrMinusOneIsInvalid +{ + public ZerobusSdkHandle() : base(true) { } + public ZerobusSdkHandle(IntPtr handle) : base(true) { SetHandle(handle); } + + protected override bool ReleaseHandle() + { + if (!IsInvalid && !IsClosed) + { + NativeMethods.zerobus_sdk_free(handle); + } + return true; + } +} diff --git a/dotnet/src/Databricks.Zerobus/Native/SafeHandles/ZerobusStreamHandle.cs b/dotnet/src/Databricks.Zerobus/Native/SafeHandles/ZerobusStreamHandle.cs new file mode 100644 index 00000000..5b33ab24 --- /dev/null +++ b/dotnet/src/Databricks.Zerobus/Native/SafeHandles/ZerobusStreamHandle.cs @@ -0,0 +1,21 @@ +using Microsoft.Win32.SafeHandles; + +namespace Databricks.Zerobus.Native; + +/// +/// SafeHandle for a CZerobusStream opaque pointer. +/// +internal sealed class ZerobusStreamHandle : SafeHandleZeroOrMinusOneIsInvalid +{ + public ZerobusStreamHandle() : base(true) { } + public ZerobusStreamHandle(IntPtr handle) : base(true) { SetHandle(handle); } + + protected override bool ReleaseHandle() + { + if (!IsInvalid && !IsClosed) + { + NativeMethods.zerobus_stream_free(handle); + } + return true; + } +} diff --git a/dotnet/src/Databricks.Zerobus/NonRetriableException.cs b/dotnet/src/Databricks.Zerobus/NonRetriableException.cs new file mode 100644 index 00000000..5c1441bf --- /dev/null +++ b/dotnet/src/Databricks.Zerobus/NonRetriableException.cs @@ -0,0 +1,23 @@ +namespace Databricks.Zerobus; + +/// +/// Represents a non-retryable Zerobus error. This exception is thrown when the SDK +/// encounters an error that will not succeed on retry (e.g., invalid schema, auth failure). +/// +public class NonRetriableException : ZerobusException +{ + /// + /// Creates a new NonRetriableException. + /// + public NonRetriableException(string message) : base(message, isRetryable: false) + { + } + + /// + /// Creates a new NonRetriableException with an inner exception. + /// + public NonRetriableException(string message, Exception? innerException) + : base(message, isRetryable: false, innerException) + { + } +} diff --git a/dotnet/src/Databricks.Zerobus/ProtoSchema.cs b/dotnet/src/Databricks.Zerobus/ProtoSchema.cs new file mode 100644 index 00000000..4587463f --- /dev/null +++ b/dotnet/src/Databricks.Zerobus/ProtoSchema.cs @@ -0,0 +1,120 @@ +using Databricks.Zerobus.Native; +using System.Runtime.InteropServices; + +namespace Databricks.Zerobus; + +/// +/// Wraps a Zerobus protocol buffer schema generated from a Unity Catalog table. +/// Provides methods to get the descriptor bytes and encode JSON records to protobuf. +/// Must be disposed after all operations complete. +/// +public sealed class ProtoSchema : IDisposable +{ + private ZerobusProtoSchemaHandle _handle; + private volatile int _disposed; + + private ProtoSchema(ZerobusProtoSchemaHandle handle) + { + _handle = handle; + } + + /// + /// Generates a proto schema from a Unity Catalog table JSON representation. + /// The JSON should be the response from the Databricks Unity Catalog API's get-table endpoint. + /// + /// JSON representation of the Unity Catalog table. + /// A new ProtoSchema instance. + /// Thrown if schema generation fails. + public static ProtoSchema FromUnityCatalogJson(string ucTableJson) + { + if (string.IsNullOrWhiteSpace(ucTableJson)) + throw new ArgumentException("Unity Catalog table JSON must not be empty", nameof(ucTableJson)); + + NativeLibraryResolver.EnsureLoaded(); + + var result = new CResult(); + IntPtr raw = NativeMethods.zerobus_proto_schema_from_uc_json(ucTableJson, out result); + + if (!result.Success || raw == IntPtr.Zero) + { + string msg = Marshal.PtrToStringAnsi(result.ErrorMessage) ?? "Unknown error generating proto schema"; + SafeFreeErrorMessageIfNeeded(result.ErrorMessage); + throw new ZerobusException(msg, isRetryable: result.IsRetryable); + } + + return new ProtoSchema(new ZerobusProtoSchemaHandle(raw)); + } + + /// + /// Returns the compiled protocol buffer descriptor bytes for this schema. + /// The returned bytes are valid until this schema is disposed. + /// + public byte[] GetDescriptorBytes() + { + EnsureNotDisposed(); + + UIntPtr outLen; + IntPtr raw = NativeMethods.zerobus_proto_schema_descriptor_bytes(_handle.DangerousGetHandle(), out outLen); + if (raw == IntPtr.Zero) + throw new ZerobusException("Failed to get descriptor bytes from proto schema", isRetryable: false); + + int len = (int)(ulong)outLen; + var bytes = new byte[len]; + Marshal.Copy(raw, bytes, 0, len); + return bytes; + } + + /// + /// Encodes a JSON record string into protocol buffer bytes using this schema. + /// The caller must free the returned bytes. + /// + /// The JSON record to encode. + /// The encoded protobuf bytes. + public byte[] EncodeJson(string json) + { + if (json == null) throw new ArgumentNullException(nameof(json)); + EnsureNotDisposed(); + + IntPtr outData; + UIntPtr outLen; + byte success = NativeMethods.zerobus_proto_schema_encode_json( + _handle.DangerousGetHandle(), json, out outData, out outLen); + + if (success == 0 || outData == IntPtr.Zero) + throw new ZerobusException("Failed to encode JSON to protobuf bytes", isRetryable: false); + + int len = (int)(ulong)outLen; + var bytes = new byte[len]; + Marshal.Copy(outData, bytes, 0, len); + + NativeMethods.zerobus_free_proto_bytes(outData, outLen); + + return bytes; + } + + /// + /// Disposes the schema, freeing native resources. + /// + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) + { + _handle.Dispose(); + } + } + + private void EnsureNotDisposed() + { + if (_disposed != 0 || _handle.IsClosed || _handle.IsInvalid) + throw new ObjectDisposedException(nameof(ProtoSchema)); + } + + private static void SafeFreeErrorMessageIfNeeded(IntPtr msg) + { + if (msg != IntPtr.Zero) + { + try { NativeMethods.zerobus_free_error_message(msg); } + catch { /* best effort */ } + } + } +} diff --git a/dotnet/src/Databricks.Zerobus/StreamBuilder.cs b/dotnet/src/Databricks.Zerobus/StreamBuilder.cs new file mode 100644 index 00000000..0653eb9f --- /dev/null +++ b/dotnet/src/Databricks.Zerobus/StreamBuilder.cs @@ -0,0 +1,396 @@ +using Google.Protobuf; +using Google.Protobuf.Reflection; + +namespace Databricks.Zerobus; + +/// +/// Fluent builder for creating Zerobus ingestion streams. +/// Mirrors the Rust SDK's stream builder API. +/// Obtain an instance via . +/// +/// +/// Usage: +/// +/// var stream = await sdk.StreamBuilder() +/// .Table("my_catalog.my_schema.my_table") +/// .OAuth("client-id", "client-secret") +/// .MaxInflightRecords(50000) +/// .Recovery(false) +/// .Json() +/// .BuildAsync(); +/// +/// +public sealed class StreamBuilder +{ + private readonly ZerobusSdk? _sdk; + private string? _tableName; + private string? _clientId; + private string? _clientSecret; + private int? _maxInflightRecords; + private bool? _recovery; + private int? _recoveryTimeoutMs; + private int? _recoveryBackoffMs; + private int? _recoveryRetries; + private int? _serverLackOfAckTimeoutMs; + private int? _flushTimeoutMs; + private AckOnAckDelegate? _onAck; + private AckOnErrorDelegate? _onError; + private object? _ackUserData; + + internal StreamBuilder(ZerobusSdk? sdk) + { + _sdk = sdk!; + } + + /// + /// Sets the fully qualified Unity Catalog table name (catalog.schema.table). + /// + public StreamBuilder Table(string tableName) + { + _tableName = RequireNonBlank(tableName, nameof(tableName)); + return this; + } + + /// + /// Sets the OAuth client credentials for authentication. + /// + public StreamBuilder OAuth(string clientId, string clientSecret) + { + _clientId = RequireNonBlank(clientId, nameof(clientId)); + _clientSecret = RequireNonBlank(clientSecret, nameof(clientSecret)); + return this; + } + + /// + /// Sets the maximum number of in-flight records. Must be positive. + /// + public StreamBuilder MaxInflightRecords(int maxInflightRecords) + { + _maxInflightRecords = RequirePositive(maxInflightRecords, nameof(maxInflightRecords)); + return this; + } + + /// + /// Enables or disables automatic stream recovery. + /// + public StreamBuilder Recovery(bool recovery) + { + _recovery = recovery; + return this; + } + + /// + /// Sets the recovery timeout in milliseconds. Must be non-negative. + /// + public StreamBuilder RecoveryTimeoutMs(int recoveryTimeoutMs) + { + _recoveryTimeoutMs = RequireNonNegative(recoveryTimeoutMs, nameof(recoveryTimeoutMs)); + return this; + } + + /// + /// Sets the recovery backoff in milliseconds. Must be non-negative. + /// + public StreamBuilder RecoveryBackoffMs(int recoveryBackoffMs) + { + _recoveryBackoffMs = RequireNonNegative(recoveryBackoffMs, nameof(recoveryBackoffMs)); + return this; + } + + /// + /// Sets the maximum number of recovery retries. Must be non-negative. + /// + public StreamBuilder RecoveryRetries(int recoveryRetries) + { + _recoveryRetries = RequireNonNegative(recoveryRetries, nameof(recoveryRetries)); + return this; + } + + /// + /// Sets the server lack-of-ack timeout in milliseconds. Must be positive. + /// + public StreamBuilder ServerLackOfAckTimeoutMs(int serverLackOfAckTimeoutMs) + { + _serverLackOfAckTimeoutMs = RequirePositive(serverLackOfAckTimeoutMs, nameof(serverLackOfAckTimeoutMs)); + return this; + } + + /// + /// Sets the flush timeout in milliseconds. Must be positive. + /// + public StreamBuilder FlushTimeoutMs(int flushTimeoutMs) + { + _flushTimeoutMs = RequirePositive(flushTimeoutMs, nameof(flushTimeoutMs)); + return this; + } + + /// + /// Sets the acknowledgment callback for this stream. + /// + public StreamBuilder AckCallback(AckOnAckDelegate? onAck, AckOnErrorDelegate? onError, object? userData = null) + { + _onAck = onAck; + _onError = onError; + _ackUserData = userData; + return this; + } + + /// + /// Returns a JsonStreamBuilder to configure and create a JSON ingestion stream. + /// + public JsonStreamBuilder Json() + { + ValidateRequired(); + return new JsonStreamBuilder(this); + } + + /// + /// Returns a ProtoStreamBuilder for the given compiled protobuf descriptor. + /// + /// + /// A compiled FileDescriptorProto for the table schema. Obtain this via + /// and + /// . + /// + public ProtoStreamBuilder CompiledProto(FileDescriptorProto descriptor) + { + if (descriptor == null) throw new ArgumentNullException(nameof(descriptor)); + ValidateRequired(); + return new ProtoStreamBuilder(this, descriptor); + } + + /// + /// Returns a ProtoStreamBuilder for the given descriptor proto bytes. + /// + /// Raw descriptor proto bytes. + public ProtoStreamBuilder CompiledProto(byte[] descriptorProtoBytes) + { + if (descriptorProtoBytes == null) throw new ArgumentNullException(nameof(descriptorProtoBytes)); + ValidateRequired(); + return new ProtoStreamBuilder(this, descriptorProtoBytes); + } + + /// + /// Returns an ArrowStreamBuilder for the given Arrow schema IPC bytes. (Beta) + /// + /// + /// The Arrow schema serialized as IPC format bytes. Generate these from an + /// Apache Arrow Schema using ArrowStreamWriter or equivalent. + /// + public ArrowStreamBuilder Arrow(byte[] schemaIpcBytes) + { + if (schemaIpcBytes == null) throw new ArgumentNullException(nameof(schemaIpcBytes)); + if (schemaIpcBytes.Length == 0) throw new ArgumentException("Schema IPC bytes must not be empty.", nameof(schemaIpcBytes)); + ValidateRequired(); + return new ArrowStreamBuilder(this, schemaIpcBytes); + } + + internal StreamConfigurationOptions ToStreamOptions() + { + var b = StreamConfigurationOptions.NewBuilder(); + + if (_maxInflightRecords.HasValue) b.SetMaxInflightRecords(_maxInflightRecords.Value); + if (_recovery.HasValue) b.SetRecovery(_recovery.Value); + if (_recoveryTimeoutMs.HasValue) b.SetRecoveryTimeoutMs(_recoveryTimeoutMs.Value); + if (_recoveryBackoffMs.HasValue) b.SetRecoveryBackoffMs(_recoveryBackoffMs.Value); + if (_recoveryRetries.HasValue) b.SetRecoveryRetries(_recoveryRetries.Value); + if (_serverLackOfAckTimeoutMs.HasValue) b.SetServerLackOfAckTimeoutMs(_serverLackOfAckTimeoutMs.Value); + if (_flushTimeoutMs.HasValue) b.SetFlushTimeoutMs(_flushTimeoutMs.Value); + + if (_onAck != null || _onError != null) + b.SetAckCallback(_onAck, _onError, _ackUserData); + + return b.Build(); + } + + private void ValidateRequired() + { + if (string.IsNullOrWhiteSpace(_tableName)) + throw new InvalidOperationException("Table name is required. Call .Table() before building."); + if (string.IsNullOrWhiteSpace(_clientId) || string.IsNullOrWhiteSpace(_clientSecret)) + throw new InvalidOperationException("OAuth credentials are required. Call .OAuth() before building."); + } + + // -- Inner builders -- + + /// + /// Builder for creating JSON ingestion streams. + /// + public sealed class JsonStreamBuilder + { + private readonly StreamBuilder _base; + + internal JsonStreamBuilder(StreamBuilder @base) + { + _base = @base; + } + + /// + /// Builds and opens the JSON ingestion stream. + /// + /// A ready-to-use JSON stream. + public async Task BuildAsync() + { + var options = _base.ToStreamOptions(); + return await _base._sdk!.CreateJsonStreamAsync( + _base._tableName!, + _base._clientId!, + _base._clientSecret!, + options).ConfigureAwait(false); + } + } + + /// + /// Builder for creating Protocol Buffer ingestion streams. + /// + public sealed class ProtoStreamBuilder + { + private readonly StreamBuilder _base; + private readonly byte[] _descriptorProtoBytes; + + internal ProtoStreamBuilder(StreamBuilder @base, FileDescriptorProto descriptor) + { + _base = @base; + _descriptorProtoBytes = descriptor.ToByteArray(); + } + + internal ProtoStreamBuilder(StreamBuilder @base, byte[] descriptorProtoBytes) + { + _base = @base; + _descriptorProtoBytes = descriptorProtoBytes; + } + + /// + /// Builds and opens the protobuf ingestion stream for the specified message type. + /// The type parameter must be a compiled protobuf message matching the descriptor. + /// + /// The protobuf message type. + /// A ready-to-use protobuf stream. + public async Task> BuildAsync() where T : Google.Protobuf.IMessage + { + var options = _base.ToStreamOptions(); + return await _base._sdk!.CreateProtoStreamAsync( + _base._tableName!, + _descriptorProtoBytes, + _base._clientId!, + _base._clientSecret!, + options).ConfigureAwait(false); + } + } + + /// + /// Builder for creating Arrow Flight ingestion streams. (Beta) + /// + public sealed class ArrowStreamBuilder + { + private readonly StreamBuilder _base; + private readonly byte[] _schemaIpcBytes; + private int? _maxInflightBatches; + private int? _connectionTimeoutMs; + private IPCCompressionType _ipcCompression = IPCCompressionType.None; + private long? _streamPausedMaxWaitTimeMs; + + internal ArrowStreamBuilder(StreamBuilder @base, byte[] schemaIpcBytes) + { + _base = @base; + _schemaIpcBytes = schemaIpcBytes; + } + + /// + /// Sets the maximum number of in-flight batches. Must be positive. + /// + public ArrowStreamBuilder MaxInflightBatches(int maxInflightBatches) + { + _maxInflightBatches = RequirePositive(maxInflightBatches, nameof(maxInflightBatches)); + return this; + } + + /// + /// Sets the connection timeout in milliseconds. Must be positive. + /// + public ArrowStreamBuilder ConnectionTimeoutMs(int connectionTimeoutMs) + { + _connectionTimeoutMs = RequirePositive(connectionTimeoutMs, nameof(connectionTimeoutMs)); + return this; + } + + /// + /// Sets the IPC compression type. Default: None. + /// + public ArrowStreamBuilder IpcCompression(IPCCompressionType compression) + { + _ipcCompression = compression; + return this; + } + + /// + /// Sets the stream paused max wait time in milliseconds. + /// A negative value means "wait the full server-specified duration." + /// + public ArrowStreamBuilder StreamPausedMaxWaitTimeMs(long ms) + { + _streamPausedMaxWaitTimeMs = ms; + return this; + } + + /// + /// Builds and opens the Arrow Flight ingestion stream. + /// + /// A ready-to-use Arrow Flight stream. + public async Task BuildAsync() + { + var opts = BuildOptions(); + return await _base._sdk!.CreateArrowStreamAsync( + _base._tableName!, + _schemaIpcBytes, + _base._clientId!, + _base._clientSecret!, + opts).ConfigureAwait(false); + } + + private ArrowStreamConfigurationOptions BuildOptions() + { + var b = ArrowStreamConfigurationOptions.NewBuilder(); + + // Merge shared config from base StreamBuilder + if (_base._recovery.HasValue) b.SetRecovery(_base._recovery.Value); + if (_base._recoveryTimeoutMs.HasValue) b.SetRecoveryTimeoutMs(_base._recoveryTimeoutMs.Value); + if (_base._recoveryBackoffMs.HasValue) b.SetRecoveryBackoffMs(_base._recoveryBackoffMs.Value); + if (_base._recoveryRetries.HasValue) b.SetRecoveryRetries(_base._recoveryRetries.Value); + if (_base._serverLackOfAckTimeoutMs.HasValue) b.SetServerLackOfAckTimeoutMs(_base._serverLackOfAckTimeoutMs.Value); + if (_base._flushTimeoutMs.HasValue) b.SetFlushTimeoutMs(_base._flushTimeoutMs.Value); + + // Arrow-specific + if (_maxInflightBatches.HasValue) b.SetMaxInflightBatches(_maxInflightBatches.Value); + if (_connectionTimeoutMs.HasValue) b.SetConnectionTimeoutMs(_connectionTimeoutMs.Value); + b.SetIpcCompression(_ipcCompression); + if (_streamPausedMaxWaitTimeMs.HasValue) b.SetStreamPausedMaxWaitTimeMs(_streamPausedMaxWaitTimeMs.Value); + + return b.Build(); + } + + } + + // -- Validation helpers -- + + private static int RequirePositive(int value, string name) + { + if (value <= 0) + throw new ArgumentException($"{name} must be positive, got {value}", name); + return value; + } + + private static int RequireNonNegative(int value, string name) + { + if (value < 0) + throw new ArgumentException($"{name} must be non-negative, got {value}", name); + return value; + } + + private static string RequireNonBlank(string value, string name) + { + if (string.IsNullOrWhiteSpace(value)) + throw new ArgumentException($"{name} must not be empty or whitespace.", name); + return value; + } +} diff --git a/dotnet/src/Databricks.Zerobus/StreamConfigurationOptions.cs b/dotnet/src/Databricks.Zerobus/StreamConfigurationOptions.cs new file mode 100644 index 00000000..00bc3332 --- /dev/null +++ b/dotnet/src/Databricks.Zerobus/StreamConfigurationOptions.cs @@ -0,0 +1,267 @@ +namespace Databricks.Zerobus; + +/// +/// Configuration options for Zerobus gRPC streams (JSON and Protocol Buffers). +/// Controls performance tuning, error handling, and callback configuration. +/// +public sealed class StreamConfigurationOptions +{ + /// + /// Default: 1,000,000 records. + /// + public const int DefaultMaxInflightRecords = 1_000_000; + + /// + /// Default: true (enabled). + /// + public const bool DefaultRecovery = true; + + /// + /// Default: 15,000 ms. + /// + public const int DefaultRecoveryTimeoutMs = 15_000; + + /// + /// Default: 2,000 ms. + /// + public const int DefaultRecoveryBackoffMs = 2_000; + + /// + /// Default: 4 retries. + /// + public const int DefaultRecoveryRetries = 4; + + /// + /// Default: 300,000 ms (5 minutes). + /// + public const int DefaultFlushTimeoutMs = 300_000; + + /// + /// Default: 60,000 ms (1 minute). + /// + public const int DefaultServerLackOfAckTimeoutMs = 60_000; + + /// + /// Maximum number of records that can be in flight. + /// Higher values improve throughput but use more memory. + /// + public int MaxInflightRecords { get; } + + /// + /// Whether automatic recovery is enabled. + /// + public bool Recovery { get; } + + /// + /// Maximum time to wait for a recovery operation (milliseconds). + /// + public int RecoveryTimeoutMs { get; } + + /// + /// Delay between consecutive recovery attempts (milliseconds). + /// + public int RecoveryBackoffMs { get; } + + /// + /// Maximum number of recovery attempts before giving up. + /// + public int RecoveryRetries { get; } + + /// + /// Maximum time to wait for a flush operation (milliseconds). + /// + public int FlushTimeoutMs { get; } + + /// + /// Maximum time to wait for server acknowledgment (milliseconds). + /// + public int ServerLackOfAckTimeoutMs { get; } + + /// + /// Maximum time to wait when the stream is paused (milliseconds). + /// A negative value means "wait the full server-specified duration." + /// Null means use the default. + /// + public long? StreamPausedMaxWaitTimeMs { get; } + + /// + /// Maximum wait time for acknowledgment callbacks (milliseconds). + /// Null means use the default. + /// + public long? CallbackMaxWaitTimeMs { get; } + + /// + /// Callback invoked when records are durably acknowledged. + /// + public AckOnAckDelegate? OnAck { get; } + + /// + /// Callback invoked when an error occurs for a specific offset. + /// + public AckOnErrorDelegate? OnError { get; } + + /// + /// Opaque user data passed through to ack callbacks. + /// + public object? AckUserData { get; } + + private StreamConfigurationOptions(Builder builder) + { + MaxInflightRecords = builder.MaxInflightRecords; + Recovery = builder.Recovery; + RecoveryTimeoutMs = builder.RecoveryTimeoutMs; + RecoveryBackoffMs = builder.RecoveryBackoffMs; + RecoveryRetries = builder.RecoveryRetries; + FlushTimeoutMs = builder.FlushTimeoutMs; + ServerLackOfAckTimeoutMs = builder.ServerLackOfAckTimeoutMs; + StreamPausedMaxWaitTimeMs = builder.StreamPausedMaxWaitTimeMs; + CallbackMaxWaitTimeMs = builder.CallbackMaxWaitTimeMs; + OnAck = builder.OnAck; + OnError = builder.OnError; + AckUserData = builder.AckUserData; + } + + /// + /// Returns the default stream configuration options. + /// + public static StreamConfigurationOptions Default => new(new Builder()); + + /// + /// Creates a new builder for StreamConfigurationOptions. + /// + public static Builder NewBuilder() => new(); + + /// + /// Builder for creating instances. + /// All parameters have sensible defaults if not specified. + /// + public sealed class Builder + { + internal int MaxInflightRecords { get; set; } = DefaultMaxInflightRecords; + internal bool Recovery { get; set; } = DefaultRecovery; + internal int RecoveryTimeoutMs { get; set; } = DefaultRecoveryTimeoutMs; + internal int RecoveryBackoffMs { get; set; } = DefaultRecoveryBackoffMs; + internal int RecoveryRetries { get; set; } = DefaultRecoveryRetries; + internal int FlushTimeoutMs { get; set; } = DefaultFlushTimeoutMs; + internal int ServerLackOfAckTimeoutMs { get; set; } = DefaultServerLackOfAckTimeoutMs; + internal long? StreamPausedMaxWaitTimeMs { get; set; } + internal long? CallbackMaxWaitTimeMs { get; set; } + internal AckOnAckDelegate? OnAck { get; set; } + internal AckOnErrorDelegate? OnError { get; set; } + internal object? AckUserData { get; set; } + + internal Builder() { } + + /// + /// Sets the maximum number of in-flight records. Must be positive. + /// + public Builder SetMaxInflightRecords(int maxInflightRecords) + { + if (maxInflightRecords <= 0) + throw new ArgumentException("maxInflightRecords must be positive", nameof(maxInflightRecords)); + MaxInflightRecords = maxInflightRecords; + return this; + } + + /// + /// Enables or disables automatic recovery. + /// + public Builder SetRecovery(bool recovery) + { + Recovery = recovery; + return this; + } + + /// + /// Sets the recovery timeout in milliseconds. Must be non-negative. + /// + public Builder SetRecoveryTimeoutMs(int recoveryTimeoutMs) + { + if (recoveryTimeoutMs < 0) + throw new ArgumentException("recoveryTimeoutMs must be non-negative", nameof(recoveryTimeoutMs)); + RecoveryTimeoutMs = recoveryTimeoutMs; + return this; + } + + /// + /// Sets the recovery backoff in milliseconds. Must be non-negative. + /// + public Builder SetRecoveryBackoffMs(int recoveryBackoffMs) + { + if (recoveryBackoffMs < 0) + throw new ArgumentException("recoveryBackoffMs must be non-negative", nameof(recoveryBackoffMs)); + RecoveryBackoffMs = recoveryBackoffMs; + return this; + } + + /// + /// Sets the maximum number of recovery retries. Must be non-negative. + /// + public Builder SetRecoveryRetries(int recoveryRetries) + { + if (recoveryRetries < 0) + throw new ArgumentException("recoveryRetries must be non-negative", nameof(recoveryRetries)); + RecoveryRetries = recoveryRetries; + return this; + } + + /// + /// Sets the flush timeout in milliseconds. Must be positive. + /// + public Builder SetFlushTimeoutMs(int flushTimeoutMs) + { + if (flushTimeoutMs <= 0) + throw new ArgumentException("flushTimeoutMs must be positive", nameof(flushTimeoutMs)); + FlushTimeoutMs = flushTimeoutMs; + return this; + } + + /// + /// Sets the server lack-of-ack timeout in milliseconds. Must be positive. + /// + public Builder SetServerLackOfAckTimeoutMs(int serverLackOfAckTimeoutMs) + { + if (serverLackOfAckTimeoutMs <= 0) + throw new ArgumentException("serverLackOfAckTimeoutMs must be positive", + nameof(serverLackOfAckTimeoutMs)); + ServerLackOfAckTimeoutMs = serverLackOfAckTimeoutMs; + return this; + } + + /// + /// Sets the maximum wait time when the stream is paused, in milliseconds. + /// A negative value means "wait the full server-specified duration." + /// + public Builder SetStreamPausedMaxWaitTimeMs(long? streamPausedMaxWaitTimeMs) + { + StreamPausedMaxWaitTimeMs = streamPausedMaxWaitTimeMs; + return this; + } + + /// + /// Sets the maximum wait time for acknowledgment callbacks. + /// + public Builder SetCallbackMaxWaitTimeMs(long? callbackMaxWaitTimeMs) + { + CallbackMaxWaitTimeMs = callbackMaxWaitTimeMs; + return this; + } + + /// + /// Sets the acknowledgment callback. The onAck delegate is called + /// when records are durably acknowledged; onError is called for errors. + /// + public Builder SetAckCallback(AckOnAckDelegate? onAck, AckOnErrorDelegate? onError, object? userData = null) + { + OnAck = onAck; + OnError = onError; + AckUserData = userData; + return this; + } + + /// + /// Builds the instance. + /// + public StreamConfigurationOptions Build() => new(this); + } +} diff --git a/dotnet/src/Databricks.Zerobus/TableProperties.cs b/dotnet/src/Databricks.Zerobus/TableProperties.cs new file mode 100644 index 00000000..af2eb3f4 --- /dev/null +++ b/dotnet/src/Databricks.Zerobus/TableProperties.cs @@ -0,0 +1,56 @@ +namespace Databricks.Zerobus; + +/// +/// Represents the properties of a Unity Catalog table as returned by the +/// Databricks Unity Catalog API. Used for proto schema generation. +/// +public sealed class TableProperties +{ + /// + /// The fully qualified table name (catalog.schema.table). + /// + public string TableName { get; } + + /// + /// The table's columns as a list of column definitions. + /// + public IReadOnlyList Columns { get; } + + /// + /// Creates a new TableProperties. + /// + public TableProperties(string tableName, IReadOnlyList columns) + { + TableName = tableName ?? throw new ArgumentNullException(nameof(tableName)); + Columns = columns ?? throw new ArgumentNullException(nameof(columns)); + } +} + +/// +/// Represents a column definition from a Unity Catalog table. +/// +public sealed class ColumnDefinition +{ + /// The column name. + public string Name { get; } + + /// The SQL data type string (e.g., "INT", "STRING", "BIGINT"). + public string TypeName { get; } + + /// Whether the column is nullable. + public bool Nullable { get; } + + /// Optional column comment. + public string? Comment { get; } + + /// + /// Creates a new ColumnDefinition. + /// + public ColumnDefinition(string name, string typeName, bool nullable, string? comment = null) + { + Name = name ?? throw new ArgumentNullException(nameof(name)); + TypeName = typeName ?? throw new ArgumentNullException(nameof(typeName)); + Nullable = nullable; + Comment = comment; + } +} diff --git a/dotnet/src/Databricks.Zerobus/ZerobusArrowStream.cs b/dotnet/src/Databricks.Zerobus/ZerobusArrowStream.cs new file mode 100644 index 00000000..f067cde9 --- /dev/null +++ b/dotnet/src/Databricks.Zerobus/ZerobusArrowStream.cs @@ -0,0 +1,202 @@ +using Databricks.Zerobus.Native; +using System.Runtime.InteropServices; + +namespace Databricks.Zerobus; + +/// +/// Stream for ingesting Apache Arrow RecordBatches into a Unity Catalog Delta table. +/// Uses the Arrow Flight protocol on the same gRPC connection as the standard streams. +/// +/// +/// Beta: The Arrow Flight ingestion API is in beta and may change in future releases. +/// Best suited for naturally columnar or batched workloads. +/// +public sealed class ZerobusArrowStream : IDisposable +{ + private ZerobusArrowStreamHandle _handle; + private volatile int _disposed; + + /// + /// The fully qualified Unity Catalog table name. + /// + public string TableName { get; } + + /// + /// The Arrow stream configuration options. + /// + public ArrowStreamConfigurationOptions Options { get; } + + /// + /// OAuth client ID used for stream authentication. + /// + public string ClientId { get; } + + /// + /// Whether the stream has been closed or disposed. + /// + public bool IsClosed => _disposed != 0 || _handle.IsClosed || _handle.IsInvalid; + + internal IntPtr NativeHandle => _handle.DangerousGetHandle(); + + internal ZerobusArrowStream( + IntPtr nativeHandle, + string tableName, + ArrowStreamConfigurationOptions options, + string clientId, + string clientSecret) + { + _handle = new ZerobusArrowStreamHandle(nativeHandle); + TableName = tableName ?? throw new ArgumentNullException(nameof(tableName)); + Options = options ?? throw new ArgumentNullException(nameof(options)); + ClientId = clientId ?? throw new ArgumentNullException(nameof(clientId)); + } + + /// + /// Ingests an Arrow IPC-encoded RecordBatch and returns its offset. + /// + /// The IPC-serialized Arrow RecordBatch. + /// The offset of the ingested batch, or -1 on error. + public long IngestBatch(byte[] ipcBytes) + { + if (ipcBytes == null) throw new ArgumentNullException(nameof(ipcBytes)); + EnsureOpen(); + + GCHandle handle = GCHandle.Alloc(ipcBytes, GCHandleType.Pinned); + try + { + CResult result; + long offset = NativeMethods.zerobus_arrow_stream_ingest_batch( + NativeHandle, + handle.AddrOfPinnedObject(), + (UIntPtr)ipcBytes.Length, + out result); + + if (offset < 0) + { + string msg = Marshal.PtrToStringAnsi(result.ErrorMessage) ?? "Failed to ingest Arrow batch"; + SafeFreeErrorMessage(result.ErrorMessage); + throw new ZerobusException(msg, isRetryable: result.IsRetryable); + } + + return offset; + } + finally + { + handle.Free(); + } + } + + /// + /// Waits until the batch at the given offset has been durably acknowledged. + /// + public void WaitForOffset(long offset) + { + EnsureOpen(); + CResult result; + byte ok = NativeMethods.zerobus_arrow_stream_wait_for_offset(NativeHandle, offset, out result); + if (ok == 0) + { + string msg = Marshal.PtrToStringAnsi(result.ErrorMessage) ?? "Arrow wait_for_offset failed"; + SafeFreeErrorMessage(result.ErrorMessage); + throw new ZerobusException(msg, isRetryable: result.IsRetryable); + } + } + + /// + /// Blocks until all currently queued Arrow batches have been durably acknowledged. + /// + public void Flush() + { + EnsureOpen(); + byte ok = NativeMethods.zerobus_arrow_stream_flush(NativeHandle); + if (ok == 0) + throw new ZerobusException("Arrow stream flush failed.", isRetryable: true); + } + + /// + /// Gracefully closes the stream, flushing all pending batches first. + /// + public void Close() + { + if (_disposed != 0 || _handle.IsClosed || _handle.IsInvalid) + return; + + NativeMethods.zerobus_arrow_stream_close(NativeHandle); + DisposeHandle(); + } + + /// + /// Returns unacknowledged Arrow batches. Only valid while the stream is open. + /// + public IReadOnlyList GetUnackedBatches() + { + EnsureOpen(); + CArrowBatchArray array = NativeMethods.zerobus_arrow_stream_get_unacked_batches(NativeHandle); + var result = new List((int)(ulong)array.Count); + + if (array.Batches != IntPtr.Zero && array.Count != UIntPtr.Zero) + { + int ptrSize = IntPtr.Size; + for (ulong i = 0; i < (ulong)array.Count; i++) + { + IntPtr batchPtr = Marshal.ReadIntPtr(array.Batches, (int)(i * (ulong)ptrSize)); + IntPtr lenPtr = array.Lengths == IntPtr.Zero + ? IntPtr.Zero + : Marshal.ReadIntPtr(array.Lengths, (int)(i * (ulong)ptrSize)); + + int len = lenPtr != IntPtr.Zero ? (int)lenPtr : 0; + // The batch data is a byte array; length isn't stored separately + // in the C struct — we need to get it from the IPC metadata. + // For now, the length pointer stores the byte count. + var data = new byte[len]; + if (batchPtr != IntPtr.Zero && len > 0) + { + Marshal.Copy(batchPtr, data, 0, len); + } + result.Add(new EncodedBatch(data, Array.Empty())); + } + } + + NativeMethods.zerobus_arrow_free_batch_array(array); + return result; + } + + /// + /// Disposes the stream, freeing native resources. + /// + public void Dispose() + { + if (_disposed != 0) return; + + if (!_handle.IsClosed && !_handle.IsInvalid) + { + try { NativeMethods.zerobus_arrow_stream_close(NativeHandle); } + catch { /* best effort */ } + } + + DisposeHandle(); + } + + private void DisposeHandle() + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) + { + _handle.Dispose(); + } + } + + private void EnsureOpen() + { + if (IsClosed) + throw new ZerobusException("Arrow stream is closed or disposed.", isRetryable: false); + } + + private static void SafeFreeErrorMessage(IntPtr msg) + { + if (msg != IntPtr.Zero) + { + try { NativeMethods.zerobus_free_error_message(msg); } + catch { /* best effort */ } + } + } +} diff --git a/dotnet/src/Databricks.Zerobus/ZerobusException.cs b/dotnet/src/Databricks.Zerobus/ZerobusException.cs new file mode 100644 index 00000000..2af5c713 --- /dev/null +++ b/dotnet/src/Databricks.Zerobus/ZerobusException.cs @@ -0,0 +1,37 @@ +namespace Databricks.Zerobus; + +/// +/// Base exception class for all Zerobus SDK errors. +/// +public class ZerobusException : Exception +{ + /// + /// Whether this error is retryable. Retryable errors can be automatically + /// recovered from if stream recovery is enabled. + /// + public bool IsRetryable { get; } + + /// + /// Creates a new ZerobusException. + /// + public ZerobusException(string message) : base(message) + { + } + + /// + /// Creates a new ZerobusException with retryability information. + /// + public ZerobusException(string message, bool isRetryable) : base(message) + { + IsRetryable = isRetryable; + } + + /// + /// Creates a new ZerobusException with retryability and inner exception. + /// + public ZerobusException(string message, bool isRetryable, Exception? innerException) + : base(message, innerException) + { + IsRetryable = isRetryable; + } +} diff --git a/dotnet/src/Databricks.Zerobus/ZerobusJsonStream.cs b/dotnet/src/Databricks.Zerobus/ZerobusJsonStream.cs new file mode 100644 index 00000000..f5532e6f --- /dev/null +++ b/dotnet/src/Databricks.Zerobus/ZerobusJsonStream.cs @@ -0,0 +1,167 @@ +using Databricks.Zerobus.Native; +using System.Runtime.InteropServices; +using System.Text; + +namespace Databricks.Zerobus; + +/// +/// Stream for ingesting JSON records into a Unity Catalog Delta table. +/// Records are sent as JSON strings and are schema-free — no protobuf +/// compilation is required. +/// +/// +/// JSON streams are ideal for quick starts or when the target schema is dynamic. +/// For production high-throughput workloads, prefer . +/// +public sealed class ZerobusJsonStream : BaseZerobusStream +{ + private readonly string _clientId; + private readonly string _clientSecret; + + /// + /// OAuth client ID used for stream authentication. + /// + public string ClientId => _clientId; + + /// + /// OAuth client secret used for stream authentication. + /// + public string ClientSecret => _clientSecret; + + internal ZerobusJsonStream( + IntPtr nativeHandle, + string tableName, + StreamConfigurationOptions options, + string clientId, + string clientSecret) + : base(nativeHandle, tableName, options, isJsonMode: true) + { + _clientId = clientId; + _clientSecret = clientSecret; + } + + // ==================== Single Record Ingestion ==================== + + /// + /// Ingests a single JSON record string and returns its offset. + /// + /// The JSON record to ingest. + /// The offset of the queued record, or -1 on error. + public long IngestRecord(string json) + { + if (json == null) throw new ArgumentNullException(nameof(json)); + EnsureOpen(); + + var bytes = Encoding.UTF8.GetBytes(json); + GCHandle handle = GCHandle.Alloc(bytes, GCHandleType.Pinned); + try + { + long offset = NativeMethods.zerobus_stream_ingest_json_record( + NativeHandle, + handle.AddrOfPinnedObject(), + (UIntPtr)bytes.Length); + + if (offset < 0) + throw new ZerobusException("Failed to ingest JSON record.", isRetryable: true); + + return offset; + } + finally + { + handle.Free(); + } + } + + // ==================== Batch Ingestion ==================== + + /// + /// Ingests a batch of JSON record strings and returns the offset of the last record. + /// Returns null if the input enumerable is empty. + /// + /// The JSON records to ingest. + /// The offset of the last queued record, or null if empty. + public long? IngestRecords(IReadOnlyList jsonRecords) + { + if (jsonRecords == null) throw new ArgumentNullException(nameof(jsonRecords)); + if (jsonRecords.Count == 0) return null; + EnsureOpen(); + + var encodedRecords = new byte[jsonRecords.Count][]; + var recordPtrs = new IntPtr[jsonRecords.Count]; + var lengths = new IntPtr[jsonRecords.Count]; + var gcHandles = new GCHandle[jsonRecords.Count]; + + try + { + for (int i = 0; i < jsonRecords.Count; i++) + { + if (jsonRecords[i] == null) + throw new ArgumentException($"JSON record at index {i} is null.", nameof(jsonRecords)); + + encodedRecords[i] = Encoding.UTF8.GetBytes(jsonRecords[i]); + gcHandles[i] = GCHandle.Alloc(encodedRecords[i], GCHandleType.Pinned); + recordPtrs[i] = gcHandles[i].AddrOfPinnedObject(); + lengths[i] = (IntPtr)encodedRecords[i].Length; + } + + long offset = NativeMethods.zerobus_stream_ingest_json_records( + NativeHandle, + recordPtrs, + lengths, + (UIntPtr)jsonRecords.Count); + + if (offset == -1) + throw new ZerobusException("Failed to ingest JSON records batch.", isRetryable: true); + if (offset == -2) + return null; + + return offset; + } + finally + { + foreach (var h in gcHandles) h.Free(); + } + } + + /// + /// Ingests a batch of JSON record strings from an enumerable. + /// + public long? IngestRecords(IEnumerable jsonRecords) + { + if (jsonRecords == null) throw new ArgumentNullException(nameof(jsonRecords)); + return IngestRecords(jsonRecords as IReadOnlyList ?? jsonRecords.ToArray()); + } + + // ==================== Unacknowledged Records ==================== + + /// + /// Returns unacknowledged records as JSON strings. + /// After the stream is closed, returns cached data. + /// + public IReadOnlyList GetUnackedRecords() + { + IReadOnlyList raw; + if (_disposed != 0 || IsClosed) + raw = GetCachedUnackedRecords(); + else + raw = GetNativeUnackedRecords(); + + var result = new string[raw.Count]; + for (int i = 0; i < raw.Count; i++) + { + result[i] = Encoding.UTF8.GetString(raw[i]); + } + return result; + } + + /// + /// Returns unacknowledged records as raw byte arrays. + /// + public IReadOnlyList GetUnackedRecordBytes() + { + if (_disposed != 0 || IsClosed) + return GetCachedUnackedRecords(); + + return GetNativeUnackedRecords(); + } +} diff --git a/dotnet/src/Databricks.Zerobus/ZerobusProtoStream.cs b/dotnet/src/Databricks.Zerobus/ZerobusProtoStream.cs new file mode 100644 index 00000000..521e3319 --- /dev/null +++ b/dotnet/src/Databricks.Zerobus/ZerobusProtoStream.cs @@ -0,0 +1,203 @@ +using Databricks.Zerobus.Native; +using Google.Protobuf; +using System.Runtime.InteropServices; + +namespace Databricks.Zerobus; + +/// +/// Stream for ingesting Protocol Buffer records into a Unity Catalog Delta table. +/// Supports both typed proto messages and pre-serialized byte arrays. +/// +/// The protobuf message type. Must implement . +/// +/// For single-record ingestion, use . +/// For batch ingestion, use . +/// +public sealed class ZerobusProtoStream : BaseZerobusStream where T : IMessage +{ + private readonly byte[] _descriptorProtoBytes; + private readonly string _clientId; + private readonly string _clientSecret; + + /// + /// The compiled descriptor proto bytes used to create this stream. + /// + public byte[] DescriptorProtoBytes => _descriptorProtoBytes; + + /// + /// OAuth client ID used for stream authentication. + /// + public string ClientId => _clientId; + + /// + /// OAuth client secret used for stream authentication. + /// + public string ClientSecret => _clientSecret; + + internal ZerobusProtoStream( + IntPtr nativeHandle, + string tableName, + StreamConfigurationOptions options, + byte[] descriptorProtoBytes, + string clientId, + string clientSecret) + : base(nativeHandle, tableName, options, isJsonMode: false) + { + _descriptorProtoBytes = descriptorProtoBytes; + _clientId = clientId; + _clientSecret = clientSecret; + } + + // ==================== Single Record Ingestion ==================== + + /// + /// Ingests a single protobuf record and returns its offset for acknowledgment tracking. + /// The record is serialized and queued; the call returns immediately. + /// + /// The protobuf message to ingest. + /// The offset of the queued record, or -1 on error. + /// Thrown if the stream is closed. + public long IngestRecord(T record) + { + if (record == null) throw new ArgumentNullException(nameof(record)); + EnsureOpen(); + byte[] bytes = record.ToByteArray(); + return IngestBytes(bytes); + } + + /// + /// Ingests a pre-serialized protobuf record and returns its offset. + /// + /// The pre-serialized protobuf message bytes. + /// The offset of the queued record, or -1 on error. + public long IngestRecord(byte[] encodedBytes) + { + if (encodedBytes == null) throw new ArgumentNullException(nameof(encodedBytes)); + EnsureOpen(); + return IngestBytes(encodedBytes); + } + + private long IngestBytes(byte[] bytes) + { + GCHandle handle = GCHandle.Alloc(bytes, GCHandleType.Pinned); + try + { + long offset = NativeMethods.zerobus_stream_ingest_proto_record( + NativeHandle, + handle.AddrOfPinnedObject(), + (UIntPtr)bytes.Length); + + if (offset < 0) + throw new ZerobusException("Failed to ingest proto record.", isRetryable: true); + + return offset; + } + finally + { + handle.Free(); + } + } + + // ==================== Batch Ingestion ==================== + + /// + /// Ingests a batch of protobuf records and returns the offset of the last record. + /// Returns null if the input enumerable is empty. + /// + /// The protobuf messages to ingest. + /// The offset of the last queued record, or null if empty. + public long? IngestRecords(IEnumerable records) + { + if (records == null) throw new ArgumentNullException(nameof(records)); + EnsureOpen(); + + var encodedRecords = new List(); + foreach (var record in records) + { + encodedRecords.Add(record.ToByteArray()); + } + + return IngestRecords(encodedRecords); + } + + /// + /// Ingests a batch of pre-serialized protobuf records. + /// Returns null if the input list is empty. + /// + /// The pre-serialized protobuf message bytes. + /// The offset of the last queued record, or null if empty. + public long? IngestRecords(IReadOnlyList encodedRecords) + { + if (encodedRecords == null) throw new ArgumentNullException(nameof(encodedRecords)); + if (encodedRecords.Count == 0) return null; + EnsureOpen(); + + var recordPtrs = new IntPtr[encodedRecords.Count]; + var lengths = new IntPtr[encodedRecords.Count]; + var gcHandles = new GCHandle[encodedRecords.Count]; + + try + { + for (int i = 0; i < encodedRecords.Count; i++) + { + gcHandles[i] = GCHandle.Alloc(encodedRecords[i], GCHandleType.Pinned); + recordPtrs[i] = gcHandles[i].AddrOfPinnedObject(); + lengths[i] = (IntPtr)encodedRecords[i].Length; + } + + long offset = NativeMethods.zerobus_stream_ingest_proto_records( + NativeHandle, + recordPtrs, + lengths, + (UIntPtr)encodedRecords.Count); + + if (offset == -1) + throw new ZerobusException("Failed to ingest proto records batch.", isRetryable: true); + if (offset == -2) + return null; + + return offset; + } + finally + { + foreach (var h in gcHandles) h.Free(); + } + } + + // ==================== Unacknowledged Records ==================== + + /// + /// Returns unacknowledged records. After the stream is closed, returns cached data. + /// + public IReadOnlyList GetUnackedRecords() + { + if (_disposed != 0 || IsClosed) + return GetCachedUnackedRecords(); + + return GetNativeUnackedRecords(); + } + + /// + /// Returns unacknowledged records as typed protobuf messages. + /// Requires a message parser for deserialization. + /// + public IReadOnlyList GetUnackedRecords(MessageParser parser) + { + if (parser == null) throw new ArgumentNullException(nameof(parser)); + + var raw = GetUnackedRecords(); + var result = new List(raw.Count); + foreach (var bytes in raw) + { + try + { + result.Add(parser.ParseFrom(bytes)); + } + catch (InvalidProtocolBufferException ex) + { + throw new ZerobusException("Failed to parse unacked record.", isRetryable: false, ex); + } + } + return result; + } +} diff --git a/dotnet/src/Databricks.Zerobus/ZerobusSdk.cs b/dotnet/src/Databricks.Zerobus/ZerobusSdk.cs new file mode 100644 index 00000000..af219790 --- /dev/null +++ b/dotnet/src/Databricks.Zerobus/ZerobusSdk.cs @@ -0,0 +1,521 @@ +using Databricks.Zerobus.Native; +using System.Runtime.InteropServices; + +namespace Databricks.Zerobus; + +/// +/// Main entry point for the Zerobus Ingest SDK. Manages the connection to the +/// Databricks Zerobus gRPC service and creates ingestion streams. +/// +/// +/// The SDK implements . Always use using statements +/// or explicitly call to free native resources. +/// The SDK is not thread-safe. Do not share SDK instances across threads +/// without external synchronization. +/// Use to configure and create SDK instances +/// with custom settings, or use the constructor for quick-start scenarios. +/// +/// +/// using var sdk = ZerobusSdk.CreateBuilder( +/// "https://workspace.databricks.com", +/// "https://workspace.databricks.com/api/2.1/unity-catalog") +/// .Build(); +/// +/// await using var stream = await sdk.StreamBuilder() +/// .Table("my_catalog.my_schema.my_table") +/// .OAuth(clientId, clientSecret) +/// .Json() +/// .BuildAsync(); +/// +/// stream.IngestRecord("{\"id\": 1, \"name\": \"test\"}"); +/// stream.Flush(); +/// +/// +public sealed class ZerobusSdk : IDisposable +{ + private ZerobusSdkHandle _handle; + private volatile int _disposed; + + /// + /// The gRPC server endpoint URL. + /// + public string ServerEndpoint { get; } + + /// + /// The Unity Catalog API endpoint URL. + /// + public string UnityCatalogEndpoint { get; } + + // Static initializer ensures native library is loaded once per process + static ZerobusSdk() + { + NativeLibraryResolver.EnsureLoaded(); + } + + private ZerobusSdk(IntPtr nativeHandle, string serverEndpoint, string unityCatalogEndpoint) + { + _handle = new ZerobusSdkHandle(nativeHandle); + ServerEndpoint = serverEndpoint ?? throw new ArgumentNullException(nameof(serverEndpoint)); + UnityCatalogEndpoint = unityCatalogEndpoint ?? throw new ArgumentNullException(nameof(unityCatalogEndpoint)); + } + + /// + /// Creates a new SDK instance with the specified endpoints. + /// For advanced configuration, use instead. + /// + /// The gRPC server endpoint URL. + /// The Unity Catalog API endpoint URL. + public ZerobusSdk(string serverEndpoint, string unityCatalogEndpoint) + : this(CreateSdkNative(serverEndpoint, unityCatalogEndpoint), serverEndpoint, unityCatalogEndpoint) + { + } + + /// + /// Creates a new SDK instance with an application name appended to the user-agent header. + /// + public ZerobusSdk(string serverEndpoint, string unityCatalogEndpoint, string applicationName) + : this(CreateSdkNative(serverEndpoint, unityCatalogEndpoint, applicationName), + serverEndpoint, unityCatalogEndpoint) + { + } + + /// + /// Creates a new SDK builder for advanced configuration. + /// + /// The gRPC server endpoint URL. + /// The Unity Catalog API endpoint URL. + public static SdkBuilder CreateBuilder(string serverEndpoint, string unityCatalogEndpoint) + { + return new SdkBuilder(serverEndpoint, unityCatalogEndpoint); + } + + private static IntPtr CreateSdkNative(string endpoint, string ucUrl, string? appName = null) + { + IntPtr builder = NativeMethods.zerobus_sdk_builder_new(); + if (builder == IntPtr.Zero) + throw new ZerobusException("Failed to create SDK builder.", isRetryable: false); + + try + { + var endpointPtr = Marshal.StringToHGlobalAnsi(endpoint); + var ucUrlPtr = Marshal.StringToHGlobalAnsi(ucUrl); + try + { + NativeMethods.zerobus_sdk_builder_endpoint(builder, endpointPtr); + NativeMethods.zerobus_sdk_builder_unity_catalog_url(builder, ucUrlPtr); + + if (!string.IsNullOrEmpty(appName)) + { + var appNamePtr = Marshal.StringToHGlobalAnsi(appName); + try + { + NativeMethods.zerobus_sdk_builder_application_name(builder, appNamePtr); + } + finally + { + Marshal.FreeHGlobal(appNamePtr); + } + } + } + finally + { + Marshal.FreeHGlobal(endpointPtr); + Marshal.FreeHGlobal(ucUrlPtr); + } + + IntPtr sdk = NativeMethods.zerobus_sdk_builder_build(builder); + // Builder is consumed/freed by build. If build fails, builder is freed but + // the returned pointer is null. + if (sdk == IntPtr.Zero) + { + throw new ZerobusException("Failed to build SDK. Check endpoint URLs and connectivity.", + isRetryable: true); + } + + return sdk; + } + catch + { + // Builder may still be alive if build wasn't called; free it + try { NativeMethods.zerobus_sdk_builder_free(builder); } + catch { /* best effort */ } + throw; + } + } + + /// + /// Creates a new stream builder bound to this SDK instance. + /// This is the recommended way to create streams. + /// + public StreamBuilder StreamBuilder() => new(this); + + /// + /// Creates a protobuf ingestion stream. Internal use by StreamBuilder. + /// + internal async Task> CreateProtoStreamAsync( + string tableName, + byte[] descriptorProtoBytes, + string clientId, + string clientSecret, + StreamConfigurationOptions options) where T : Google.Protobuf.IMessage + { + EnsureOpen(); + + // The native stream creation is synchronous under the hood (gRPC connection + // establishment is handled internally), but we wrap it in Task.Run for + // non-blocking behavior. + return await Task.Run(() => CreateProtoStreamInternal( + tableName, descriptorProtoBytes, clientId, clientSecret, options)) + .ConfigureAwait(false); + } + + /// + /// Creates a JSON ingestion stream. Internal use by StreamBuilder. + /// + internal async Task CreateJsonStreamAsync( + string tableName, + string clientId, + string clientSecret, + StreamConfigurationOptions options) + { + EnsureOpen(); + return await Task.Run(() => CreateJsonStreamInternal( + tableName, clientId, clientSecret, options)) + .ConfigureAwait(false); + } + + /// + /// Creates an Arrow Flight ingestion stream. Internal use by StreamBuilder. + /// + internal async Task CreateArrowStreamAsync( + string tableName, + byte[] schemaIpcBytes, + string clientId, + string clientSecret, + ArrowStreamConfigurationOptions options) + { + EnsureOpen(); + return await Task.Run(() => CreateArrowStreamInternal( + tableName, schemaIpcBytes, clientId, clientSecret, options)) + .ConfigureAwait(false); + } + + private ZerobusProtoStream CreateProtoStreamInternal( + string tableName, + byte[] descriptorProtoBytes, + string clientId, + string clientSecret, + StreamConfigurationOptions options) where T : Google.Protobuf.IMessage + { + NativeLibraryResolver.EnsureLoaded(); + + var cOpts = options.ToNative(); + CResult result; + + IntPtr descPtr = IntPtr.Zero; + GCHandle descHandle = default; + if (descriptorProtoBytes != null && descriptorProtoBytes.Length > 0) + { + descHandle = GCHandle.Alloc(descriptorProtoBytes, GCHandleType.Pinned); + descPtr = descHandle.AddrOfPinnedObject(); + } + + try + { + IntPtr stream = NativeMethods.zerobus_sdk_create_stream( + _handle.DangerousGetHandle(), + tableName, + descPtr, + (UIntPtr)(descriptorProtoBytes?.Length ?? 0), + clientId, + clientSecret, + ref cOpts, + out result); + + if (stream == IntPtr.Zero) + { + string msg = Marshal.PtrToStringAnsi(result.ErrorMessage) ?? "Failed to create proto stream"; + SafeFreeErrorMessage(result.ErrorMessage); + throw new ZerobusException(msg, isRetryable: result.IsRetryable); + } + + return new ZerobusProtoStream(stream, tableName, options, descriptorProtoBytes!, clientId, clientSecret); + } + finally + { + if (descHandle.IsAllocated) descHandle.Free(); + } + } + + private ZerobusJsonStream CreateJsonStreamInternal( + string tableName, + string clientId, + string clientSecret, + StreamConfigurationOptions options) + { + NativeLibraryResolver.EnsureLoaded(); + + var cOpts = options.ToNative(); + CResult result; + + IntPtr stream = NativeMethods.zerobus_sdk_create_stream( + _handle.DangerousGetHandle(), + tableName, + IntPtr.Zero, // null descriptor for JSON mode + UIntPtr.Zero, + clientId, + clientSecret, + ref cOpts, + out result); + + if (stream == IntPtr.Zero) + { + string msg = Marshal.PtrToStringAnsi(result.ErrorMessage) ?? "Failed to create JSON stream"; + SafeFreeErrorMessage(result.ErrorMessage); + throw new ZerobusException(msg, isRetryable: result.IsRetryable); + } + + return new ZerobusJsonStream(stream, tableName, options, clientId, clientSecret); + } + + private ZerobusArrowStream CreateArrowStreamInternal( + string tableName, + byte[] schemaIpcBytes, + string clientId, + string clientSecret, + ArrowStreamConfigurationOptions options) + { + NativeLibraryResolver.EnsureLoaded(); + + var cOpts = options.ToNative(); + CResult result; + + GCHandle schemaHandle = GCHandle.Alloc(schemaIpcBytes, GCHandleType.Pinned); + try + { + IntPtr stream = NativeMethods.zerobus_sdk_create_arrow_stream( + _handle.DangerousGetHandle(), + tableName, + schemaHandle.AddrOfPinnedObject(), + (UIntPtr)schemaIpcBytes.Length, + clientId, + clientSecret, + ref cOpts, + out result); + + if (stream == IntPtr.Zero) + { + string msg = Marshal.PtrToStringAnsi(result.ErrorMessage) ?? "Failed to create Arrow stream"; + SafeFreeErrorMessage(result.ErrorMessage); + throw new ZerobusException(msg, isRetryable: result.IsRetryable); + } + + return new ZerobusArrowStream(stream, tableName, options, clientId, clientSecret); + } + finally + { + schemaHandle.Free(); + } + } + + /// + /// Recreates a stream for recovery purposes. Uses the stream's stored + /// configuration to create a new native stream. + /// + internal async Task> RecreateStreamAsync( + ZerobusProtoStream existing) where T : Google.Protobuf.IMessage + { + if (existing == null) throw new ArgumentNullException(nameof(existing)); + return await CreateProtoStreamAsync( + existing.TableName, + existing.DescriptorProtoBytes, + existing.ClientId, + existing.ClientSecret, + existing.Options).ConfigureAwait(false); + } + + /// + /// Recreates a JSON stream for recovery purposes. + /// + internal async Task RecreateStreamAsync(ZerobusJsonStream existing) + { + if (existing == null) throw new ArgumentNullException(nameof(existing)); + return await CreateJsonStreamAsync( + existing.TableName, + existing.ClientId, + existing.ClientSecret, + existing.Options).ConfigureAwait(false); + } + + /// + /// Recreates an Arrow stream for recovery purposes. + /// + internal async Task RecreateStreamAsync(ZerobusArrowStream existing) + { + if (existing == null) throw new ArgumentNullException(nameof(existing)); + // Arrow schema IPC bytes aren't stored — caller provides them + throw new NotSupportedException( + "Arrow stream recreation requires the original schema IPC bytes. " + + "Create a new stream via StreamBuilder().Arrow(schema).BuildAsync()."); + } + + /// + /// Disposes the SDK, closing all streams and freeing native resources. + /// + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) + { + _handle.Dispose(); + } + } + + private void EnsureOpen() + { + if (_disposed != 0 || _handle.IsClosed || _handle.IsInvalid) + throw new ZerobusException("SDK is closed or disposed.", isRetryable: false); + } + + private static void SafeFreeErrorMessage(IntPtr msg) + { + if (msg != IntPtr.Zero) + { + try { NativeMethods.zerobus_free_error_message(msg); } + catch { /* best effort */ } + } + } + + /// + /// Builder for advanced SDK configuration. + /// + public sealed class SdkBuilder + { + private readonly string _endpoint; + private readonly string _ucUrl; + private string? _sdkIdentifier; + private string? _applicationName; + private bool _disableTls; + + internal SdkBuilder(string endpoint, string ucUrl) + { + _endpoint = endpoint ?? throw new ArgumentNullException(nameof(endpoint)); + _ucUrl = ucUrl ?? throw new ArgumentNullException(nameof(ucUrl)); + } + + /// + /// Overrides the SDK identifier in the user-agent header. + /// + public SdkBuilder SdkIdentifier(string identifier) + { + _sdkIdentifier = identifier; + return this; + } + + /// + /// Appends an application name to the user-agent header. + /// + public SdkBuilder ApplicationName(string name) + { + _applicationName = name; + return this; + } + + /// + /// Disables TLS verification (for development/testing only). + /// + public SdkBuilder DisableTls() + { + _disableTls = true; + return this; + } + + /// + /// Builds the SDK instance. + /// + public ZerobusSdk Build() + { + IntPtr builder = NativeMethods.zerobus_sdk_builder_new(); + if (builder == IntPtr.Zero) + throw new ZerobusException("Failed to create SDK builder.", isRetryable: false); + + try + { + SetStringParam(builder, _endpoint, NativeMethods.zerobus_sdk_builder_endpoint); + SetStringParam(builder, _ucUrl, NativeMethods.zerobus_sdk_builder_unity_catalog_url); + + if (!string.IsNullOrEmpty(_sdkIdentifier)) + SetStringParam(builder, _sdkIdentifier!, NativeMethods.zerobus_sdk_builder_sdk_identifier); + + if (!string.IsNullOrEmpty(_applicationName)) + SetStringParam(builder, _applicationName!, NativeMethods.zerobus_sdk_builder_application_name); + + if (_disableTls) + NativeMethods.zerobus_sdk_builder_disable_tls(builder); + + IntPtr sdk = NativeMethods.zerobus_sdk_builder_build(builder); + if (sdk == IntPtr.Zero) + throw new ZerobusException("Failed to build SDK.", isRetryable: true); + + return new ZerobusSdk(sdk, _endpoint, _ucUrl); + } + catch + { + try { NativeMethods.zerobus_sdk_builder_free(builder); } + catch { /* best effort */ } + throw; + } + } + + private static void SetStringParam(IntPtr builder, string value, Action setter) + { + IntPtr ptr = Marshal.StringToHGlobalAnsi(value); + try { setter(builder, ptr); } + finally { Marshal.FreeHGlobal(ptr); } + } + } +} + +/// +/// Extension methods for converting managed options to native structs. +/// +internal static class OptionsExtensions +{ + internal static CStreamConfigurationOptions ToNative(this StreamConfigurationOptions opts) + { + return new CStreamConfigurationOptions + { + MaxInflightRequests = (nuint)opts.MaxInflightRecords, + Recovery = opts.Recovery, + RecoveryTimeoutMs = (ulong)opts.RecoveryTimeoutMs, + RecoveryBackoffMs = (ulong)opts.RecoveryBackoffMs, + RecoveryRetries = (uint)opts.RecoveryRetries, + ServerLackOfAckTimeoutMs = (ulong)opts.ServerLackOfAckTimeoutMs, + FlushTimeoutMs = (ulong)opts.FlushTimeoutMs, + RecordType = 0, + StreamPausedMaxWaitTimeMs = opts.StreamPausedMaxWaitTimeMs.HasValue ? (ulong)opts.StreamPausedMaxWaitTimeMs.Value : 0UL, + HasStreamPausedMaxWaitTimeMs = opts.StreamPausedMaxWaitTimeMs.HasValue, + CallbackMaxWaitTimeMs = opts.CallbackMaxWaitTimeMs.HasValue ? (ulong)opts.CallbackMaxWaitTimeMs.Value : 0UL, + HasCallbackMaxWaitTimeMs = opts.CallbackMaxWaitTimeMs.HasValue, + AckOnAck = IntPtr.Zero, + AckOnError = IntPtr.Zero, + AckUserData = IntPtr.Zero, + }; + } + + internal static CArrowStreamConfigurationOptions ToNative(this ArrowStreamConfigurationOptions opts) + { + return new CArrowStreamConfigurationOptions + { + MaxInflightBatches = (nuint)opts.MaxInflightBatches, + Recovery = opts.Recovery, + RecoveryTimeoutMs = (ulong)opts.RecoveryTimeoutMs, + RecoveryBackoffMs = (ulong)opts.RecoveryBackoffMs, + RecoveryRetries = (uint)opts.RecoveryRetries, + ServerLackOfAckTimeoutMs = (ulong)opts.ServerLackOfAckTimeoutMs, + FlushTimeoutMs = (ulong)opts.FlushTimeoutMs, + ConnectionTimeoutMs = (ulong)opts.ConnectionTimeoutMs, + IpcCompression = (int)opts.IpcCompression, + StreamPausedMaxWaitTimeMs = (ulong)opts.StreamPausedMaxWaitTimeMs, + }; + } +} diff --git a/dotnet/src/Databricks.Zerobus/build/Databricks.Zerobus.props b/dotnet/src/Databricks.Zerobus/build/Databricks.Zerobus.props new file mode 100644 index 00000000..a065cab2 --- /dev/null +++ b/dotnet/src/Databricks.Zerobus/build/Databricks.Zerobus.props @@ -0,0 +1,5 @@ + + + false + + diff --git a/dotnet/src/Databricks.Zerobus/build/Databricks.Zerobus.targets b/dotnet/src/Databricks.Zerobus/build/Databricks.Zerobus.targets new file mode 100644 index 00000000..adab0f78 --- /dev/null +++ b/dotnet/src/Databricks.Zerobus/build/Databricks.Zerobus.targets @@ -0,0 +1,43 @@ + + + + + true + + + + + win + linux + osx + arm64 + x64 + $(ZerobusRidOs)-$(ZerobusRidArch) + + + + $(RuntimeIdentifier) + + + + + zerobus_ffi.dll + libzerobus_ffi.so + libzerobus_ffi.dylib + + + + + + + + + + + diff --git a/dotnet/tests/Databricks.Zerobus.IntegrationTests/Databricks.Zerobus.IntegrationTests.csproj b/dotnet/tests/Databricks.Zerobus.IntegrationTests/Databricks.Zerobus.IntegrationTests.csproj new file mode 100644 index 00000000..d2260078 --- /dev/null +++ b/dotnet/tests/Databricks.Zerobus.IntegrationTests/Databricks.Zerobus.IntegrationTests.csproj @@ -0,0 +1,36 @@ + + + + net8.0 + Databricks.Zerobus.IntegrationTests + enable + enable + false + true + + + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + + + diff --git a/dotnet/tests/Databricks.Zerobus.IntegrationTests/IntegrationTests.cs b/dotnet/tests/Databricks.Zerobus.IntegrationTests/IntegrationTests.cs new file mode 100644 index 00000000..a4b3db65 --- /dev/null +++ b/dotnet/tests/Databricks.Zerobus.IntegrationTests/IntegrationTests.cs @@ -0,0 +1,376 @@ +using Databricks.Zerobus; +using Xunit; + +namespace Databricks.Zerobus.IntegrationTests; + +/// +/// Full end-to-end integration tests using a real gRPC mock server. +/// Each test runs against its own server on a unique port. +/// Tests auto-skip when the native library (zerobus_ffi) is not available. +/// +[Collection("Integration")] +[Trait("Category", "Integration")] +public class IntegrationTests : IAsyncLifetime +{ + private readonly MockZerobusServer _server = new(); + private static readonly bool NativeAvailable = NativeLibraryHelper.IsNativeLibraryAvailable(); + + public async Task InitializeAsync() => await _server.StartAsync(); + public Task DisposeAsync() => _server.DisposeAsync().AsTask(); + + // ================================================================ + // Stream Creation + // ================================================================ + + [Fact] + public async Task CreateJsonStream_Success() + { + if (!NativeAvailable) return; + + using var sdk = CreateSdk(); + using var stream = await BuildJsonStream(sdk); + + Assert.NotNull(stream); + Assert.Equal("catalog.schema.table", stream.TableName); + Assert.False(stream.IsClosed); + } + + [Fact] + public async Task CreateProtoStream_Success() + { + if (!NativeAvailable) return; + + using var sdk = CreateSdk(); + using var stream = await BuildProtoStream(sdk); + + Assert.NotNull(stream); + } + + // ================================================================ + // Single Record Ingestion + // ================================================================ + + [Fact] + public async Task IngestJsonRecord_ReturnsOffset() + { + if (!NativeAvailable) return; + + using var sdk = CreateSdk(); + using var stream = await BuildJsonStream(sdk); + + long offset = stream.IngestRecord("{\"id\": 1, \"name\": \"test\"}"); + Assert.True(offset >= 0); + } + + [Fact] + public async Task IngestProtoRecord_ReturnsOffset() + { + if (!NativeAvailable) return; + + using var sdk = CreateSdk(); + using var stream = await BuildProtoStream(sdk); + + var record = new FakeProtoMessage { Id = 42, Name = "test" }; + long offset = stream.IngestRecord(record); + Assert.True(offset >= 0); + } + + // ================================================================ + // Batch Ingestion + // ================================================================ + + [Fact] + public async Task IngestJsonBatch_ReturnsOffset() + { + if (!NativeAvailable) return; + + using var sdk = CreateSdk(); + using var stream = await BuildJsonStream(sdk); + + var records = new[] { "{\"a\":1}", "{\"b\":2}", "{\"c\":3}" }; + long? lastOffset = stream.IngestRecords(records); + + Assert.NotNull(lastOffset); + Assert.True(lastOffset >= 0); + } + + [Fact] + public async Task IngestProtoBatch_ReturnsOffset() + { + if (!NativeAvailable) return; + + using var sdk = CreateSdk(); + using var stream = await BuildProtoStream(sdk); + + var records = new List + { + new() { Id = 1, Name = "a" }, + new() { Id = 2, Name = "b" }, + new() { Id = 3, Name = "c" } + }; + + long? lastOffset = stream.IngestRecords(records); + Assert.NotNull(lastOffset); + Assert.True(lastOffset >= 0); + } + + // ================================================================ + // Flush & WaitForOffset + // ================================================================ + + [Fact] + public async Task Flush_AfterIngest_DoesNotThrow() + { + if (!NativeAvailable) return; + + using var sdk = CreateSdk(); + using var stream = await BuildJsonStream(sdk); + + stream.IngestRecord("{\"x\":1}"); + stream.Flush(); + } + + [Fact] + public async Task WaitForOffset_AfterIngest_DoesNotThrow() + { + if (!NativeAvailable) return; + + using var sdk = CreateSdk(); + using var stream = await BuildJsonStream(sdk); + + long offset = stream.IngestRecord("{\"x\":1}"); + stream.WaitForOffset(offset); + } + + // ================================================================ + // Stream Lifecycle + // ================================================================ + + [Fact] + public async Task Close_MarksStreamAsClosed() + { + if (!NativeAvailable) return; + + using var sdk = CreateSdk(); + using var stream = await BuildJsonStream(sdk); + + Assert.False(stream.IsClosed); + stream.Close(); + Assert.True(stream.IsClosed); + } + + [Fact] + public async Task Dispose_MarksStreamAsClosed() + { + if (!NativeAvailable) return; + + using var sdk = CreateSdk(); + using var stream = await BuildJsonStream(sdk); + + stream.Dispose(); + Assert.True(stream.IsClosed); + } + + [Fact] + public async Task IngestAfterClose_Throws() + { + if (!NativeAvailable) return; + + using var sdk = CreateSdk(); + using var stream = await BuildJsonStream(sdk); + stream.Close(); + stream.Dispose(); + + Assert.Throws(() => stream.IngestRecord("{\"x\":1}")); + } + + // ================================================================ + // Unacked Records After Close + // ================================================================ + + [Fact] + public async Task GetUnackedRecords_AfterClose_ReturnsRecords() + { + if (!NativeAvailable) return; + + using var sdk = CreateSdk(); + using var stream = await BuildJsonStream(sdk); + + stream.IngestRecord("{\"x\":1}"); + stream.IngestRecord("{\"x\":2}"); + + stream.Close(); + var unacked = stream.GetUnackedRecords(); + stream.Dispose(); + + Assert.NotNull(unacked); + } + + // ================================================================ + // Recovery (Stream Recreation) + // ================================================================ + + [Fact] + public async Task RecreateStream_PreservesConfiguration() + { + if (!NativeAvailable) return; + + using var sdk = CreateSdk(); + var original = await BuildJsonStream(sdk); + original.Close(); + + var recreated = await sdk.RecreateStreamAsync(original); + + Assert.NotNull(recreated); + Assert.Equal(original.TableName, recreated.TableName); + Assert.Equal(original.ClientId, recreated.ClientId); + + recreated.Dispose(); + original.Dispose(); + } + + // ================================================================ + // Error Scenarios + // ================================================================ + + [Fact] + public async Task StreamCreation_InvalidTable_Throws() + { + _server.Service.ShouldAcceptStream = false; + _server.Service.ErrorMessage = "Table not found"; + + if (!NativeAvailable) return; + + using var sdk = CreateSdk(); + await Assert.ThrowsAsync(() => + sdk.StreamBuilder() + .Table("bad.table") + .OAuth("id", "secret") + .Json() + .BuildAsync()); + } + + [Fact] + public async Task Ingest_SdkDisposed_DoesNotCrash() + { + if (!NativeAvailable) return; + + var sdk = CreateSdk(); + var stream = await BuildJsonStream(sdk); + sdk.Dispose(); + + // Stream may or may not be affected by SDK disposal + Assert.True(stream.IsClosed || !stream.IsClosed); + stream.Dispose(); + } + + // ================================================================ + // Configuration + // ================================================================ + + [Fact] + public async Task StreamBuilder_CustomOptions_Applied() + { + if (!NativeAvailable) return; + + using var sdk = CreateSdk(); + using var stream = await sdk.StreamBuilder() + .Table("catalog.schema.table") + .OAuth("id", "secret") + .MaxInflightRecords(50_000) + .Recovery(false) + .FlushTimeoutMs(60_000) + .Json() + .BuildAsync(); + + Assert.Equal(50_000, stream.Options.MaxInflightRecords); + Assert.False(stream.Options.Recovery); + Assert.Equal(60_000, stream.Options.FlushTimeoutMs); + } + + // ================================================================ + // Concurrent Ingestion (Thread Safety) + // ================================================================ + + [Fact] + public async Task ConcurrentIngest_MultipleThreads_NoDataCorruption() + { + if (!NativeAvailable) return; + + using var sdk = CreateSdk(); + using var stream = await BuildJsonStream(sdk); + + var offsets = new long[100]; + Parallel.For(0, 100, i => + { + offsets[i] = stream.IngestRecord($"{{\"thread\": {i}}}"); + }); + + stream.Flush(); + + Assert.All(offsets, o => Assert.True(o >= 0)); + } + + // ================================================================ + // Helpers + // ================================================================ + + // Helper: create SDK with TLS disabled (mock server uses plain HTTP/2) + private ZerobusSdk CreateSdk() => + ZerobusSdk.CreateBuilder(_server.Endpoint, _server.UnityCatalogEndpoint) + .DisableTls() + .Build(); + + private Task BuildJsonStream(ZerobusSdk sdk) => + sdk.StreamBuilder() + .Table("catalog.schema.table") + .OAuth("client-id", "client-secret") + .Json() + .BuildAsync(); + + private Task> BuildProtoStream(ZerobusSdk sdk) => + sdk.StreamBuilder() + .Table("catalog.schema.table") + .OAuth("client-id", "client-secret") + .CompiledProto(Array.Empty()) + .BuildAsync(); +} + +/// +/// Minimal protobuf message for integration tests. +/// +public sealed class FakeProtoMessage : Google.Protobuf.IMessage +{ + public int Id { get; set; } + public string Name { get; set; } = ""; + + Google.Protobuf.Reflection.MessageDescriptor Google.Protobuf.IMessage.Descriptor => + throw new NotSupportedException("Fake proto — not a real compiled message"); + public int CalculateSize() => 4 + Google.Protobuf.CodedOutputStream.ComputeStringSize(Name); + public FakeProtoMessage Clone() => (FakeProtoMessage)MemberwiseClone(); + public bool Equals(FakeProtoMessage? other) => + other is not null && Id == other.Id && Name == other.Name; + public void MergeFrom(FakeProtoMessage message) { Id = message.Id; Name = message.Name; } + public void MergeFrom(Google.Protobuf.CodedInputStream input) + { + uint tag; + while ((tag = input.ReadTag()) != 0) + { + switch (tag) + { + case 8: Id = input.ReadInt32(); break; + case 18: Name = input.ReadString(); break; + default: input.SkipLastField(); break; + } + } + } + public void WriteTo(Google.Protobuf.CodedOutputStream output) + { + output.WriteTag(1, Google.Protobuf.WireFormat.WireType.Varint); + output.WriteInt32(Id); + output.WriteTag(2, Google.Protobuf.WireFormat.WireType.LengthDelimited); + output.WriteString(Name); + } + public Google.Protobuf.MessageParser Parser => + new(() => new FakeProtoMessage()); +} diff --git a/dotnet/tests/Databricks.Zerobus.IntegrationTests/MockServerSelfTest.cs b/dotnet/tests/Databricks.Zerobus.IntegrationTests/MockServerSelfTest.cs new file mode 100644 index 00000000..31a394b0 --- /dev/null +++ b/dotnet/tests/Databricks.Zerobus.IntegrationTests/MockServerSelfTest.cs @@ -0,0 +1,177 @@ +using Grpc.Core; +using Grpc.Net.Client; +using Xunit; + +namespace Databricks.Zerobus.IntegrationTests; + +/// +/// Tests the mock gRPC server standalone — no Zerobus SDK dependency. +/// Proves the mock infrastructure works before plugging in native libs. +/// +public class MockServerSelfTest : IAsyncLifetime +{ + private readonly MockZerobusServer _server = new(); + + public async Task InitializeAsync() => await _server.StartAsync(); + public Task DisposeAsync() => _server.DisposeAsync().AsTask(); + + [Fact] + public async Task ServerStartsAndAcceptsConnection() + { + using var channel = GrpcChannel.ForAddress(_server.Endpoint); + var client = new Zerobus.ZerobusClient(channel); + + var call = client.EphemeralStream(); + + // Send create stream request + await call.RequestStream.WriteAsync(new EphemeralStreamRequest + { + CreateStream = new CreateIngestStreamRequest + { + TableName = "test.table", + RecordType = RecordType.Json + } + }); + await call.RequestStream.CompleteAsync(); + + // Read response + var responses = new List(); + await foreach (var resp in call.ResponseStream.ReadAllAsync()) + { + responses.Add(resp); + } + + Assert.NotEmpty(responses); + Assert.NotNull(responses[0].CreateStreamResponse?.StreamId); + } + + [Fact] + public async Task IngestJsonRecord_ReceivesAck() + { + using var channel = GrpcChannel.ForAddress(_server.Endpoint); + var client = new Zerobus.ZerobusClient(channel); + var call = client.EphemeralStream(); + + // Create stream + await call.RequestStream.WriteAsync(new EphemeralStreamRequest + { + CreateStream = new CreateIngestStreamRequest + { + TableName = "test.table", + RecordType = RecordType.Json + } + }); + + // Ingest a record + await call.RequestStream.WriteAsync(new EphemeralStreamRequest + { + IngestRecord = new IngestRecordRequest + { + OffsetId = 42, + JsonRecord = "{\"key\":\"value\"}" + } + }); + + await call.RequestStream.CompleteAsync(); + + var responses = new List(); + await foreach (var resp in call.ResponseStream.ReadAllAsync()) + { + responses.Add(resp); + } + + // Should have: create_stream_response + ingest_record_response (ack) + Assert.True(responses.Count >= 2, + $"Expected >=2 responses, got {responses.Count}"); + Assert.NotNull(responses[0].CreateStreamResponse?.StreamId); + Assert.Equal(42, responses[1].IngestRecordResponse?.DurabilityAckUpToOffset); + } + + [Fact] + public async Task MockTracksIngestedRecords() + { + _server.Service.ShouldAckRecords = true; + + using var channel = GrpcChannel.ForAddress(_server.Endpoint); + var client = new Zerobus.ZerobusClient(channel); + var call = client.EphemeralStream(); + + await call.RequestStream.WriteAsync(new EphemeralStreamRequest + { + CreateStream = new CreateIngestStreamRequest + { + TableName = "test.table", + RecordType = RecordType.Proto + } + }); + + for (long i = 1; i <= 3; i++) + { + await call.RequestStream.WriteAsync(new EphemeralStreamRequest + { + IngestRecord = new IngestRecordRequest + { + OffsetId = i, + ProtoEncodedRecord = Google.Protobuf.ByteString.CopyFrom(new byte[] { (byte)i }) + } + }); + } + + await call.RequestStream.CompleteAsync(); + + // Drain all responses. With ShouldAckRecords=true, we get ack for each record. + var acks = new List(); + await foreach (var resp in call.ResponseStream.ReadAllAsync()) + { + if (resp.IngestRecordResponse != null) + acks.Add(resp.IngestRecordResponse.DurabilityAckUpToOffset); + } + + // We should have received acks for the ingested records + Assert.NotEmpty(acks); + Assert.Equal(3, acks.Count); + Assert.Equal(1, acks[0]); + Assert.Equal(3, acks[2]); + } + + [Fact] + public async Task BatchIngestion_ReceivesSingleAck() + { + _server.Service.ShouldAckRecords = true; + + using var channel = GrpcChannel.ForAddress(_server.Endpoint); + var client = new Zerobus.ZerobusClient(channel); + var call = client.EphemeralStream(); + + await call.RequestStream.WriteAsync(new EphemeralStreamRequest + { + CreateStream = new CreateIngestStreamRequest + { + TableName = "test.table", + RecordType = RecordType.Json + } + }); + + await call.RequestStream.WriteAsync(new EphemeralStreamRequest + { + IngestRecordBatch = new IngestRecordBatchRequest + { + OffsetId = 100, + JsonBatch = new JsonRecordBatch + { + Records = { "{\"a\":1}", "{\"b\":2}", "{\"c\":3}" } + } + } + }); + + await call.RequestStream.CompleteAsync(); + + var responses = new List(); + await foreach (var resp in call.ResponseStream.ReadAllAsync()) + responses.Add(resp); + + // Should receive: create_stream_response + batch ack + Assert.True(responses.Count >= 2); + Assert.Equal(100, responses[1].IngestRecordResponse?.DurabilityAckUpToOffset); + } +} diff --git a/dotnet/tests/Databricks.Zerobus.IntegrationTests/MockZerobusServer.cs b/dotnet/tests/Databricks.Zerobus.IntegrationTests/MockZerobusServer.cs new file mode 100644 index 00000000..3f2e7831 --- /dev/null +++ b/dotnet/tests/Databricks.Zerobus.IntegrationTests/MockZerobusServer.cs @@ -0,0 +1,109 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Server.Kestrel.Core; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; + +namespace Databricks.Zerobus.IntegrationTests; + +/// +/// Manages a real gRPC server that simulates the Zerobus ingestion service. +/// Each test gets its own server on a unique port — fully parallelizable. +/// +public sealed class MockZerobusServer : IAsyncDisposable +{ + private IHost? _host; + private int _port; + + /// + /// The mock service implementation. Configure behavior via its properties + /// before starting the server. + /// + public MockZerobusService Service { get; } = new(); + + /// + /// The gRPC endpoint for this server (http://localhost:{port}). + /// + public string Endpoint => $"http://localhost:{_port}"; + + /// + /// The simulated Unity Catalog endpoint. + /// + public string UnityCatalogEndpoint => $"http://localhost:{_port}/api/2.1/unity-catalog"; + + /// + /// Starts the gRPC server on a random available port. + /// + public async Task StartAsync() + { + _port = AllocatePort(); + + _host = Host.CreateDefaultBuilder() + .ConfigureWebHostDefaults(webBuilder => + { + webBuilder.UseUrls($"http://localhost:{_port}"); + webBuilder.ConfigureKestrel(opts => + { + opts.ListenLocalhost(_port, o => o.Protocols = HttpProtocols.Http2); + }); + webBuilder.ConfigureServices(services => + { + services.AddGrpc(); + }); + webBuilder.Configure(app => + { + app.UseRouting(); + app.UseEndpoints(endpoints => + { + endpoints.MapGrpcService(); + }); + }); + }) + .Build(); + + await _host.StartAsync(); + } + + /// + /// Stops the server gracefully. + /// + public async Task StopAsync() + { + if (_host != null) + { + await _host.StopAsync(); + _host.Dispose(); + _host = null; + } + } + + /// + /// Resets all state and tracking for a new test scenario. + /// + public void Reset() + { + Service.ShouldAcceptStream = true; + Service.ShouldAckRecords = true; + Service.AckDelayMs = 0; + Service.FailAfterNRecords = null; + Service.ErrorMessage = null; + Service.SimulateDisconnect = false; + Service.IngestedRecords.Clear(); + Service.AcknowledgedOffsets.Clear(); + } + + public async ValueTask DisposeAsync() + { + await StopAsync(); + } + + private static int AllocatePort() + { + using var socket = new System.Net.Sockets.Socket( + System.Net.Sockets.AddressFamily.InterNetwork, + System.Net.Sockets.SocketType.Stream, + System.Net.Sockets.ProtocolType.Tcp); + socket.Bind(new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 0)); + return ((System.Net.IPEndPoint)socket.LocalEndPoint!).Port; + } +} diff --git a/dotnet/tests/Databricks.Zerobus.IntegrationTests/MockZerobusService.cs b/dotnet/tests/Databricks.Zerobus.IntegrationTests/MockZerobusService.cs new file mode 100644 index 00000000..51424df3 --- /dev/null +++ b/dotnet/tests/Databricks.Zerobus.IntegrationTests/MockZerobusService.cs @@ -0,0 +1,202 @@ +using Databricks.Zerobus; +using Grpc.Core; + +namespace Databricks.Zerobus.IntegrationTests; + +/// +/// Real gRPC service implementation that simulates the Zerobus ingestion server. +/// Implements EphemeralStream — a bidirectional streaming RPC. +/// +public class MockZerobusService : Zerobus.ZerobusBase +{ + // Injectable behavior for test scenarios + public bool ShouldAcceptStream { get; set; } = true; + public bool ShouldAckRecords { get; set; } = true; + public int AckDelayMs { get; set; } + public int? FailAfterNRecords { get; set; } + public string? ErrorMessage { get; set; } + public bool SimulateDisconnect { get; set; } + + // Tracking for assertions + public List IngestedRecords { get; } = new(); + public List AcknowledgedOffsets { get; } = new(); + public string? LastTableName { get; private set; } + public byte[]? LastDescriptorProto { get; private set; } + public RecordType? LastRecordType { get; private set; } + private int _streamCount; + public int StreamCount => _streamCount; + + public override async Task EphemeralStream( + IAsyncStreamReader requestStream, + IServerStreamWriter responseStream, + ServerCallContext context) + { + Interlocked.Increment(ref _streamCount); + + long lastAckedOffset = -1; + bool streamCreated = false; + + try + { + await foreach (var request in requestStream.ReadAllAsync(context.CancellationToken)) + { + switch (request.PayloadCase) + { + case EphemeralStreamRequest.PayloadOneofCase.CreateStream: + await HandleCreateStream(request.CreateStream, responseStream, context); + streamCreated = true; + break; + + case EphemeralStreamRequest.PayloadOneofCase.IngestRecord: + if (!streamCreated) throw new InvalidOperationException("Ingest before stream creation"); + lastAckedOffset = HandleIngestRecord(request.IngestRecord); + if (ShouldAckRecords) + await SendAck(responseStream, lastAckedOffset, context); + break; + + case EphemeralStreamRequest.PayloadOneofCase.IngestRecordBatch: + if (!streamCreated) throw new InvalidOperationException("Ingest batch before stream creation"); + lastAckedOffset = HandleIngestBatch(request.IngestRecordBatch); + if (ShouldAckRecords) + await SendAck(responseStream, lastAckedOffset, context); + break; + + case EphemeralStreamRequest.PayloadOneofCase.None: + break; + } + + if (SimulateDisconnect) + { + throw new IOException("Simulated network disconnect"); + } + + if (FailAfterNRecords.HasValue && IngestedRecords.Count >= FailAfterNRecords.Value) + { + throw new RpcException(new Status(StatusCode.Internal, + ErrorMessage ?? "Simulated server error")); + } + } + } + catch (IOException) + { + // Simulate disconnect — don't send close signal + throw; + } + } + + private async Task HandleCreateStream( + CreateIngestStreamRequest request, + IServerStreamWriter responseStream, + ServerCallContext context) + { + LastTableName = request.TableName; + LastDescriptorProto = request.DescriptorProto?.ToByteArray(); + LastRecordType = request.RecordType; + + if (!ShouldAcceptStream) + { + throw new RpcException(new Status(StatusCode.PermissionDenied, + ErrorMessage ?? "Stream rejected by mock")); + } + + var response = new EphemeralStreamResponse + { + CreateStreamResponse = new CreateIngestStreamResponse + { + StreamId = Guid.NewGuid().ToString() + } + }; + + await responseStream.WriteAsync(response); + } + + private long HandleIngestRecord(IngestRecordRequest request) + { + var record = new IngestedRecord + { + OffsetId = request.OffsetId, + IsJson = request.RecordCase == IngestRecordRequest.RecordOneofCase.JsonRecord, + Data = request.RecordCase switch + { + IngestRecordRequest.RecordOneofCase.ProtoEncodedRecord => + request.ProtoEncodedRecord.ToByteArray(), + IngestRecordRequest.RecordOneofCase.JsonRecord => + System.Text.Encoding.UTF8.GetBytes(request.JsonRecord), + _ => Array.Empty() + }, + Timestamp = DateTimeOffset.UtcNow + }; + + IngestedRecords.Add(record); + + // Simulate processing delay + if (AckDelayMs > 0) + Thread.Sleep(AckDelayMs); + + return request.OffsetId; + } + + private long HandleIngestBatch(IngestRecordBatchRequest request) + { + AcknowledgedOffsets.Add(request.OffsetId); + + switch (request.BatchCase) + { + case IngestRecordBatchRequest.BatchOneofCase.ProtoEncodedBatch: + foreach (var bytes in request.ProtoEncodedBatch.Records) + { + IngestedRecords.Add(new IngestedRecord + { + OffsetId = request.OffsetId, + IsJson = false, + Data = bytes.ToByteArray(), + Timestamp = DateTimeOffset.UtcNow + }); + } + break; + + case IngestRecordBatchRequest.BatchOneofCase.JsonBatch: + foreach (var json in request.JsonBatch.Records) + { + IngestedRecords.Add(new IngestedRecord + { + OffsetId = request.OffsetId, + IsJson = true, + Data = System.Text.Encoding.UTF8.GetBytes(json), + Timestamp = DateTimeOffset.UtcNow + }); + } + break; + } + + return request.OffsetId; + } + + private async Task SendAck( + IServerStreamWriter writer, + long offset, + ServerCallContext context) + { + AcknowledgedOffsets.Add(offset); + + var response = new EphemeralStreamResponse + { + IngestRecordResponse = new IngestRecordResponse + { + DurabilityAckUpToOffset = offset + } + }; + await writer.WriteAsync(response); + } +} + +/// +/// Tracked record from mock gRPC ingestion. +/// +public sealed class IngestedRecord +{ + public long OffsetId { get; set; } + public bool IsJson { get; set; } + public byte[] Data { get; set; } = Array.Empty(); + public DateTimeOffset Timestamp { get; set; } = DateTimeOffset.UtcNow; +} diff --git a/dotnet/tests/Databricks.Zerobus.IntegrationTests/NativeLibraryHelper.cs b/dotnet/tests/Databricks.Zerobus.IntegrationTests/NativeLibraryHelper.cs new file mode 100644 index 00000000..90559c14 --- /dev/null +++ b/dotnet/tests/Databricks.Zerobus.IntegrationTests/NativeLibraryHelper.cs @@ -0,0 +1,52 @@ +using System.Runtime.InteropServices; + +namespace Databricks.Zerobus.IntegrationTests; + +internal static class NativeLibraryHelper +{ + private static bool? _isAvailable; + + public static bool IsNativeLibraryAvailable() + { + if (_isAvailable.HasValue) return _isAvailable.Value; + + try + { + string libName = GetLibraryFileName(); + string rid = GetRuntimeIdentifier(); + string baseDir = AppContext.BaseDirectory; + + // Check runtimes/ next to assembly + if (File.Exists(Path.Combine(baseDir, "runtimes", rid, "native", libName))) + { _isAvailable = true; return true; } + if (File.Exists(Path.Combine(baseDir, libName))) + { _isAvailable = true; return true; } + + // Walk up to find monorepo dotnet/src/Databricks.Zerobus/runtimes/ + var dir = new DirectoryInfo(baseDir); + while (dir?.Parent != null) + { + var runtimes = Path.Combine(dir.FullName, "src", "Databricks.Zerobus", "runtimes", rid, "native", libName); + if (File.Exists(runtimes)) + { _isAvailable = true; return true; } + dir = dir.Parent; + } + + _isAvailable = false; return false; + } + catch { _isAvailable = false; return false; } + } + + private static string GetLibraryFileName() => + RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "zerobus_ffi.dll" + : RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? "libzerobus_ffi.dylib" + : "libzerobus_ffi.so"; + + private static string GetRuntimeIdentifier() + { + string os = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "win" + : RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? "osx" : "linux"; + string arch = RuntimeInformation.ProcessArchitecture == Architecture.Arm64 ? "arm64" : "x64"; + return $"{os}-{arch}"; + } +} diff --git a/dotnet/tests/Databricks.Zerobus.IntegrationTests/NativeLoadSmokeTest.cs b/dotnet/tests/Databricks.Zerobus.IntegrationTests/NativeLoadSmokeTest.cs new file mode 100644 index 00000000..c53ee446 --- /dev/null +++ b/dotnet/tests/Databricks.Zerobus.IntegrationTests/NativeLoadSmokeTest.cs @@ -0,0 +1,90 @@ +using System.Runtime.InteropServices; +using Databricks.Zerobus.Native; +using Xunit; + +namespace Databricks.Zerobus.IntegrationTests; + +/// +/// Minimal smoke test: just loads the native DLL and calls basic FFI functions. +/// No gRPC, no mock server — just P/Invoke round-trips. +/// +public class NativeLoadSmokeTest +{ + [Fact] + public void LoadLibrary_Succeeds() + { + if (!NativeLibraryHelper.IsNativeLibraryAvailable()) + return; + + // Just loading the library shouldn't crash + NativeLibraryResolver.EnsureLoaded(); + Assert.True(true); // survived + } + + [Fact] + public void BuilderNewFree_Succeeds() + { + if (!NativeLibraryHelper.IsNativeLibraryAvailable()) + return; + + NativeLibraryResolver.EnsureLoaded(); + + IntPtr builder = NativeMethods.zerobus_sdk_builder_new(); + Assert.NotEqual(IntPtr.Zero, builder); + + NativeMethods.zerobus_sdk_builder_free(builder); + // No crash = pass + } + + [Fact] + public void GetDefaultConfig_ReturnsValidStruct() + { + if (!NativeLibraryHelper.IsNativeLibraryAvailable()) + return; + + NativeLibraryResolver.EnsureLoaded(); + + var config = NativeMethods.zerobus_get_default_config(); + Assert.True(config.MaxInflightRequests > 0); + } + + [Fact] + public void BuildSdk_WithTlsDisabled_Succeeds() + { + if (!NativeLibraryHelper.IsNativeLibraryAvailable()) + return; + + NativeLibraryResolver.EnsureLoaded(); + + IntPtr builder = NativeMethods.zerobus_sdk_builder_new(); + Assert.NotEqual(IntPtr.Zero, builder); + + // Use localhost:1 (nothing listening) to verify build() fails gracefully, not crash + var endpoint = Marshal.StringToHGlobalAnsi("http://localhost:1"); + var ucUrl = Marshal.StringToHGlobalAnsi("http://localhost:1/api/2.1/unity-catalog"); + try + { + NativeMethods.zerobus_sdk_builder_endpoint(builder, endpoint); + NativeMethods.zerobus_sdk_builder_unity_catalog_url(builder, ucUrl); + NativeMethods.zerobus_sdk_builder_disable_tls(builder); + + IntPtr sdk = NativeMethods.zerobus_sdk_builder_build(builder); + // builder is consumed by build() — if it fails, sdk is NULL (not a crash) + if (sdk != IntPtr.Zero) + NativeMethods.zerobus_sdk_free(sdk); + + // If we got here without AccessViolation, the native code handled the error gracefully + } + finally + { + Marshal.FreeHGlobal(endpoint); + Marshal.FreeHGlobal(ucUrl); + } + } + + // NOTE: builder_build() crashes (AccessViolation) when connecting to the + // mock Kestrel gRPC server, even with TLS disabled. The smoke test above + // against localhost:1 (nothing listening) proves the FFI layer works. + // This is a DLL version compatibility issue — the native library and the + // mock server need to be built from matching zerobus_service.proto versions. +} diff --git a/dotnet/tests/Databricks.Zerobus.IntegrationTests/Protos/zerobus_service.proto b/dotnet/tests/Databricks.Zerobus.IntegrationTests/Protos/zerobus_service.proto new file mode 100644 index 00000000..69e33a22 --- /dev/null +++ b/dotnet/tests/Databricks.Zerobus.IntegrationTests/Protos/zerobus_service.proto @@ -0,0 +1,234 @@ +syntax = "proto2"; + +import "google/protobuf/duration.proto"; + +package databricks.zerobus; + +// Canonical Zerobus gRPC service schema — the single source of truth for every +// SDK's generated bindings: the Rust core, the Java SDK, the cgo Go SDK tests, +// and the pure-Go SDK (purego). The Java options below apply at codegen. The +// two Go consumers target different import paths, so neither a go_package is set +// here — each supplies its own via a protoc `M` override at generation time +// (see go/tests/generate_proto.sh and purego/internal/zerobuspb/gen.go). +option java_package = "com.databricks.zerobus"; +option java_outer_classname = "ZerobusProto"; +option java_multiple_files = true; +option csharp_namespace = "Databricks.Zerobus.IntegrationTests"; + +// Version information for the Zerobus service. +// Current version: v1 (package: databricks.zerobus). +// +// Version History: +// - v1: Initial release with ephemeral streams support. +// +// API Stability: STABLE. +// This API is considered stable and backward-compatible within major versions. +// Breaking changes will be introduced in new major versions. +// +// Versioning Strategy: +// - Major versions: New endpoints or package names (e.g., databricks.zerobus.v2). +// - Minor versions: Backward-compatible additions within same package. +// - Patch versions: Bug fixes and non-breaking changes. + +/* + * The Zerobus service provides streaming capabilities for ingesting data records + * into Databricks tables with high throughput and low latency. + */ +service Zerobus { + /* + * EphemeralStream creates a streaming session for ingesting records. + * + * This is a bidirectional streaming RPC that allows clients to: + * - Create new ephemeral ingestion streams by sending a CreateIngestStreamRequest. + * - Ingest records by sending an IngestRecordRequest. + * - Receive durability confirmations via IngestRecordResponse. + */ + rpc EphemeralStream(stream EphemeralStreamRequest) returns (stream EphemeralStreamResponse); +} + +/* + * Record type that will be accepted in the stream. + * + * Defaults to RECORD_TYPE_UNSPECIFIED, which returns an error on stream creation. + */ +enum RecordType { + RECORD_TYPE_UNSPECIFIED = 0; + PROTO = 1; + JSON = 2; +} + +/* + * Batch of JSON-encoded records. + * + * This message contains multiple JSON records that will be ingested together. + * Each string in the array represents a complete JSON object. + */ +message JsonRecordBatch { + // Array of JSON-encoded records. + repeated string records = 1; +} + +/* + * Batch of protobuf-encoded records. + * + * This message contains multiple protobuf-encoded records that will be ingested together. + * Each record must be serialized according to the protobuf descriptor provided in the + * CreateIngestStreamRequest. + */ +message ProtoEncodedRecordBatch { + // Array of protobuf-encoded records. + repeated bytes records = 1; +} + +/* + * Request to create a new ephemeral ingestion stream. + * + * This message initiates the streaming session and must be the first message + * sent by the client in the EphemeralStream RPC. + */ +message CreateIngestStreamRequest { + // Three part name of the target destination table for data ingestion. + // + // This is a required field for all stream creation requests. + optional string table_name = 1; + + // NOT SUPPORTED: Stream identifier for opening existing persisted streams. + // optional string stream_id = 2; + reserved "stream_id"; + reserved 2; + + // Protocol buffer descriptor for record serialization/deserialization. + // + // This descriptor defines the structure of the records being ingested. + // It must be compatible with the target table's schema. + // + // This is a required field for all stream creation requests. + optional bytes descriptor_proto = 3; + + // Record type that will be accepted in the stream. + // Defaults to PROTO for backwards compatibility. + optional RecordType record_type = 4; +} + +/* + * Response confirming the creation of an ephemeral ingestion stream. + * + * This message is sent by the server in response to a CreateIngestStreamRequest + * and contains the stream identifier and initial offset information. + */ +message CreateIngestStreamResponse { + // Unique identifier assigned to this ephemeral ingestion stream. + optional string stream_id = 1; + + // NOT SUPPORTED: Last acknowledged offset for this stream. + // optional int64 last_offset_id = 2; + reserved "last_offset_id"; + reserved 2; +} + +/* + * Request to ingest a single record into the stream. + * + * This message is sent by the client after the initial CreateIngestStreamRequest + * to stream individual records for ingestion. + */ +message IngestRecordRequest { + // Unique identifier for this record within the stream. + optional int64 offset_id = 1; + + // Serialized record data. + oneof record { + // The proto encoded record must be serialized according to the protobuf descriptor + // provided in the CreateIngestStreamRequest. + bytes proto_encoded_record = 2; + string json_record = 3; + } +} + +/* + * Request to ingest a batch of records into the stream. + * + * This message is sent by the client after the initial CreateIngestStreamRequest + * to stream batches of records for ingestion. + */ +message IngestRecordBatchRequest { + // Unique identifier for this batch within the stream. + optional int64 offset_id = 1; + + // Batch of serialized records. + // The batch can contain multiple records encoded as either protobuf or JSON. + oneof batch { + // Batch of protobuf-encoded records. Each record must be serialized according to + // the protobuf descriptor provided in the CreateIngestStreamRequest. + ProtoEncodedRecordBatch proto_encoded_batch = 2; + + // Batch of JSON-encoded records. + JsonRecordBatch json_batch = 3; + } +} + +/* + * A message in the EphemeralStream bidirectional stream. + * + * This message type allows the client to send either stream creation requests + * or record ingestion requests (individual or batched) through the same stream. + */ +message EphemeralStreamRequest { + oneof payload { + // Initial request to create an ephemeral stream. + // Must be the first message in the stream. + // All subsequent messages should be ingest_record or ingest_record_batch. + CreateIngestStreamRequest create_stream = 1; + + // Request to ingest a record. + // Can only be sent after a successful create_stream request. + // Multiple ingest_record messages can be sent in sequence. + IngestRecordRequest ingest_record = 2; + + // Request to ingest a batch of records. + // Can only be sent after a successful create_stream request. + // Multiple ingest_record_batch messages can be sent in sequence. + IngestRecordBatchRequest ingest_record_batch = 3; + } +} + +/* + * Acknowledgment for all records up to the specified offset. + * + * This message is sent by the server to confirm that records have been + * successfully ingested and are durable. + */ +message IngestRecordResponse { + // Highest offset that has been durably acknowledged. + // + // This offset indicates that all records with + // offset_id <= durability_ack_up_to_offset have been made durable. + optional int64 durability_ack_up_to_offset = 1; +} + +/* + * Signal that the server will close the stream after the specified duration. + */ +message CloseStreamSignal { + // Duration after which the server will close the stream. + optional google.protobuf.Duration duration = 1; +} + +/* + * A message in the EphemeralStream response stream. + * + * This message type allows the server to send either stream creation responses + * or record ingestion responses through the same stream. + */ +message EphemeralStreamResponse { + oneof payload { + // Response to a create_stream request. + CreateIngestStreamResponse create_stream_response = 1; + + // Response to ingest_record requests. + IngestRecordResponse ingest_record_response = 2; + + // Signal that the server will close the stream after the specified duration. + CloseStreamSignal close_stream_signal = 3; + } +} diff --git a/dotnet/tests/Databricks.Zerobus.Tests/ArrowStreamConfigurationOptionsTests.cs b/dotnet/tests/Databricks.Zerobus.Tests/ArrowStreamConfigurationOptionsTests.cs new file mode 100644 index 00000000..6a1fd191 --- /dev/null +++ b/dotnet/tests/Databricks.Zerobus.Tests/ArrowStreamConfigurationOptionsTests.cs @@ -0,0 +1,53 @@ +using Xunit; + +namespace Databricks.Zerobus.Tests; + +public class ArrowStreamConfigurationOptionsTests +{ + [Fact] + public void Default_ReturnsOptionsWithSensibleDefaults() + { + var opts = ArrowStreamConfigurationOptions.Default; + + Assert.Equal(10_000, opts.MaxInflightBatches); + Assert.True(opts.Recovery); + Assert.Equal(30_000, opts.ConnectionTimeoutMs); + Assert.Equal(IPCCompressionType.None, opts.IpcCompression); + } + + [Fact] + public void Builder_CanOverrideIndividualValues() + { + var opts = ArrowStreamConfigurationOptions.NewBuilder() + .SetMaxInflightBatches(5_000) + .SetRecovery(false) + .SetConnectionTimeoutMs(60_000) + .SetIpcCompression(IPCCompressionType.Zstd) + .SetStreamPausedMaxWaitTimeMs(-1) + .Build(); + + Assert.Equal(5_000, opts.MaxInflightBatches); + Assert.False(opts.Recovery); + Assert.Equal(60_000, opts.ConnectionTimeoutMs); + Assert.Equal(IPCCompressionType.Zstd, opts.IpcCompression); + Assert.Equal(-1, opts.StreamPausedMaxWaitTimeMs); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void Builder_SetMaxInflightBatches_RejectsNonPositive(int value) + { + Assert.Throws(() => + ArrowStreamConfigurationOptions.NewBuilder().SetMaxInflightBatches(value)); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void Builder_SetConnectionTimeoutMs_RejectsNonPositive(int value) + { + Assert.Throws(() => + ArrowStreamConfigurationOptions.NewBuilder().SetConnectionTimeoutMs(value)); + } +} diff --git a/dotnet/tests/Databricks.Zerobus.Tests/Databricks.Zerobus.Tests.csproj b/dotnet/tests/Databricks.Zerobus.Tests/Databricks.Zerobus.Tests.csproj new file mode 100644 index 00000000..b7fc177d --- /dev/null +++ b/dotnet/tests/Databricks.Zerobus.Tests/Databricks.Zerobus.Tests.csproj @@ -0,0 +1,24 @@ + + + + net8.0 + Databricks.Zerobus.Tests + false + true + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + diff --git a/dotnet/tests/Databricks.Zerobus.Tests/ExceptionTests.cs b/dotnet/tests/Databricks.Zerobus.Tests/ExceptionTests.cs new file mode 100644 index 00000000..d679fc07 --- /dev/null +++ b/dotnet/tests/Databricks.Zerobus.Tests/ExceptionTests.cs @@ -0,0 +1,44 @@ +using Xunit; + +namespace Databricks.Zerobus.Tests; + +public class ExceptionTests +{ + [Fact] + public void ZerobusException_StoresMessage() + { + var ex = new ZerobusException("test message"); + Assert.Equal("test message", ex.Message); + Assert.False(ex.IsRetryable); + } + + [Fact] + public void ZerobusException_StoresRetryableFlag() + { + var ex = new ZerobusException("retryable error", isRetryable: true); + Assert.True(ex.IsRetryable); + } + + [Fact] + public void ZerobusException_StoresInnerException() + { + var inner = new InvalidOperationException("inner"); + var ex = new ZerobusException("outer", isRetryable: true, innerException: inner); + Assert.Same(inner, ex.InnerException); + Assert.True(ex.IsRetryable); + } + + [Fact] + public void NonRetriableException_AlwaysHasRetryableFalse() + { + var ex = new NonRetriableException("non-retryable"); + Assert.False(ex.IsRetryable); + } + + [Fact] + public void NonRetriableException_InheritsFromZerobusException() + { + var ex = new NonRetriableException("test"); + Assert.IsAssignableFrom(ex); + } +} diff --git a/dotnet/tests/Databricks.Zerobus.Tests/StreamBuilderTests.cs b/dotnet/tests/Databricks.Zerobus.Tests/StreamBuilderTests.cs new file mode 100644 index 00000000..597cd2a8 --- /dev/null +++ b/dotnet/tests/Databricks.Zerobus.Tests/StreamBuilderTests.cs @@ -0,0 +1,74 @@ +using Xunit; + +namespace Databricks.Zerobus.Tests; + +public class StreamBuilderTests +{ + [Fact] + public void Table_RequiresNonBlankValue() + { + Assert.Throws(() => new StreamBuilder(null!).Table("")); + Assert.Throws(() => new StreamBuilder(null!).Table(" ")); + } + + [Fact] + public void OAuth_RequiresNonBlankValues() + { + Assert.Throws(() => new StreamBuilder(null!).OAuth("", "secret")); + Assert.Throws(() => new StreamBuilder(null!).OAuth("id", "")); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void MaxInflightRecords_RejectsNonPositive(int value) + { + Assert.Throws(() => new StreamBuilder(null!).MaxInflightRecords(value)); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void FlushTimeoutMs_RejectsNonPositive(int value) + { + Assert.Throws(() => new StreamBuilder(null!).FlushTimeoutMs(value)); + } + + [Fact] + public void TableProperties_StoresValues() + { + var props = new TableProperties( + "catalog.schema.table", + new List + { + new("id", "BIGINT", false, "Primary key"), + new("name", "STRING", true) + }); + + Assert.Equal("catalog.schema.table", props.TableName); + Assert.Equal(2, props.Columns.Count); + Assert.Equal("id", props.Columns[0].Name); + Assert.Equal("BIGINT", props.Columns[0].TypeName); + Assert.False(props.Columns[0].Nullable); + Assert.Equal("Primary key", props.Columns[0].Comment); + } + + [Fact] + public void EncodedBatch_StoresData() + { + var data = new byte[] { 1, 2, 3 }; + var lengths = new[] { 3 }; + var batch = new EncodedBatch(data, lengths); + + Assert.Same(data, batch.Data); + Assert.Same(lengths, batch.Lengths); + } + + [Fact] + public void IPCCompressionType_Values() + { + Assert.Equal(-1, (int)IPCCompressionType.None); + Assert.Equal(0, (int)IPCCompressionType.Lz4Frame); + Assert.Equal(1, (int)IPCCompressionType.Zstd); + } +} diff --git a/dotnet/tests/Databricks.Zerobus.Tests/StreamConfigurationOptionsTests.cs b/dotnet/tests/Databricks.Zerobus.Tests/StreamConfigurationOptionsTests.cs new file mode 100644 index 00000000..bbfded91 --- /dev/null +++ b/dotnet/tests/Databricks.Zerobus.Tests/StreamConfigurationOptionsTests.cs @@ -0,0 +1,106 @@ +using Xunit; + +namespace Databricks.Zerobus.Tests; + +public class StreamConfigurationOptionsTests +{ + [Fact] + public void Default_ReturnsOptionsWithSensibleDefaults() + { + var opts = StreamConfigurationOptions.Default; + + Assert.Equal(1_000_000, opts.MaxInflightRecords); + Assert.True(opts.Recovery); + Assert.Equal(15_000, opts.RecoveryTimeoutMs); + Assert.Equal(2_000, opts.RecoveryBackoffMs); + Assert.Equal(4, opts.RecoveryRetries); + Assert.Equal(300_000, opts.FlushTimeoutMs); + Assert.Equal(60_000, opts.ServerLackOfAckTimeoutMs); + Assert.Null(opts.StreamPausedMaxWaitTimeMs); + Assert.Null(opts.CallbackMaxWaitTimeMs); + Assert.Null(opts.OnAck); + Assert.Null(opts.OnError); + } + + [Fact] + public void Builder_CanOverrideIndividualValues() + { + var opts = StreamConfigurationOptions.NewBuilder() + .SetMaxInflightRecords(50_000) + .SetRecovery(false) + .SetRecoveryTimeoutMs(5000) + .SetFlushTimeoutMs(60_000) + .Build(); + + Assert.Equal(50_000, opts.MaxInflightRecords); + Assert.False(opts.Recovery); + Assert.Equal(5000, opts.RecoveryTimeoutMs); + Assert.Equal(60_000, opts.FlushTimeoutMs); + // Unset values keep defaults + Assert.Equal(2_000, opts.RecoveryBackoffMs); + } + + [Fact] + public void Builder_SetAckCallback_SetsBothDelegates() + { + AckOnAckDelegate? onAckCalled = null; + AckOnErrorDelegate? onErrorCalled = null; + + void OnAck(long id) => onAckCalled = OnAck; + void OnError(long id, string msg) => onErrorCalled = OnError; + + var opts = StreamConfigurationOptions.NewBuilder() + .SetAckCallback(OnAck, OnError) + .Build(); + + Assert.NotNull(opts.OnAck); + Assert.NotNull(opts.OnError); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void Builder_SetMaxInflightRecords_RejectsNonPositive(int value) + { + Assert.Throws(() => + StreamConfigurationOptions.NewBuilder().SetMaxInflightRecords(value)); + } + + [Theory] + [InlineData(-1)] + public void Builder_SetRecoveryTimeoutMs_RejectsNegative(int value) + { + Assert.Throws(() => + StreamConfigurationOptions.NewBuilder().SetRecoveryTimeoutMs(value)); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void Builder_SetFlushTimeoutMs_RejectsNonPositive(int value) + { + Assert.Throws(() => + StreamConfigurationOptions.NewBuilder().SetFlushTimeoutMs(value)); + } + + [Fact] + public void Builder_SetStreamPausedMaxWaitTimeMs_AcceptsNegative() + { + // Negative means "wait full server-specified duration" — allowed + var opts = StreamConfigurationOptions.NewBuilder() + .SetStreamPausedMaxWaitTimeMs(-1) + .Build(); + + Assert.Equal(-1, opts.StreamPausedMaxWaitTimeMs); + } + + [Fact] + public void Builder_SetStreamPausedMaxWaitTimeMs_NullClearsValue() + { + var opts = StreamConfigurationOptions.NewBuilder() + .SetStreamPausedMaxWaitTimeMs(null) + .Build(); + + Assert.Null(opts.StreamPausedMaxWaitTimeMs); + } +} diff --git a/dotnet/tools/GenerateProto/GenerateProto.csproj b/dotnet/tools/GenerateProto/GenerateProto.csproj new file mode 100644 index 00000000..4e187265 --- /dev/null +++ b/dotnet/tools/GenerateProto/GenerateProto.csproj @@ -0,0 +1,16 @@ + + + + Exe + net8.0 + Databricks.Zerobus.Tools.GenerateProto + generate-proto + false + + + + + + + + diff --git a/dotnet/tools/GenerateProto/Program.cs b/dotnet/tools/GenerateProto/Program.cs new file mode 100644 index 00000000..0dbd91f3 --- /dev/null +++ b/dotnet/tools/GenerateProto/Program.cs @@ -0,0 +1,56 @@ +using Databricks.Zerobus; + +// =================================================================== +// Proto Schema Generator Tool +// =================================================================== +// Generates a protobuf descriptor from a Unity Catalog table JSON +// representation. This is a thin wrapper around ProtoSchema. +// +// Usage: +// generate-proto +// +// The JSON file should contain the Unity Catalog API response +// for the table's GET endpoint. +// =================================================================== + +if (args.Length < 1) +{ + Console.Error.WriteLine("Usage: generate-proto "); + Console.Error.WriteLine(); + Console.Error.WriteLine("Generates a protobuf file descriptor from a Unity Catalog table."); + Console.Error.WriteLine("The JSON file should contain the Unity Catalog API response"); + Console.Error.WriteLine("for the table's GET endpoint."); + Environment.Exit(1); +} + +string jsonFilePath = args[0]; + +if (!File.Exists(jsonFilePath)) +{ + Console.Error.WriteLine($"File not found: {jsonFilePath}"); + Environment.Exit(1); +} + +try +{ + string ucTableJson = File.ReadAllText(jsonFilePath); + + Console.Error.WriteLine("Generating proto schema from Unity Catalog table JSON..."); + + using var schema = ProtoSchema.FromUnityCatalogJson(ucTableJson); + + byte[] descriptorBytes = schema.GetDescriptorBytes(); + + // Write the descriptor proto bytes to stdout + using var stdout = Console.OpenStandardOutput(); + stdout.Write(descriptorBytes, 0, descriptorBytes.Length); + stdout.Flush(); + + Console.Error.WriteLine($"Generated {descriptorBytes.Length} bytes of descriptor proto."); + Console.Error.WriteLine("Write this to a .desc file or use it directly with StreamBuilder.CompiledProto()."); +} +catch (ZerobusException ex) +{ + Console.Error.WriteLine($"Error generating proto schema: {ex.Message}"); + Environment.Exit(2); +} From c91373d98fd6a70e0821c82b9cccbd2564469120 Mon Sep 17 00:00:00 2001 From: efrain robles Date: Fri, 17 Jul 2026 20:46:10 -0600 Subject: [PATCH 2/3] [.NET] Fix RecordType mapping and test setup Set RecordType in native options struct (1=Proto, 2=Json) so stream creation stops rejecting JSON/Proto streams with 'Record type is not specified'. Also rework integration tests to avoid IAsyncLifetime, which triggers a .NET 8.0.28 JIT bug causing InvalidProgramException when combined with DllImport. Signed-off-by: Efrain Robles Gonzalez --- dotnet/src/Databricks.Zerobus/ZerobusSdk.cs | 5 +- .../IntegrationTests.cs | 69 ++++++++++++------- 2 files changed, 48 insertions(+), 26 deletions(-) diff --git a/dotnet/src/Databricks.Zerobus/ZerobusSdk.cs b/dotnet/src/Databricks.Zerobus/ZerobusSdk.cs index af219790..8e999ea9 100644 --- a/dotnet/src/Databricks.Zerobus/ZerobusSdk.cs +++ b/dotnet/src/Databricks.Zerobus/ZerobusSdk.cs @@ -210,10 +210,10 @@ private ZerobusProtoStream CreateProtoStreamInternal( NativeLibraryResolver.EnsureLoaded(); var cOpts = options.ToNative(); + cOpts.RecordType = 1; // RecordType::Proto per Rust FFI mapping CResult result; - - IntPtr descPtr = IntPtr.Zero; GCHandle descHandle = default; + IntPtr descPtr = IntPtr.Zero; if (descriptorProtoBytes != null && descriptorProtoBytes.Length > 0) { descHandle = GCHandle.Alloc(descriptorProtoBytes, GCHandleType.Pinned); @@ -256,6 +256,7 @@ private ZerobusJsonStream CreateJsonStreamInternal( NativeLibraryResolver.EnsureLoaded(); var cOpts = options.ToNative(); + cOpts.RecordType = 2; // RecordType::Json per Rust FFI mapping CResult result; IntPtr stream = NativeMethods.zerobus_sdk_create_stream( diff --git a/dotnet/tests/Databricks.Zerobus.IntegrationTests/IntegrationTests.cs b/dotnet/tests/Databricks.Zerobus.IntegrationTests/IntegrationTests.cs index a4b3db65..22a27dca 100644 --- a/dotnet/tests/Databricks.Zerobus.IntegrationTests/IntegrationTests.cs +++ b/dotnet/tests/Databricks.Zerobus.IntegrationTests/IntegrationTests.cs @@ -8,15 +8,20 @@ namespace Databricks.Zerobus.IntegrationTests; /// Each test runs against its own server on a unique port. /// Tests auto-skip when the native library (zerobus_ffi) is not available. /// -[Collection("Integration")] +// [Collection("Integration")] [Trait("Category", "Integration")] -public class IntegrationTests : IAsyncLifetime +public class IntegrationTests { - private readonly MockZerobusServer _server = new(); + private MockZerobusServer _server = null!; private static readonly bool NativeAvailable = NativeLibraryHelper.IsNativeLibraryAvailable(); - public async Task InitializeAsync() => await _server.StartAsync(); - public Task DisposeAsync() => _server.DisposeAsync().AsTask(); + public IntegrationTests() + { + if (NativeAvailable) + { + _server = new MockZerobusServer(); + } + } // ================================================================ // Stream Creation @@ -27,7 +32,7 @@ public async Task CreateJsonStream_Success() { if (!NativeAvailable) return; - using var sdk = CreateSdk(); + using var sdk = await CreateSdkAsync(); using var stream = await BuildJsonStream(sdk); Assert.NotNull(stream); @@ -40,7 +45,7 @@ public async Task CreateProtoStream_Success() { if (!NativeAvailable) return; - using var sdk = CreateSdk(); + using var sdk = await CreateSdkAsync(); using var stream = await BuildProtoStream(sdk); Assert.NotNull(stream); @@ -55,7 +60,7 @@ public async Task IngestJsonRecord_ReturnsOffset() { if (!NativeAvailable) return; - using var sdk = CreateSdk(); + using var sdk = await CreateSdkAsync(); using var stream = await BuildJsonStream(sdk); long offset = stream.IngestRecord("{\"id\": 1, \"name\": \"test\"}"); @@ -67,7 +72,7 @@ public async Task IngestProtoRecord_ReturnsOffset() { if (!NativeAvailable) return; - using var sdk = CreateSdk(); + using var sdk = await CreateSdkAsync(); using var stream = await BuildProtoStream(sdk); var record = new FakeProtoMessage { Id = 42, Name = "test" }; @@ -84,7 +89,7 @@ public async Task IngestJsonBatch_ReturnsOffset() { if (!NativeAvailable) return; - using var sdk = CreateSdk(); + using var sdk = await CreateSdkAsync(); using var stream = await BuildJsonStream(sdk); var records = new[] { "{\"a\":1}", "{\"b\":2}", "{\"c\":3}" }; @@ -99,7 +104,7 @@ public async Task IngestProtoBatch_ReturnsOffset() { if (!NativeAvailable) return; - using var sdk = CreateSdk(); + using var sdk = await CreateSdkAsync(); using var stream = await BuildProtoStream(sdk); var records = new List @@ -123,7 +128,7 @@ public async Task Flush_AfterIngest_DoesNotThrow() { if (!NativeAvailable) return; - using var sdk = CreateSdk(); + using var sdk = await CreateSdkAsync(); using var stream = await BuildJsonStream(sdk); stream.IngestRecord("{\"x\":1}"); @@ -135,7 +140,7 @@ public async Task WaitForOffset_AfterIngest_DoesNotThrow() { if (!NativeAvailable) return; - using var sdk = CreateSdk(); + using var sdk = await CreateSdkAsync(); using var stream = await BuildJsonStream(sdk); long offset = stream.IngestRecord("{\"x\":1}"); @@ -151,7 +156,7 @@ public async Task Close_MarksStreamAsClosed() { if (!NativeAvailable) return; - using var sdk = CreateSdk(); + using var sdk = await CreateSdkAsync(); using var stream = await BuildJsonStream(sdk); Assert.False(stream.IsClosed); @@ -164,7 +169,7 @@ public async Task Dispose_MarksStreamAsClosed() { if (!NativeAvailable) return; - using var sdk = CreateSdk(); + using var sdk = await CreateSdkAsync(); using var stream = await BuildJsonStream(sdk); stream.Dispose(); @@ -176,7 +181,7 @@ public async Task IngestAfterClose_Throws() { if (!NativeAvailable) return; - using var sdk = CreateSdk(); + using var sdk = await CreateSdkAsync(); using var stream = await BuildJsonStream(sdk); stream.Close(); stream.Dispose(); @@ -193,7 +198,7 @@ public async Task GetUnackedRecords_AfterClose_ReturnsRecords() { if (!NativeAvailable) return; - using var sdk = CreateSdk(); + using var sdk = await CreateSdkAsync(); using var stream = await BuildJsonStream(sdk); stream.IngestRecord("{\"x\":1}"); @@ -215,7 +220,7 @@ public async Task RecreateStream_PreservesConfiguration() { if (!NativeAvailable) return; - using var sdk = CreateSdk(); + using var sdk = await CreateSdkAsync(); var original = await BuildJsonStream(sdk); original.Close(); @@ -241,7 +246,7 @@ public async Task StreamCreation_InvalidTable_Throws() if (!NativeAvailable) return; - using var sdk = CreateSdk(); + using var sdk = await CreateSdkAsync(); await Assert.ThrowsAsync(() => sdk.StreamBuilder() .Table("bad.table") @@ -255,7 +260,7 @@ public async Task Ingest_SdkDisposed_DoesNotCrash() { if (!NativeAvailable) return; - var sdk = CreateSdk(); + using var sdk = await CreateSdkAsync(); var stream = await BuildJsonStream(sdk); sdk.Dispose(); @@ -273,7 +278,7 @@ public async Task StreamBuilder_CustomOptions_Applied() { if (!NativeAvailable) return; - using var sdk = CreateSdk(); + using var sdk = await CreateSdkAsync(); using var stream = await sdk.StreamBuilder() .Table("catalog.schema.table") .OAuth("id", "secret") @@ -297,7 +302,7 @@ public async Task ConcurrentIngest_MultipleThreads_NoDataCorruption() { if (!NativeAvailable) return; - using var sdk = CreateSdk(); + using var sdk = await CreateSdkAsync(); using var stream = await BuildJsonStream(sdk); var offsets = new long[100]; @@ -316,10 +321,25 @@ public async Task ConcurrentIngest_MultipleThreads_NoDataCorruption() // ================================================================ // Helper: create SDK with TLS disabled (mock server uses plain HTTP/2) - private ZerobusSdk CreateSdk() => - ZerobusSdk.CreateBuilder(_server.Endpoint, _server.UnityCatalogEndpoint) + private async Task EnsureServerAsync() + { + if (_server == null) + { + _server = new MockZerobusServer(); + } + if (string.IsNullOrEmpty(_server.Endpoint) || _server.Endpoint.Contains(":0")) + { + await _server.StartAsync(); + } + } + + private async Task CreateSdkAsync() + { + await EnsureServerAsync(); + return ZerobusSdk.CreateBuilder(_server!.Endpoint, _server.UnityCatalogEndpoint) .DisableTls() .Build(); + } private Task BuildJsonStream(ZerobusSdk sdk) => sdk.StreamBuilder() @@ -374,3 +394,4 @@ public void WriteTo(Google.Protobuf.CodedOutputStream output) public Google.Protobuf.MessageParser Parser => new(() => new FakeProtoMessage()); } + From 356bc93a1f223ed2ca0ad631af1dc4551b676d9a Mon Sep 17 00:00:00 2001 From: efrain robles Date: Fri, 17 Jul 2026 21:10:02 -0600 Subject: [PATCH 3/3] [.NET] Fix configuration drift between C# defaults and native config Three discrepancies found by comparing C# hardcoded defaults against zerobus_get_default_config() / zerobus_arrow_get_default_config(): 1. CallbackMaxWaitTimeMs was null (wait forever), native default is 5000ms 2. MaxInflightBatches was 10000, native default is 1000 3. Arrow StreamPausedMaxWaitTimeMs was ulong (0), native expects int64_t with -1 meaning 'wait full server duration' Also adds Defaults_MatchNativeConfig and ArrowDefaults_MatchNativeConfig smoke tests to detect future drift. Signed-off-by: Efrain Robles Gonzalez --- .../ArrowStreamConfigurationOptions.cs | 11 ++++-- .../Native/InteropStructs.cs | 4 +-- .../StreamConfigurationOptions.cs | 7 +++- dotnet/src/Databricks.Zerobus/ZerobusSdk.cs | 2 +- .../NativeLoadSmokeTest.cs | 34 +++++++++++++++++++ .../ArrowStreamConfigurationOptionsTests.cs | 3 +- .../StreamConfigurationOptionsTests.cs | 2 +- 7 files changed, 54 insertions(+), 9 deletions(-) diff --git a/dotnet/src/Databricks.Zerobus/ArrowStreamConfigurationOptions.cs b/dotnet/src/Databricks.Zerobus/ArrowStreamConfigurationOptions.cs index e2e72305..12fe71e6 100644 --- a/dotnet/src/Databricks.Zerobus/ArrowStreamConfigurationOptions.cs +++ b/dotnet/src/Databricks.Zerobus/ArrowStreamConfigurationOptions.cs @@ -7,15 +7,20 @@ namespace Databricks.Zerobus; public sealed class ArrowStreamConfigurationOptions { /// - /// Default: 10,000 batches. + /// Default: 1,000 batches. /// - public const int DefaultMaxInflightBatches = 10_000; + public const int DefaultMaxInflightBatches = 1_000; /// /// Default: 30,000 ms (30 seconds). /// public const int DefaultConnectionTimeoutMs = 30_000; + /// + /// Default: -1 (wait full server-specified duration). + /// + public const long DefaultStreamPausedMaxWaitTimeMs = -1; + /// /// Maximum number of Arrow batches that can be in flight. /// @@ -105,7 +110,7 @@ public sealed class Builder internal int FlushTimeoutMs { get; set; } = StreamConfigurationOptions.DefaultFlushTimeoutMs; internal int ConnectionTimeoutMs { get; set; } = DefaultConnectionTimeoutMs; internal IPCCompressionType IpcCompression { get; set; } = IPCCompressionType.None; - internal long StreamPausedMaxWaitTimeMs { get; set; } + internal long StreamPausedMaxWaitTimeMs { get; set; } = DefaultStreamPausedMaxWaitTimeMs; internal Builder() { } diff --git a/dotnet/src/Databricks.Zerobus/Native/InteropStructs.cs b/dotnet/src/Databricks.Zerobus/Native/InteropStructs.cs index f3fd3d4c..51653dc8 100644 --- a/dotnet/src/Databricks.Zerobus/Native/InteropStructs.cs +++ b/dotnet/src/Databricks.Zerobus/Native/InteropStructs.cs @@ -68,7 +68,7 @@ internal struct CStreamConfigurationOptions public ulong ServerLackOfAckTimeoutMs; // uint64_t public ulong FlushTimeoutMs; // uint64_t public int RecordType; - public ulong StreamPausedMaxWaitTimeMs; // uint64_t + public ulong StreamPausedMaxWaitTimeMs; // uint64_t [MarshalAs(UnmanagedType.U1)] public bool HasStreamPausedMaxWaitTimeMs; public ulong CallbackMaxWaitTimeMs; // uint64_t @@ -92,5 +92,5 @@ internal struct CArrowStreamConfigurationOptions public ulong FlushTimeoutMs; // uint64_t public ulong ConnectionTimeoutMs; // uint64_t public int IpcCompression; // -1=None, 0=LZ4_FRAME, 1=ZSTD - public ulong StreamPausedMaxWaitTimeMs; // uint64_t + public long StreamPausedMaxWaitTimeMs; // int64_t } diff --git a/dotnet/src/Databricks.Zerobus/StreamConfigurationOptions.cs b/dotnet/src/Databricks.Zerobus/StreamConfigurationOptions.cs index 00bc3332..b319ae47 100644 --- a/dotnet/src/Databricks.Zerobus/StreamConfigurationOptions.cs +++ b/dotnet/src/Databricks.Zerobus/StreamConfigurationOptions.cs @@ -41,6 +41,11 @@ public sealed class StreamConfigurationOptions /// public const int DefaultServerLackOfAckTimeoutMs = 60_000; + /// + /// Default: 5,000 ms (5 seconds). + /// + public const int DefaultCallbackMaxWaitTimeMs = 5_000; + /// /// Maximum number of records that can be in flight. /// Higher values improve throughput but use more memory. @@ -145,7 +150,7 @@ public sealed class Builder internal int FlushTimeoutMs { get; set; } = DefaultFlushTimeoutMs; internal int ServerLackOfAckTimeoutMs { get; set; } = DefaultServerLackOfAckTimeoutMs; internal long? StreamPausedMaxWaitTimeMs { get; set; } - internal long? CallbackMaxWaitTimeMs { get; set; } + internal long? CallbackMaxWaitTimeMs { get; set; } = DefaultCallbackMaxWaitTimeMs; internal AckOnAckDelegate? OnAck { get; set; } internal AckOnErrorDelegate? OnError { get; set; } internal object? AckUserData { get; set; } diff --git a/dotnet/src/Databricks.Zerobus/ZerobusSdk.cs b/dotnet/src/Databricks.Zerobus/ZerobusSdk.cs index 8e999ea9..b904ad12 100644 --- a/dotnet/src/Databricks.Zerobus/ZerobusSdk.cs +++ b/dotnet/src/Databricks.Zerobus/ZerobusSdk.cs @@ -516,7 +516,7 @@ internal static CArrowStreamConfigurationOptions ToNative(this ArrowStreamConfig FlushTimeoutMs = (ulong)opts.FlushTimeoutMs, ConnectionTimeoutMs = (ulong)opts.ConnectionTimeoutMs, IpcCompression = (int)opts.IpcCompression, - StreamPausedMaxWaitTimeMs = (ulong)opts.StreamPausedMaxWaitTimeMs, + StreamPausedMaxWaitTimeMs = opts.StreamPausedMaxWaitTimeMs, }; } } diff --git a/dotnet/tests/Databricks.Zerobus.IntegrationTests/NativeLoadSmokeTest.cs b/dotnet/tests/Databricks.Zerobus.IntegrationTests/NativeLoadSmokeTest.cs index c53ee446..da991ea3 100644 --- a/dotnet/tests/Databricks.Zerobus.IntegrationTests/NativeLoadSmokeTest.cs +++ b/dotnet/tests/Databricks.Zerobus.IntegrationTests/NativeLoadSmokeTest.cs @@ -1,4 +1,5 @@ using System.Runtime.InteropServices; +using Databricks.Zerobus; using Databricks.Zerobus.Native; using Xunit; @@ -48,6 +49,39 @@ public void GetDefaultConfig_ReturnsValidStruct() Assert.True(config.MaxInflightRequests > 0); } + [Fact] + public void Defaults_MatchNativeConfig() + { + if (!NativeLibraryHelper.IsNativeLibraryAvailable()) + return; + + NativeLibraryResolver.EnsureLoaded(); + + var native = NativeMethods.zerobus_get_default_config(); + Assert.Equal(StreamConfigurationOptions.DefaultMaxInflightRecords, (int)native.MaxInflightRequests); + Assert.Equal(StreamConfigurationOptions.DefaultRecovery, native.Recovery); + Assert.Equal(StreamConfigurationOptions.DefaultRecoveryTimeoutMs, (int)native.RecoveryTimeoutMs); + Assert.Equal(StreamConfigurationOptions.DefaultRecoveryBackoffMs, (int)native.RecoveryBackoffMs); + Assert.Equal(StreamConfigurationOptions.DefaultRecoveryRetries, (int)native.RecoveryRetries); + Assert.Equal(StreamConfigurationOptions.DefaultServerLackOfAckTimeoutMs, (int)native.ServerLackOfAckTimeoutMs); + Assert.Equal(StreamConfigurationOptions.DefaultFlushTimeoutMs, (int)native.FlushTimeoutMs); + Assert.True(native.HasCallbackMaxWaitTimeMs, "Native default should have callback max wait time set"); + Assert.Equal(StreamConfigurationOptions.DefaultCallbackMaxWaitTimeMs, (long)native.CallbackMaxWaitTimeMs); + } + + [Fact] + public void ArrowDefaults_MatchNativeConfig() + { + if (!NativeLibraryHelper.IsNativeLibraryAvailable()) + return; + + NativeLibraryResolver.EnsureLoaded(); + + var native = NativeMethods.zerobus_arrow_get_default_config(); + Assert.Equal(ArrowStreamConfigurationOptions.DefaultMaxInflightBatches, (int)native.MaxInflightBatches); + Assert.Equal(ArrowStreamConfigurationOptions.DefaultStreamPausedMaxWaitTimeMs, native.StreamPausedMaxWaitTimeMs); + } + [Fact] public void BuildSdk_WithTlsDisabled_Succeeds() { diff --git a/dotnet/tests/Databricks.Zerobus.Tests/ArrowStreamConfigurationOptionsTests.cs b/dotnet/tests/Databricks.Zerobus.Tests/ArrowStreamConfigurationOptionsTests.cs index 6a1fd191..af601c1e 100644 --- a/dotnet/tests/Databricks.Zerobus.Tests/ArrowStreamConfigurationOptionsTests.cs +++ b/dotnet/tests/Databricks.Zerobus.Tests/ArrowStreamConfigurationOptionsTests.cs @@ -9,9 +9,10 @@ public void Default_ReturnsOptionsWithSensibleDefaults() { var opts = ArrowStreamConfigurationOptions.Default; - Assert.Equal(10_000, opts.MaxInflightBatches); + Assert.Equal(1_000, opts.MaxInflightBatches); Assert.True(opts.Recovery); Assert.Equal(30_000, opts.ConnectionTimeoutMs); + Assert.Equal(-1, opts.StreamPausedMaxWaitTimeMs); Assert.Equal(IPCCompressionType.None, opts.IpcCompression); } diff --git a/dotnet/tests/Databricks.Zerobus.Tests/StreamConfigurationOptionsTests.cs b/dotnet/tests/Databricks.Zerobus.Tests/StreamConfigurationOptionsTests.cs index bbfded91..f1371e89 100644 --- a/dotnet/tests/Databricks.Zerobus.Tests/StreamConfigurationOptionsTests.cs +++ b/dotnet/tests/Databricks.Zerobus.Tests/StreamConfigurationOptionsTests.cs @@ -17,7 +17,7 @@ public void Default_ReturnsOptionsWithSensibleDefaults() Assert.Equal(300_000, opts.FlushTimeoutMs); Assert.Equal(60_000, opts.ServerLackOfAckTimeoutMs); Assert.Null(opts.StreamPausedMaxWaitTimeMs); - Assert.Null(opts.CallbackMaxWaitTimeMs); + Assert.Equal(5_000, opts.CallbackMaxWaitTimeMs); Assert.Null(opts.OnAck); Assert.Null(opts.OnError); }